| 1 | """Contract tests for the restored first-run NUX wizard in SKILL.md. |
| 2 | |
| 3 | Step 0 has two branches: a **Claude Code Modal Flow** (AskUserQuestion-driven, |
| 4 | the restored v3.0.0 NUX) and a **Non-Modal Prose Flow** for hosts without modals |
| 5 | (OpenClaw, Codex, Cursor, Gemini CLI). These tests assert the structural |
| 6 | guarantees of both branches, plus the cross-cutting copy rules: the hard |
| 7 | "Step 0 before Step 1" gate, Digg threaded alongside yt-dlp, the 10,000-free-calls |
| 8 | credit count, and Threads/Pinterest kept out of the onboarding offers. They read |
| 9 | SKILL.md as text - the model's runtime contract - matching |
| 10 | tests/test_runtime_preflight_contract.py. |
| 11 | |
| 12 | These lock the flow against silent re-erosion (the failure mode that orphaned the |
| 13 | wizard in PR #659 and flattened it before this restoration). |
| 14 | """ |
| 15 | |
| 16 | import unittest |
| 17 | from pathlib import Path |
| 18 | |
| 19 | from lib import setup_wizard |
| 20 | |
| 21 | ROOT = Path(__file__).resolve().parents[1] |
| 22 | SKILL_MD = ROOT / "skills" / "last30days" / "SKILL.md" |
| 23 | |
| 24 | |
| 25 | class TestOnboardingContract(unittest.TestCase): |
| 26 | def setUp(self): |
| 27 | self.text = SKILL_MD.read_text(encoding="utf-8") |
| 28 | # Scope assertions to Step 0 so generic substrings elsewhere in the file |
| 29 | # do not satisfy ordering/presence checks. |
| 30 | start = self.text.index("## Step 0: First-Run Setup Wizard") |
| 31 | end = self.text.index("## CRITICAL: Parse User Intent", start) |
| 32 | self.step0 = self.text[start:end] |
| 33 | # Branch slices. |
| 34 | modal_start = self.step0.index("### Claude Code Modal Flow") |
| 35 | prose_start = self.step0.index("### Non-Modal Prose Flow") |
| 36 | manual_start = self.step0.index("### Manual Setup Guide") |
| 37 | self.modal = self.step0[modal_start:prose_start] |
| 38 | self.prose = self.step0[prose_start:manual_start] |
| 39 | self.manual = self.step0[manual_start:] |
| 40 | |
| 41 | # --- Platform split + hard gate --- |
| 42 | |
| 43 | def test_platform_split_present(self): |
| 44 | """Step 0 routes modal-capable hosts and prose hosts to distinct flows.""" |
| 45 | self.assertIn("Platform split", self.step0) |
| 46 | self.assertIn("### Claude Code Modal Flow", self.step0) |
| 47 | self.assertIn("### Non-Modal Prose Flow", self.step0) |
| 48 | |
| 49 | def test_hard_gate_step0_before_step1(self): |
| 50 | """The erosion-resistant gate that orphaned the wizard in #659 is restored.""" |
| 51 | self.assertIn("ALWAYS execute Step 0 BEFORE Step 1", self.step0) |
| 52 | |
| 53 | # --- Modal flow: the restored NUX, stages in order --- |
| 54 | |
| 55 | def test_modal_flow_stage_order(self): |
| 56 | """Welcome -> setup modal -> cookie consent -> SC offer -> opt-in -> picker.""" |
| 57 | anchors = [ |
| 58 | "Welcome to /last30days!", # welcome pitch, embedded in the setup modal |
| 59 | "How would you like to set up?", |
| 60 | "your browser's x.com cookies", # cookie-consent modal |
| 61 | "Want to add TikTok and Instagram?", # SC offer |
| 62 | "Which ScrapeCreators sources?", # source opt-in |
| 63 | "What do you want to research first?", # topic picker |
| 64 | ] |
| 65 | idxs = [self.modal.find(a) for a in anchors] |
| 66 | for a, i in zip(anchors, idxs): |
| 67 | self.assertGreater(i, -1, f"modal flow missing stage anchor: {a!r}") |
| 68 | self.assertEqual(idxs, sorted(idxs), "modal flow stages are out of order") |
| 69 | |
| 70 | def test_modal_uses_askuserquestion(self): |
| 71 | self.assertIn("AskUserQuestion", self.modal) |
| 72 | |
| 73 | def test_cookie_consent_names_all_installed_clis(self): |
| 74 | """The cookie-consent modal must not frame X cookies as instead-of the CLIs, |
| 75 | and must name arXiv + Techmeme (not just 'YouTube + Digg') since auto-setup |
| 76 | installs all four regardless of the cookie choice.""" |
| 77 | consent = self.modal[self.modal.find("your browser's x.com cookies"):] |
| 78 | consent = consent[: consent.find("Full Disk Access")] # bound to the consent modal |
| 79 | for cli in ("yt-dlp", "Digg", "arXiv", "Techmeme"): |
| 80 | self.assertIn(cli, consent, cli) |
| 81 | # The "skip X" option still installs the CLIs (not framed as X-or-CLIs). |
| 82 | self.assertIn("Skip X - just the CLIs", consent) |
| 83 | |
| 84 | def test_github_option_advertises_auto_clipboard(self): |
| 85 | """The recommended GitHub option tells the user the code is auto-copied to |
| 86 | their clipboard, so they just paste it.""" |
| 87 | self.assertIn("clipboard automatically", self.modal) |
| 88 | |
| 89 | def test_modal_cookie_consent_before_setup(self): |
| 90 | consent = self.modal.find("your browser's x.com cookies") |
| 91 | setup = self.modal.find("last30days.py setup") |
| 92 | self.assertGreater(consent, -1, "no cookie-consent modal in modal flow") |
| 93 | self.assertGreater(setup, -1, "no setup invocation in modal flow") |
| 94 | self.assertLess(consent, setup, "cookie consent must precede setup in modal flow") |
| 95 | |
| 96 | def test_topic_picker_skips_when_topic_supplied(self): |
| 97 | """The picker documents skipping when the user already gave a topic.""" |
| 98 | self.assertIn("What do you want to research first?", self.modal) |
| 99 | self.assertIn("SKIP this picker", self.modal) |
| 100 | |
| 101 | # --- Prose flow: same work, modal-free --- |
| 102 | |
| 103 | def test_prose_flow_has_no_modals(self): |
| 104 | self.assertNotIn("AskUserQuestion", self.prose) |
| 105 | |
| 106 | def test_prose_cookie_consent_before_setup(self): |
| 107 | consent = self.prose.find("Cookie consent") |
| 108 | setup = self.prose.find("last30days.py setup") |
| 109 | self.assertGreater(consent, -1, "no cookie-consent step in prose flow") |
| 110 | self.assertGreater(setup, -1, "no setup invocation in prose flow") |
| 111 | self.assertLess(consent, setup, "cookie consent must precede setup in prose flow") |
| 112 | |
| 113 | def test_prose_decline_uses_from_browser_off(self): |
| 114 | self.assertIn("FROM_BROWSER=off", self.prose) |
| 115 | |
| 116 | # --- Full Disk Access remediation (both branches) --- |
| 117 | |
| 118 | def test_full_disk_access_remediation_present(self): |
| 119 | self.assertIn("Permission denied reading Cookies.binarycookies", self.modal) |
| 120 | self.assertIn("Full Disk Access", self.modal) |
| 121 | self.assertIn("Permission denied reading Cookies.binarycookies", self.prose) |
| 122 | self.assertIn("Full Disk Access", self.prose) |
| 123 | |
| 124 | def test_skip_path_writes_setup_complete(self): |
| 125 | """The 'Skip for now' setup choice must write SETUP_COMPLETE or the wizard loops.""" |
| 126 | skip_idx = self.modal.find("If the user picks Skip for now") |
| 127 | self.assertGreater(skip_idx, -1, "no Skip-for-now handling in modal flow") |
| 128 | # The skip branch must persist the completion flag in its own paragraph. |
| 129 | skip_para = self.modal[skip_idx:skip_idx + 400] |
| 130 | self.assertIn("SETUP_COMPLETE=true", skip_para) |
| 131 | |
| 132 | # --- ScrapeCreators signup + persisted edge case --- |
| 133 | |
| 134 | def test_scrapecreators_signup_present_both_branches(self): |
| 135 | self.assertIn("setup --github", self.modal) |
| 136 | self.assertIn("setup --github", self.prose) |
| 137 | |
| 138 | def test_persisted_false_edge_case_documented(self): |
| 139 | self.assertIn('"persisted": false', self.step0) |
| 140 | |
| 141 | # --- Digg threaded alongside yt-dlp everywhere it appears --- |
| 142 | |
| 143 | def test_digg_threaded_with_ytdlp(self): |
| 144 | self.assertIn("Digg", self.modal) |
| 145 | self.assertIn("Digg", self.prose) |
| 146 | self.assertIn("Digg", self.manual) |
| 147 | # The Auto-setup modal option names every installed CLI, not just two. |
| 148 | self.assertIn("yt-dlp (YouTube), Digg, arXiv, Techmeme", self.modal) |
| 149 | |
| 150 | # --- Credit count = 10,000, no conflicting numbers in onboarding --- |
| 151 | |
| 152 | def test_credit_count_is_10000(self): |
| 153 | self.assertIn("10,000 free calls", self.step0) |
| 154 | self.assertNotIn("1,000 free", self.step0) |
| 155 | self.assertNotIn("1000 free credit", self.step0) |
| 156 | self.assertNotIn("1000 credits", self.step0) |
| 157 | self.assertNotIn("100 free call", self.step0) |
| 158 | |
| 159 | # --- Threads/Pinterest live ONLY in the Step 5 "Everything" opt-in --- |
| 160 | |
| 161 | def _modal_step5(self): |
| 162 | start = self.modal.index("**Step 5:") |
| 163 | end = self.modal.index("**Step 6:", start) |
| 164 | return self.modal[start:end] |
| 165 | |
| 166 | def _modal_before_step5(self): |
| 167 | # Welcome (Step 1) through the Step 4 ScrapeCreators offer. |
| 168 | return self.modal[: self.modal.index("**Step 5:")] |
| 169 | |
| 170 | def test_threads_pinterest_only_in_step5_everything(self): |
| 171 | """Threads/Pinterest are offered in the Step 5 Everything tier, and |
| 172 | |
| 173 | must NOT appear in the welcome or the Step 4 offer (where they would |
| 174 | read as default-on). They are opt-in via INCLUDE_SOURCES. |
| 175 | """ |
| 176 | step5 = self._modal_step5() |
| 177 | self.assertIn("Threads", step5) |
| 178 | self.assertIn("Pinterest", step5) |
| 179 | before = self._modal_before_step5() |
| 180 | self.assertNotIn("Threads", before) |
| 181 | self.assertNotIn("Pinterest", before) |
| 182 | |
| 183 | def test_offer_copy_names_comments_and_auto_enrichment(self): |
| 184 | """The Step 4 offer states comments are part of the default value and |
| 185 | describes the key's real Reddit/YouTube roles (empty-path Reddit |
| 186 | search backfill + yt-dlp transcript backstop) — not rate-limit |
| 187 | escalation or SC Reddit comment enrichment on the free path.""" |
| 188 | before = self._modal_before_step5() |
| 189 | self.assertIn("comments", before.lower()) |
| 190 | self.assertIn("Reddit", before) |
| 191 | self.assertIn("YouTube", before) |
| 192 | self.assertIn("10,000 free calls", before) |
| 193 | # Empty-only search backup (not transport/rate-limit escalation). |
| 194 | self.assertIn("returns no items", before) |
| 195 | self.assertNotIn("when they hit rate limits", before) |
| 196 | # Free-path comments are shreddit; do not claim SC comment preference. |
| 197 | self.assertNotIn("prefers ScrapeCreators for Reddit", before) |
| 198 | self.assertNotIn("enriches Reddit comments", before) |
| 199 | |
| 200 | def test_step5_does_not_claim_merged_reddit_auto_enrichment(self): |
| 201 | """Step 5 must not contradict Step 4 with 'public + ScrapeCreators' merge.""" |
| 202 | step5 = self._modal_step5() |
| 203 | self.assertNotIn("public + ScrapeCreators", step5) |
| 204 | self.assertNotIn("Reddit auto-enrichment", step5) |
| 205 | self.assertIn("empty-only", step5) |
| 206 | |
| 207 | def test_recommended_tier_writes_comments_by_default(self): |
| 208 | """Comments are the DEFAULT: the recommended option enables YouTube + |
| 209 | TikTok + Instagram comments (posts on -> comments on).""" |
| 210 | step5 = self._modal_step5() |
| 211 | self.assertIn( |
| 212 | "INCLUDE_SOURCES=tiktok,instagram,youtube_comments,tiktok_comments,instagram_comments", |
| 213 | step5, |
| 214 | ) |
| 215 | # There is no posts-only tier. |
| 216 | self.assertIn("recommended", step5.lower()) |
| 217 | self.assertIn("comments", step5.lower()) |
| 218 | |
| 219 | def test_everything_tier_writes_full_include_sources(self): |
| 220 | """The Everything option persists the full list incl. Threads + Pinterest.""" |
| 221 | step5 = self._modal_step5() |
| 222 | self.assertIn( |
| 223 | "INCLUDE_SOURCES=tiktok,instagram,youtube_comments,tiktok_comments,instagram_comments,threads,pinterest", |
| 224 | step5, |
| 225 | ) |
| 226 | |
| 227 | # --- Chrome-first cookie scan (U2/U3) --- |
| 228 | |
| 229 | def test_cookie_consent_leads_with_chrome(self): |
| 230 | """Both flows tell the user Chrome is checked first, with the Keychain cue.""" |
| 231 | for slice_name, slice_text in (("modal", self.modal), ("prose", self.prose)): |
| 232 | self.assertIn("Chrome", slice_text, f"{slice_name} cookie copy omits Chrome") |
| 233 | self.assertIn("Always Allow", slice_text, f"{slice_name} omits the Keychain cue") |
| 234 | |
| 235 | def test_fda_reframed_as_safari_fallback(self): |
| 236 | """Full Disk Access is framed as Safari-only, not the default path.""" |
| 237 | self.assertNotIn("scan your browser (Firefox/Safari)", self.modal) |
| 238 | |
| 239 | def test_welcome_embedded_in_modal(self): |
| 240 | """The welcome pitch lives INSIDE the setup modal (the only always-visible |
| 241 | surface), not as a separate message/command that Claude Code folds away. |
| 242 | The engine --welcome command is kept for the non-modal prose flow.""" |
| 243 | # Pitch is in the modal question. |
| 244 | self.assertIn("Welcome to /last30days!", self.modal) |
| 245 | self.assertIn("How would you like to set up?", self.modal) |
| 246 | # The modal flow explicitly does NOT run a separate --welcome command. |
| 247 | self.assertIn("Do NOT run a separate `--welcome`", self.modal) |
| 248 | # The non-modal flow still uses the engine welcome command. |
| 249 | self.assertIn("last30days.py --welcome", self.prose) |
| 250 | |
| 251 | def test_stocktwits_surfaced_as_conditional(self): |
| 252 | """StockTwits is advertised in the engine welcome as a ticker/crypto-gated |
| 253 | source (welcome text moved out of SKILL.md into the engine).""" |
| 254 | self.assertIn("StockTwits", setup_wizard.render_welcome()) |
| 255 | |
| 256 | # --- Honest GitHub device-code copy (U4/U7) --- |
| 257 | |
| 258 | def test_no_false_instant_gh_promise(self): |
| 259 | """The '~2 seconds - no browser' claim (a nonexistent code path) is gone.""" |
| 260 | self.assertNotIn("~2 seconds - no browser", self.step0) |
| 261 | self.assertNotIn("Registers via GitHub CLI in ~2 seconds", self.step0) |
| 262 | |
| 263 | def test_device_code_surfacing_orchestration_present(self): |
| 264 | """Both flows use the deterministic two-command split (start returns the |
| 265 | code fast, then poll) instead of a background-and-surface spinner.""" |
| 266 | self.assertIn("setup --github-start", self.modal) |
| 267 | self.assertIn("setup --github-poll", self.modal) |
| 268 | self.assertIn("setup --github-start", self.prose) |
| 269 | self.assertIn("setup --github-poll", self.prose) |
| 270 | |
| 271 | def test_already_registered_status_handled(self): |
| 272 | self.assertIn("already_registered", self.modal) |
| 273 | self.assertIn("already_registered", self.prose) |
| 274 | |
| 275 | # --- Welcome must render before the modal (U1) --- |
| 276 | |
| 277 | def test_welcome_pitch_is_in_the_modal_question(self): |
| 278 | """The welcome pitch names the core sources inside the modal question, so |
| 279 | the user sees it without expanding folded tool output. The old skip-prone |
| 280 | 'IMMEDIATELY call AskUserQuestion' wording stays gone.""" |
| 281 | # Pitch names the core sources right in the modal. |
| 282 | for source in ("Reddit", "X,", "YouTube", "TikTok"): |
| 283 | self.assertIn(source, self.modal, source) |
| 284 | self.assertNotIn("Then IMMEDIATELY call AskUserQuestion", self.modal) |
| 285 | |
| 286 | # --- Device code surfaced with a clipboard-paste hint (U3) --- |
| 287 | |
| 288 | def test_device_code_clipboard_paste_instruction(self): |
| 289 | """The GitHub flow tells the user the code is on their clipboard to paste, |
| 290 | |
| 291 | and makes surfacing the code a required step (the bug the user hit). |
| 292 | """ |
| 293 | self.assertIn("on your clipboard", self.modal) |
| 294 | # Surfacing the code is a required, explicit step in the new split flow. |
| 295 | self.assertIn("SHOW THE CODE", self.modal) |
| 296 | self.assertIn("just paste", self.modal) |
| 297 | |
| 298 | # --- Honest 'authorized but no key' branch, distinct from auth-failed (U4) --- |
| 299 | |
| 300 | def test_authorized_but_no_key_branch_present(self): |
| 301 | """A key-fetch failure after successful auth is handled honestly (likely |
| 302 | |
| 303 | an already-linked account), not lumped into 'auth didn't complete'. |
| 304 | """ |
| 305 | for slice_name, slice_text in (("modal", self.modal), ("prose", self.prose)): |
| 306 | self.assertIn("Authorized but failed to fetch API key", slice_text, slice_name) |
| 307 | self.assertIn("already linked", slice_text, slice_name) |
| 308 | |
| 309 | # --- Legacy guarantees retained --- |
| 310 | |
| 311 | def test_old_silent_wizard_instruction_removed(self): |
| 312 | self.assertNotIn("Follow the wizard's prompts end-to-end", self.text) |
| 313 | |
| 314 | def test_consent_is_conversational_contract_documented(self): |
| 315 | self.assertIn("Named onboarding contract", self.step0) |
| 316 | self.assertIn("non-interactive subprocess", self.step0) |
| 317 | |
| 318 | |
| 319 | if __name__ == "__main__": |
| 320 | unittest.main() |
| 321 |