返回 last30days-skill
test_onboarding_contract.py
根目录 / tests / test_onboarding_contract.py
1 """Contract tests for the restored first-run NUX wizard in SKILL.md.
2
3 Step 0 has three branches: a **Claude Code Modal Flow** (AskUserQuestion-driven,
4 the restored v3.0.0 NUX), a **Non-Modal Prose Flow** for hosts without modals
5 (OpenClaw, Codex, Cursor, Gemini CLI), and a **Grok Bot Prose Flow** (the X
6 connector lane first, keys written only through the engine, no browser-session
7 reads). These tests assert the structural guarantees of the branches, plus the cross-cutting copy rules: the hard
8 "Step 0 before Step 1" gate, Digg threaded alongside yt-dlp, the 10,000-free-calls
9 credit count, and Threads/Pinterest kept out of the onboarding offers. They read
10 SKILL.md as text - the model's runtime contract - matching
11 tests/test_runtime_preflight_contract.py.
12
13 These lock the flow against silent re-erosion (the failure mode that orphaned the
14 wizard in PR #659 and flattened it before this restoration).
15 """
16
17 import unittest
18 from pathlib import Path
19
20 from lib import setup_wizard
21
22 ROOT = Path(__file__).resolve().parents[1]
23 SKILL_MD = ROOT / "skills" / "last30days" / "SKILL.md"
24 AGENTS_MD = ROOT / "AGENTS.md"
25
26
27 class TestOnboardingContract(unittest.TestCase):
28 def setUp(self):
29 self.text = SKILL_MD.read_text(encoding="utf-8")
30 # Scope assertions to Step 0 so generic substrings elsewhere in the file
31 # do not satisfy ordering/presence checks.
32 start = self.text.index("## Step 0: First-Run Setup Wizard")
33 end = self.text.index("## CRITICAL: Parse User Intent", start)
34 self.step0 = self.text[start:end]
35 # Branch slices.
36 modal_start = self.step0.index("### Claude Code Modal Flow")
37 prose_start = self.step0.index("### Non-Modal Prose Flow")
38 grok_start = self.step0.index("### Grok Bot Prose Flow")
39 manual_start = self.step0.index("### Manual Setup Guide")
40 self.modal = self.step0[modal_start:prose_start]
41 self.prose = self.step0[prose_start:grok_start]
42 self.grok = self.step0[grok_start:manual_start]
43 self.manual = self.step0[manual_start:]
44
45 # --- Platform split + hard gate ---
46
47 def test_platform_split_present(self):
48 """Step 0 routes modal-capable hosts, prose hosts, and Grok Bot to
49 three distinct flows, in that order."""
50 self.assertIn("Platform split", self.step0)
51 self.assertIn("### Claude Code Modal Flow", self.step0)
52 self.assertIn("### Non-Modal Prose Flow", self.step0)
53 self.assertIn("### Grok Bot Prose Flow", self.step0)
54 split = self.step0[self.step0.index("Platform split"):self.step0.index("### Claude Code Modal Flow")]
55 self.assertIn("Grok Bot Prose Flow", split)
56 self.assertEqual(3, len([h for h in ("### Claude Code Modal Flow", "### Non-Modal Prose Flow", "### Grok Bot Prose Flow") if h in self.step0]))
57 self.assertLess(self.step0.index("### Non-Modal Prose Flow"), self.step0.index("### Grok Bot Prose Flow"))
58
59 def test_agents_md_names_three_step0_branches(self):
60 """AGENTS.md's onboarding rule and this contract move together."""
61 agents = AGENTS_MD.read_text(encoding="utf-8")
62 self.assertIn("Step 0 has THREE branches", agents)
63 self.assertNotIn("Step 0 has TWO branches", agents)
64 for name in ("Claude Code Modal Flow", "Non-Modal Prose Flow", "Grok Bot Prose Flow"):
65 self.assertIn(name, agents, name)
66
67 def test_grok_flow_is_prose_and_connector_first(self):
68 """The third branch has no modals, leads with the X connector, and
69 never routes through a browser-session step."""
70 self.assertNotIn("AskUserQuestion", self.grok)
71 self.assertNotIn("cookie", self.grok.lower())
72 self.assertLess(self.grok.index("search_posts_all"), self.grok.index("X_BEARER_TOKEN"))
73 self.assertIn("setup --store-key", self.grok)
74
75 def test_first_run_flows_do_not_invoke_preflight(self):
76 """Status and permission inspection are not a required first-run beat.
77 `--preflight` stays an opt-in inspector; Step 0 must not dump `.env`."""
78 self.assertNotIn("--preflight", self.modal)
79 self.assertNotIn("--preflight", self.prose)
80 self.assertNotIn("--preflight", self.grok)
81 self.assertIn("Do not run it as a required first-run step", self.step0)
82 self.assertIn("Do not print `.env` contents or credential values", self.step0)
83
84 def test_first_run_gate_defers_to_step0_credential_sources(self):
85 """The cheap SETUP_COMPLETE grep is not itself a first-run verdict."""
86 start = self.text.index("**FIRST-RUN GATE")
87 end = self.text.index("\n## Step 0: First-Run Setup Wizard")
88 gate = self.text[start:end]
89 self.assertIn("FIRST_RUN_DETECTED", gate)
90 self.assertIn("A missing `.env` alone is not a first run", gate)
91 self.assertIn("That section decides first-run from every credential source", gate)
92
93 def test_complete_does_not_treat_setup_stdout_as_source_list(self):
94 self.assertIn(
95 "Setup stdout is what this run installed, not the runtime source list",
96 self.prose,
97 )
98 self.assertIn(
99 "Setup stdout is what this run installed, not the runtime source list",
100 self.grok,
101 )
102
103 def test_hard_gate_step0_before_step1(self):
104 """The erosion-resistant gate that orphaned the wizard in #659 is restored."""
105 self.assertIn("ALWAYS execute Step 0 BEFORE Step 1", self.step0)
106
107 def test_waiting_topic_continues_after_x_decline_or_setup_skip(self):
108 """Declining optional X access must never strand the requested topic."""
109 self.assertIn("RESEARCH CONTINUATION OVERRIDE", self.step0)
110 self.assertIn("declining or skipping X must never stop", self.step0)
111 self.assertIn("immediately research it with `--no-browser-cookies`", self.modal)
112 self.assertIn("immediately research it with `--no-browser-cookies`", self.prose)
113 self.assertIn("a skip or no answer is never consent", self.step0)
114
115 def test_waiting_topic_defers_optional_prompts_and_x_retry(self):
116 self.assertIn("skip the ScrapeCreators offer", self.step0)
117 self.assertIn("Do not ask another X question in the same run", self.step0)
118 self.assertIn("Offer ONE retry only when no research topic is waiting", self.modal)
119 self.assertIn("Offer ONE retry only when no research topic is waiting", self.prose)
120
121 def test_deferred_onboarding_resumes_after_the_findings(self):
122 """Deferral is same-run only: SETUP_COMPLETE=true means later runs skip
123 Step 0, so a skip-X-with-topic run must itself resume the ScrapeCreators
124 offer after the findings or the offer is dropped forever."""
125 self.assertIn("RESUME the deferred onboarding in the SAME run", self.step0)
126 self.assertIn(
127 "this run is the only chance to make the offer", self.step0
128 )
129 # Both flows: Skip-for-now, Skip-X modal option, and the prose no-path
130 # all resume the deferred offer in the same run after the findings.
131 self.assertEqual(
132 2,
133 self.modal.count(
134 "then resume Step 4 (and Step 5 if a key is saved) in the same run"
135 ),
136 )
137 self.assertIn("then resume the deferred onboarding in the same run", self.prose)
138 # The resume never turns back into a second X consent ask.
139 self.assertIn("the resume never re-asks X/browser-cookie consent", self.step0)
140 self.assertIn("Do not re-ask cookie consent as part of the resume", self.prose)
141
142 def test_x_handle_resolution_and_plan_follow_active_sources(self):
143 self.assertIn("If `ACTIVE_SOURCES_LIST` contains `x`", self.text)
144 self.assertIn("every applicable source from `ACTIVE_SOURCES_LIST`", self.text)
145 self.assertIn("Preserve X whenever it is active", self.text)
146
147 def test_post_report_x_note_is_non_blocking(self):
148 self.assertNotIn("Just-in-time X unlock", self.text)
149 self.assertIn("Optional X omission", self.text)
150 self.assertIn("finish the useful findings first", self.text)
151
152 # --- Modal flow: the restored NUX, stages in order ---
153
154 def test_modal_flow_stage_order(self):
155 """Welcome -> setup modal -> cookie consent -> SC offer -> opt-in -> picker."""
156 anchors = [
157 "Welcome to /last30days!", # welcome pitch, embedded in the setup modal
158 "How would you like to set up?",
159 "your browser's x.com cookies", # cookie-consent modal
160 "Want to add TikTok and Instagram?", # SC offer
161 "Which ScrapeCreators sources?", # source opt-in
162 "What do you want to research first?", # topic picker
163 ]
164 idxs = [self.modal.find(a) for a in anchors]
165 for a, i in zip(anchors, idxs):
166 self.assertGreater(i, -1, f"modal flow missing stage anchor: {a!r}")
167 self.assertEqual(idxs, sorted(idxs), "modal flow stages are out of order")
168
169 def test_modal_uses_askuserquestion(self):
170 self.assertIn("AskUserQuestion", self.modal)
171
172 def test_cookie_consent_names_all_installed_clis(self):
173 """The cookie-consent modal must not frame X cookies as instead-of the CLIs,
174 and must name arXiv + Techmeme (not just 'YouTube + Digg') since auto-setup
175 installs all four regardless of the cookie choice."""
176 consent = self.modal[self.modal.find("your browser's x.com cookies"):]
177 consent = consent[: consent.find("Full Disk Access")] # bound to the consent modal
178 for cli in ("yt-dlp", "Digg", "arXiv", "Techmeme"):
179 self.assertIn(cli, consent, cli)
180 # The "skip X" option still installs the CLIs (not framed as X-or-CLIs).
181 self.assertIn("Skip X - just the CLIs", consent)
182
183 def test_github_option_advertises_auto_clipboard(self):
184 """The recommended GitHub option tells the user the code is auto-copied to
185 their clipboard, so they just paste it."""
186 self.assertIn("clipboard automatically", self.modal)
187
188 def test_modal_cookie_consent_before_setup(self):
189 consent = self.modal.find("your browser's x.com cookies")
190 setup = self.modal.find("last30days.py setup")
191 self.assertGreater(consent, -1, "no cookie-consent modal in modal flow")
192 self.assertGreater(setup, -1, "no setup invocation in modal flow")
193 self.assertLess(consent, setup, "cookie consent must precede setup in modal flow")
194
195 def test_topic_picker_skips_when_topic_supplied(self):
196 """The picker documents skipping when the user already gave a topic."""
197 self.assertIn("What do you want to research first?", self.modal)
198 self.assertIn("SKIP this picker", self.modal)
199
200 # --- Prose flow: same work, modal-free ---
201
202 def test_prose_flow_has_no_modals(self):
203 self.assertNotIn("AskUserQuestion", self.prose)
204
205 def test_prose_cookie_consent_before_setup(self):
206 consent = self.prose.find("Cookie consent")
207 setup = self.prose.find("last30days.py setup")
208 self.assertGreater(consent, -1, "no cookie-consent step in prose flow")
209 self.assertGreater(setup, -1, "no setup invocation in prose flow")
210 self.assertLess(consent, setup, "cookie consent must precede setup in prose flow")
211
212 def test_prose_decline_uses_from_browser_off(self):
213 self.assertIn("FROM_BROWSER=off", self.prose)
214
215 # --- Full Disk Access remediation (both branches) ---
216
217 def test_full_disk_access_remediation_present(self):
218 self.assertIn("Permission denied reading Cookies.binarycookies", self.modal)
219 self.assertIn("Full Disk Access", self.modal)
220 self.assertIn("Permission denied reading Cookies.binarycookies", self.prose)
221 self.assertIn("Full Disk Access", self.prose)
222
223 def test_skip_path_writes_setup_complete(self):
224 """The 'Skip for now' setup choice must write SETUP_COMPLETE or the wizard loops."""
225 skip_idx = self.modal.find("If the user picks Skip for now")
226 self.assertGreater(skip_idx, -1, "no Skip-for-now handling in modal flow")
227 # The skip branch must persist the completion flag in its own paragraph.
228 skip_para = self.modal[skip_idx:skip_idx + 400]
229 self.assertIn("SETUP_COMPLETE=true", skip_para)
230
231 # --- ScrapeCreators signup + persisted edge case ---
232
233 def test_scrapecreators_signup_present_both_branches(self):
234 self.assertIn("setup --github", self.modal)
235 self.assertIn("setup --github", self.prose)
236
237 def test_persisted_false_edge_case_documented(self):
238 self.assertIn('"persisted": false', self.step0)
239
240 # --- Digg threaded alongside yt-dlp everywhere it appears ---
241
242 def test_digg_threaded_with_ytdlp(self):
243 self.assertIn("Digg", self.modal)
244 self.assertIn("Digg", self.prose)
245 self.assertIn("Digg", self.manual)
246 # The Auto-setup modal option names every installed CLI, not just two.
247 self.assertIn("yt-dlp (YouTube), Digg, arXiv, Techmeme", self.modal)
248
249 # --- Credit count = 10,000, no conflicting numbers in onboarding ---
250
251 def test_credit_count_is_10000(self):
252 self.assertIn("10,000 free calls", self.step0)
253 self.assertNotIn("1,000 free", self.step0)
254 self.assertNotIn("1000 free credit", self.step0)
255 self.assertNotIn("1000 credits", self.step0)
256 self.assertNotIn("100 free call", self.step0)
257
258 # --- Threads/Pinterest live ONLY in the Step 5 "Everything" opt-in ---
259
260 def _modal_step5(self):
261 start = self.modal.index("**Step 5:")
262 end = self.modal.index("**Step 6:", start)
263 return self.modal[start:end]
264
265 def _modal_before_step5(self):
266 # Welcome (Step 1) through the Step 4 ScrapeCreators offer.
267 return self.modal[: self.modal.index("**Step 5:")]
268
269 def test_threads_pinterest_only_in_step5_everything(self):
270 """Threads/Pinterest are offered in the Step 5 Everything tier, and
271
272 must NOT appear in the welcome or the Step 4 offer (where they would
273 read as default-on). They are opt-in via INCLUDE_SOURCES.
274 """
275 step5 = self._modal_step5()
276 self.assertIn("Threads", step5)
277 self.assertIn("Pinterest", step5)
278 before = self._modal_before_step5()
279 self.assertNotIn("Threads", before)
280 self.assertNotIn("Pinterest", before)
281
282 def test_offer_copy_names_comments_and_auto_enrichment(self):
283 """The Step 4 offer states comments are part of the default value and
284 describes the key's real Reddit/YouTube roles (Reddit search backfill
285 below the 5-item floor + yt-dlp transcript backstop), not rate-limit
286 escalation or SC Reddit comment enrichment on the free path."""
287 before = self._modal_before_step5()
288 self.assertIn("comments", before.lower())
289 self.assertIn("Reddit", before)
290 self.assertIn("YouTube", before)
291 self.assertIn("10,000 free calls", before)
292 # Floor-triggered search backfill with the 0 opt-out (not
293 # transport/rate-limit escalation, and no longer empty-only).
294 self.assertIn("fewer than 5 items", before)
295 self.assertIn("LAST30DAYS_REDDIT_SC_MIN_ITEMS=0", before)
296 self.assertNotIn("returns no items", before)
297 self.assertNotIn("empty-only", before)
298 self.assertNotIn("when they hit rate limits", before)
299 # Free-path comments are shreddit; do not claim SC comment preference.
300 self.assertNotIn("prefers ScrapeCreators for Reddit", before)
301 self.assertNotIn("enriches Reddit comments", before)
302
303 def test_step5_does_not_claim_merged_reddit_auto_enrichment(self):
304 """Step 5 must not contradict Step 4 with 'public + ScrapeCreators' merge."""
305 step5 = self._modal_step5()
306 self.assertNotIn("public + ScrapeCreators", step5)
307 self.assertNotIn("Reddit auto-enrichment", step5)
308 self.assertIn("fewer than 5 items", step5)
309 self.assertNotIn("empty-only", step5)
310
311 def test_recommended_tier_writes_comments_by_default(self):
312 """Comments are the DEFAULT: the recommended option enables YouTube +
313 TikTok + Instagram comments (posts on -> comments on)."""
314 step5 = self._modal_step5()
315 self.assertIn(
316 "INCLUDE_SOURCES=tiktok,instagram,youtube_comments,tiktok_comments,instagram_comments",
317 step5,
318 )
319 # There is no posts-only tier.
320 self.assertIn("recommended", step5.lower())
321 self.assertIn("comments", step5.lower())
322
323 def test_everything_tier_writes_full_include_sources(self):
324 """The Everything option persists the full list incl. Threads + Pinterest."""
325 step5 = self._modal_step5()
326 self.assertIn(
327 "INCLUDE_SOURCES=tiktok,instagram,youtube_comments,tiktok_comments,instagram_comments,threads,pinterest",
328 step5,
329 )
330
331 # --- Chrome-first cookie scan (U2/U3) ---
332
333 def test_cookie_consent_leads_with_chrome(self):
334 """Both flows tell the user Chrome is checked first, with the Keychain cue."""
335 for slice_name, slice_text in (("modal", self.modal), ("prose", self.prose)):
336 self.assertIn("Chrome", slice_text, f"{slice_name} cookie copy omits Chrome")
337 self.assertIn("Always Allow", slice_text, f"{slice_name} omits the Keychain cue")
338
339 def test_fda_reframed_as_safari_fallback(self):
340 """Full Disk Access is framed as Safari-only, not the default path."""
341 self.assertNotIn("scan your browser (Firefox/Safari)", self.modal)
342
343 def test_welcome_embedded_in_modal(self):
344 """The welcome pitch lives INSIDE the setup modal (the only always-visible
345 surface), not as a separate message/command that Claude Code folds away.
346 The engine --welcome command is kept for the non-modal prose flow."""
347 # Pitch is in the modal question.
348 self.assertIn("Welcome to /last30days!", self.modal)
349 self.assertIn("How would you like to set up?", self.modal)
350 # The modal flow explicitly does NOT run a separate --welcome command.
351 self.assertIn("Do NOT run a separate `--welcome`", self.modal)
352 # The non-modal flow still uses the engine welcome command.
353 self.assertIn("last30days.py --welcome", self.prose)
354
355 def test_stocktwits_surfaced_as_conditional(self):
356 """StockTwits is advertised in the engine welcome as a ticker/crypto-gated
357 source (welcome text moved out of SKILL.md into the engine)."""
358 self.assertIn("StockTwits", setup_wizard.render_welcome())
359
360 # --- Honest GitHub device-code copy (U4/U7) ---
361
362 def test_no_false_instant_gh_promise(self):
363 """The '~2 seconds - no browser' claim (a nonexistent code path) is gone."""
364 self.assertNotIn("~2 seconds - no browser", self.step0)
365 self.assertNotIn("Registers via GitHub CLI in ~2 seconds", self.step0)
366
367 def test_device_code_surfacing_orchestration_present(self):
368 """Both flows use the deterministic two-command split (start returns the
369 code fast, then poll) instead of a background-and-surface spinner."""
370 self.assertIn("setup --github-start", self.modal)
371 self.assertIn("setup --github-poll", self.modal)
372 self.assertIn("setup --github-start", self.prose)
373 self.assertIn("setup --github-poll", self.prose)
374
375 def test_already_registered_status_handled(self):
376 self.assertIn("already_registered", self.modal)
377 self.assertIn("already_registered", self.prose)
378
379 # --- Welcome must render before the modal (U1) ---
380
381 def test_welcome_pitch_is_in_the_modal_question(self):
382 """The welcome pitch names the core sources inside the modal question, so
383 the user sees it without expanding folded tool output. The old skip-prone
384 'IMMEDIATELY call AskUserQuestion' wording stays gone."""
385 # Pitch names the core sources right in the modal.
386 for source in ("Reddit", "X,", "YouTube", "TikTok"):
387 self.assertIn(source, self.modal, source)
388 self.assertNotIn("Then IMMEDIATELY call AskUserQuestion", self.modal)
389
390 # --- Device code surfaced with a clipboard-paste hint (U3) ---
391
392 def test_device_code_clipboard_paste_instruction(self):
393 """The GitHub flow tells the user the code is on their clipboard to paste,
394
395 and makes surfacing the code a required step (the bug the user hit).
396 """
397 self.assertIn("on your clipboard", self.modal)
398 # Surfacing the code is a required, explicit step in the new split flow.
399 self.assertIn("SHOW THE CODE", self.modal)
400 self.assertIn("just paste", self.modal)
401
402 # --- Honest 'authorized but no key' branch, distinct from auth-failed (U4) ---
403
404 def test_authorized_but_no_key_branch_present(self):
405 """A key-fetch failure after successful auth is handled honestly (likely
406
407 an already-linked account), not lumped into 'auth didn't complete'.
408 """
409 for slice_name, slice_text in (("modal", self.modal), ("prose", self.prose)):
410 self.assertIn("Authorized but failed to fetch API key", slice_text, slice_name)
411 self.assertIn("already linked", slice_text, slice_name)
412
413 def test_upstream_profile_error_branch_distinct_from_already_linked(self):
414 """A ScrapeCreators /profile 5xx must not be diagnosed as already-linked (#882)."""
415 for slice_name, slice_text in (("modal", self.modal), ("prose", self.prose)):
416 self.assertIn("ScrapeCreators profile failed", slice_text, slice_name)
417 self.assertIn("upstream_error", slice_text, slice_name)
418 self.assertIn("server error", slice_text, slice_name)
419 # Guidance forbids the already-linked misdiagnosis on this path.
420 self.assertIn("do **NOT** say", slice_text, slice_name)
421 self.assertIn("already linked", slice_text, slice_name)
422
423 # --- Legacy guarantees retained ---
424
425 def test_old_silent_wizard_instruction_removed(self):
426 self.assertNotIn("Follow the wizard's prompts end-to-end", self.text)
427
428 def test_consent_is_conversational_contract_documented(self):
429 self.assertIn("Named onboarding contract", self.step0)
430 self.assertIn("non-interactive subprocess", self.step0)
431
432
433 if __name__ == "__main__":
434 unittest.main()
435
435 lines PYTHON