| 1 | """End-to-end regression test for the 2026-04-22 `Prompting GPT Image 2` bug. |
| 2 | |
| 3 | Guards the failing run's Resolved-block shape end-to-end: stubs |
| 4 | `grounding.web_search` to return the OpenAI-only subs that caused the |
| 5 | original failure, then asserts that `auto_resolve` now returns the widened |
| 6 | list and emits the expected stderr trace. |
| 7 | |
| 8 | If this test starts failing after a `scripts/lib/categories.py` edit, either |
| 9 | the fix regressed or the map intentionally dropped the `ai_image_generation` |
| 10 | category — update the test deliberately. |
| 11 | |
| 12 | Fixture reference: `tests/fixtures/prompting-gpt-image-2-resolved-block.md`. |
| 13 | """ |
| 14 | |
| 15 | import io |
| 16 | import unittest |
| 17 | from contextlib import redirect_stderr |
| 18 | from unittest.mock import patch |
| 19 | |
| 20 | from lib import resolve |
| 21 | |
| 22 | OPENAI_BRAND_SUBREDDIT_RESULTS = [ |
| 23 | { |
| 24 | "title": "r/OpenAI community hub", |
| 25 | "snippet": "Discussion at r/ChatGPT and r/singularity about GPT Image 2.", |
| 26 | "url": "https://reddit.com/r/OpenAI/", |
| 27 | }, |
| 28 | { |
| 29 | "title": "r/ChatGPTpromptengineering prompt collection", |
| 30 | "snippet": "Also see r/artificial for broader AI chatter.", |
| 31 | "url": "", |
| 32 | }, |
| 33 | ] |
| 34 | |
| 35 | EMPTY_RESULTS: list[dict] = [] |
| 36 | |
| 37 | |
| 38 | def _fake_websearch(label_to_items: dict[str, list[dict]]): |
| 39 | def _search(query, date_range, config): |
| 40 | if "subreddit" in query: |
| 41 | return label_to_items.get("subreddit", EMPTY_RESULTS), {} |
| 42 | if "news" in query: |
| 43 | return label_to_items.get("news", EMPTY_RESULTS), {} |
| 44 | if "handle" in query: |
| 45 | return label_to_items.get("x_handle", EMPTY_RESULTS), {} |
| 46 | if "github" in query: |
| 47 | return label_to_items.get("github", EMPTY_RESULTS), {} |
| 48 | return EMPTY_RESULTS, {} |
| 49 | |
| 50 | return _search |
| 51 | |
| 52 | |
| 53 | class PromptingGptImage2RegressionGuard(unittest.TestCase): |
| 54 | """The named 2026-04-22 failure mode. Resolved block must include peers.""" |
| 55 | |
| 56 | @patch("lib.resolve.grounding.web_search") |
| 57 | def test_auto_resolve_widens_to_image_gen_peers(self, mock_search): |
| 58 | mock_search.side_effect = _fake_websearch({ |
| 59 | "subreddit": OPENAI_BRAND_SUBREDDIT_RESULTS, |
| 60 | }) |
| 61 | |
| 62 | result = resolve.auto_resolve( |
| 63 | "Prompting GPT Image 2", |
| 64 | {"BRAVE_API_KEY": "fake"}, |
| 65 | ) |
| 66 | |
| 67 | subs_lower = [s.lower() for s in result["subreddits"]] |
| 68 | |
| 69 | # Original WebSearch-returned brand subs preserved |
| 70 | self.assertIn("openai", subs_lower) |
| 71 | self.assertIn("chatgpt", subs_lower) |
| 72 | self.assertIn("singularity", subs_lower) |
| 73 | |
| 74 | # At least three of the image-gen peers were added |
| 75 | expected_peers = {"stablediffusion", "midjourney", "dalle2", "aiart", "promptengineering"} |
| 76 | found_peers = expected_peers.intersection(subs_lower) |
| 77 | self.assertGreaterEqual( |
| 78 | len(found_peers), |
| 79 | 3, |
| 80 | f"Expected at least 3 image-gen peer subs, found: {found_peers}. " |
| 81 | f"Actual subs: {result['subreddits']}", |
| 82 | ) |
| 83 | |
| 84 | self.assertEqual(result["category"], "ai_image_generation") |
| 85 | |
| 86 | @patch("lib.resolve.grounding.web_search") |
| 87 | def test_stderr_contains_category_match_log_line(self, mock_search): |
| 88 | mock_search.side_effect = _fake_websearch({ |
| 89 | "subreddit": OPENAI_BRAND_SUBREDDIT_RESULTS, |
| 90 | }) |
| 91 | |
| 92 | buf = io.StringIO() |
| 93 | with redirect_stderr(buf): |
| 94 | resolve.auto_resolve( |
| 95 | "Prompting GPT Image 2", |
| 96 | {"BRAVE_API_KEY": "fake"}, |
| 97 | ) |
| 98 | |
| 99 | self.assertIn("Matched category=ai_image_generation", buf.getvalue()) |
| 100 | |
| 101 | @patch("lib.resolve.grounding.web_search") |
| 102 | def test_cap_enforced_end_to_end(self, mock_search): |
| 103 | # Synthesize a subreddit response with 9 brand subs |
| 104 | many_subs_items = [ |
| 105 | {"title": f"r/Brand{i}", "snippet": "", "url": ""} |
| 106 | for i in range(9) |
| 107 | ] |
| 108 | mock_search.side_effect = _fake_websearch({ |
| 109 | "subreddit": many_subs_items, |
| 110 | }) |
| 111 | |
| 112 | result = resolve.auto_resolve( |
| 113 | "Prompting GPT Image 2", |
| 114 | {"BRAVE_API_KEY": "fake"}, |
| 115 | ) |
| 116 | |
| 117 | self.assertLessEqual(len(result["subreddits"]), resolve.MAX_SUBS) |
| 118 | # The first WebSearch sub is still present (brand subs never evicted) |
| 119 | self.assertIn("Brand0", result["subreddits"]) |
| 120 | |
| 121 | @patch("lib.resolve.grounding.web_search") |
| 122 | def test_uncategorized_topic_does_not_inject_peers(self, mock_search): |
| 123 | mock_search.side_effect = _fake_websearch({ |
| 124 | "subreddit": [{"title": "r/Kanye is wild", "snippet": "", "url": ""}], |
| 125 | }) |
| 126 | |
| 127 | buf = io.StringIO() |
| 128 | with redirect_stderr(buf): |
| 129 | result = resolve.auto_resolve( |
| 130 | "Kanye West latest album", |
| 131 | {"BRAVE_API_KEY": "fake"}, |
| 132 | ) |
| 133 | |
| 134 | self.assertEqual(result["subreddits"], ["Kanye"]) |
| 135 | self.assertIsNone(result["category"]) |
| 136 | self.assertNotIn("Matched category=", buf.getvalue()) |
| 137 | |
| 138 | if __name__ == "__main__": |
| 139 | unittest.main() |
| 140 |