| 1 | """Adversarial query tests for the v3 planner. |
| 2 | |
| 3 | These target edge cases found during regression analysis: slash-separated |
| 4 | comparisons, 'difference between X and Y' phrasing, trailing context |
| 5 | leaking into entities, degenerate inputs, and false-positive resistance. |
| 6 | """ |
| 7 | |
| 8 | import unittest |
| 9 | |
| 10 | from lib import planner |
| 11 | |
| 12 | |
| 13 | class TestSlashSeparatedComparison(unittest.TestCase): |
| 14 | """'React/Vue/Svelte' should be detected as comparison intent |
| 15 | and produce entity subqueries.""" |
| 16 | |
| 17 | def test_slash_triggers_comparison_intent(self): |
| 18 | self.assertEqual(planner._infer_intent("React/Vue/Svelte"), "comparison") |
| 19 | |
| 20 | def test_slash_extracts_entities(self): |
| 21 | entities = planner._comparison_entities("React/Vue/Svelte") |
| 22 | self.assertEqual(len(entities), 3) |
| 23 | self.assertIn("React", entities) |
| 24 | self.assertIn("Vue", entities) |
| 25 | self.assertIn("Svelte", entities) |
| 26 | |
| 27 | def test_slash_forces_deterministic(self): |
| 28 | self.assertTrue(planner._should_force_deterministic_plan("React/Vue")) |
| 29 | |
| 30 | def test_url_slash_does_not_trigger_comparison(self): |
| 31 | self.assertNotEqual(planner._infer_intent("https://example.com/path"), "comparison") |
| 32 | |
| 33 | def test_slash_trailing_context_stripped(self): |
| 34 | entities = planner._comparison_entities("React/Vue/Svelte for frontend in 2026") |
| 35 | for entity in entities: |
| 36 | self.assertNotIn("frontend", entity.lower(), |
| 37 | f"Trailing context leaked: '{entity}'") |
| 38 | |
| 39 | |
| 40 | class TestDifferenceBetweenPhrasing(unittest.TestCase): |
| 41 | """'difference between X and Y' should extract both entities.""" |
| 42 | |
| 43 | def test_intent_is_comparison(self): |
| 44 | self.assertEqual( |
| 45 | planner._infer_intent("difference between OpenClaw and NanoClaw"), |
| 46 | "comparison", |
| 47 | ) |
| 48 | |
| 49 | def test_entities_extracted(self): |
| 50 | entities = planner._comparison_entities("difference between OpenClaw and NanoClaw") |
| 51 | self.assertEqual(len(entities), 2) |
| 52 | self.assertIn("OpenClaw", entities) |
| 53 | self.assertIn("NanoClaw", entities) |
| 54 | |
| 55 | def test_forces_deterministic(self): |
| 56 | self.assertTrue( |
| 57 | planner._should_force_deterministic_plan("difference between OpenClaw and NanoClaw") |
| 58 | ) |
| 59 | |
| 60 | |
| 61 | class TestAndFalsePositive(unittest.TestCase): |
| 62 | """'and' must not split entities outside 'difference between' context.""" |
| 63 | |
| 64 | def test_pros_and_cons_no_entities(self): |
| 65 | entities = planner._comparison_entities("pros and cons of AI") |
| 66 | self.assertEqual(entities, []) |
| 67 | |
| 68 | def test_react_and_vue_no_entities(self): |
| 69 | # No "vs" or "difference between" -- just "and" |
| 70 | entities = planner._comparison_entities("React and Vue") |
| 71 | self.assertEqual(entities, []) |
| 72 | |
| 73 | |
| 74 | class TestTrailingContextStripping(unittest.TestCase): |
| 75 | """Trailing preposition phrases must not leak into entity strings.""" |
| 76 | |
| 77 | def test_for_stripped(self): |
| 78 | entities = planner._comparison_entities("A vs B for production use") |
| 79 | self.assertNotIn("production", entities[-1].lower()) |
| 80 | |
| 81 | def test_in_stripped(self): |
| 82 | entities = planner._comparison_entities("A vs B in 2026") |
| 83 | self.assertNotIn("2026", entities[-1]) |
| 84 | |
| 85 | def test_with_stripped(self): |
| 86 | entities = planner._comparison_entities("A vs B with better security") |
| 87 | self.assertNotIn("security", entities[-1].lower()) |
| 88 | |
| 89 | def test_core_entity_preserved(self): |
| 90 | entities = planner._comparison_entities("Fly.io vs Railway.app for deployment") |
| 91 | self.assertTrue(any("Fly" in e for e in entities)) |
| 92 | self.assertTrue(any("Railway" in e for e in entities)) |
| 93 | |
| 94 | |
| 95 | class TestDuplicateEntities(unittest.TestCase): |
| 96 | |
| 97 | def test_deduped(self): |
| 98 | entities = planner._comparison_entities("OpenClaw vs OpenClaw") |
| 99 | self.assertEqual(len(entities), 1) |
| 100 | |
| 101 | |
| 102 | class TestFiveWayComparison(unittest.TestCase): |
| 103 | |
| 104 | def test_capped_at_max(self): |
| 105 | topic = "A vs B vs C vs D vs E vs F" |
| 106 | entities = planner._comparison_entities(topic) |
| 107 | self.assertLessEqual(len(entities), planner._max_subqueries("comparison")) |
| 108 | |
| 109 | def test_does_not_crash(self): |
| 110 | plan = planner.plan_query( |
| 111 | topic="A vs B vs C vs D vs E", |
| 112 | available_sources=["reddit", "x", "grounding"], |
| 113 | requested_sources=None, |
| 114 | depth="default", |
| 115 | provider=None, |
| 116 | model=None, |
| 117 | ) |
| 118 | from lib import competitors |
| 119 | self.assertLessEqual( |
| 120 | len(plan.subqueries), |
| 121 | competitors.COMPARISON_ENTITY_MAX + 1, |
| 122 | ) |
| 123 | |
| 124 | |
| 125 | class TestDegenerateInputs(unittest.TestCase): |
| 126 | |
| 127 | def test_single_word(self): |
| 128 | plan = planner.plan_query( |
| 129 | topic="Bitcoin", |
| 130 | available_sources=["reddit"], |
| 131 | requested_sources=None, |
| 132 | depth="default", |
| 133 | provider=None, |
| 134 | model=None, |
| 135 | ) |
| 136 | self.assertGreater(len(plan.subqueries), 0) |
| 137 | |
| 138 | def test_empty_vs_split(self): |
| 139 | plan = planner.plan_query( |
| 140 | topic="vs vs vs", |
| 141 | available_sources=["reddit"], |
| 142 | requested_sources=None, |
| 143 | depth="default", |
| 144 | provider=None, |
| 145 | model=None, |
| 146 | ) |
| 147 | for sq in plan.subqueries: |
| 148 | self.assertTrue(sq.search_query.strip()) |
| 149 | |
| 150 | def test_very_long_comparison(self): |
| 151 | topic = " vs ".join(f"Tool{i}" for i in range(20)) |
| 152 | plan = planner.plan_query( |
| 153 | topic=topic, |
| 154 | available_sources=["reddit", "x"], |
| 155 | requested_sources=None, |
| 156 | depth="default", |
| 157 | provider=None, |
| 158 | model=None, |
| 159 | ) |
| 160 | from lib import competitors |
| 161 | self.assertLessEqual( |
| 162 | len(plan.subqueries), |
| 163 | competitors.COMPARISON_ENTITY_MAX + 1, |
| 164 | ) |
| 165 | |
| 166 | |
| 167 | class TestMixedCaseAndPunctuation(unittest.TestCase): |
| 168 | |
| 169 | def test_uppercase_vs_period(self): |
| 170 | self.assertEqual( |
| 171 | planner._infer_intent("OpenClaw VS. NanoClaw VS. IronClaw"), |
| 172 | "comparison", |
| 173 | ) |
| 174 | |
| 175 | def test_entities_preserved_with_mixed_case(self): |
| 176 | entities = planner._comparison_entities("OpenClaw VS. NanoClaw VS. IronClaw") |
| 177 | self.assertGreaterEqual(len(entities), 3) |
| 178 | |
| 179 | |
| 180 | class TestSubstringEntity(unittest.TestCase): |
| 181 | |
| 182 | def test_react_vs_react_native_not_collapsed(self): |
| 183 | entities = planner._comparison_entities("React vs React Native") |
| 184 | self.assertEqual(len(entities), 2) |
| 185 | self.assertTrue(any("Native" in e for e in entities)) |
| 186 | |
| 187 | |
| 188 | class TestNoiseWordEntities(unittest.TestCase): |
| 189 | """Entities that are also common English words (Swift, Rust, Go) |
| 190 | must survive entity extraction.""" |
| 191 | |
| 192 | def test_swift_preserved(self): |
| 193 | entities = planner._comparison_entities("Swift vs Rust vs Go") |
| 194 | self.assertGreaterEqual(len(entities), 3) |
| 195 | |
| 196 | def test_go_not_stripped(self): |
| 197 | entities = planner._comparison_entities("Swift vs Rust vs Go") |
| 198 | self.assertTrue(any("Go" in e for e in entities)) |
| 199 | |
| 200 | if __name__ == "__main__": |
| 201 | unittest.main() |
| 202 |