| 1 | import unittest |
| 2 | from unittest.mock import patch |
| 3 | |
| 4 | from lib import cjk, dedupe, relevance |
| 5 | |
| 6 | |
| 7 | class _BigramBase(unittest.TestCase): |
| 8 | """Force the dictionary-free bigram path so assertions are deterministic |
| 9 | regardless of whether jieba is installed in the test environment.""" |
| 10 | |
| 11 | def setUp(self): |
| 12 | patcher = patch.object(cjk, "_jieba", None) |
| 13 | patcher.start() |
| 14 | self.addCleanup(patcher.stop) |
| 15 | |
| 16 | |
| 17 | class TestCjkSegment(_BigramBase): |
| 18 | def test_has_cjk(self): |
| 19 | self.assertTrue(cjk.has_cjk("国产大模型")) |
| 20 | self.assertTrue(cjk.has_cjk("GPT4很强")) |
| 21 | self.assertFalse(cjk.has_cjk("hello world")) |
| 22 | self.assertFalse(cjk.has_cjk("")) |
| 23 | |
| 24 | def test_ascii_path_unchanged(self): |
| 25 | # Non-CJK text keeps whitespace/word tokenization. |
| 26 | self.assertEqual(cjk.segment("best react hooks"), ["best", "react", "hooks"]) |
| 27 | |
| 28 | def test_chinese_bigrams(self): |
| 29 | toks = cjk.segment("大模型") |
| 30 | self.assertIn("大模", toks) |
| 31 | self.assertIn("模型", toks) |
| 32 | |
| 33 | def test_mixed_language(self): |
| 34 | toks = cjk.segment("GPT4很强 react") |
| 35 | self.assertIn("gpt4", toks) |
| 36 | self.assertIn("react", toks) |
| 37 | self.assertIn("很强", toks) |
| 38 | |
| 39 | def test_single_cjk_char(self): |
| 40 | self.assertEqual(cjk.segment("中"), ["中"]) |
| 41 | |
| 42 | |
| 43 | class TestChineseRelevance(_BigramBase): |
| 44 | def test_chinese_query_matches_chinese_text(self): |
| 45 | q = relevance.PreparedQuery("国产大模型 测评") |
| 46 | score = relevance.token_overlap_relevance(q, "这是国产大模型的最新测评") |
| 47 | self.assertGreater(score, 0.5) |
| 48 | |
| 49 | def test_chinese_query_rejects_unrelated_text(self): |
| 50 | q = relevance.PreparedQuery("国产大模型 测评") |
| 51 | score = relevance.token_overlap_relevance(q, "今天天气很好适合出门散步") |
| 52 | self.assertEqual(score, 0.0) |
| 53 | |
| 54 | def test_english_relevance_not_regressed(self): |
| 55 | q = relevance.PreparedQuery("react hooks") |
| 56 | self.assertGreaterEqual(relevance.token_overlap_relevance(q, "a guide to react hooks"), 0.9) |
| 57 | |
| 58 | def test_cjk_phrase_bonus_applies_on_contiguous_match(self): |
| 59 | # A multi-token CJK query ("国产大模型 测评") whose words appear |
| 60 | # contiguously in the text earns the phrase bonus via the space-stripped |
| 61 | # containment retry; the same words scattered apart do not. |
| 62 | q = relevance.PreparedQuery("国产大模型 测评") |
| 63 | contiguous = relevance.token_overlap_relevance(q, "国产大模型测评合集") |
| 64 | scattered = relevance.token_overlap_relevance(q, "测评了很多东西也聊到国产大模型") |
| 65 | self.assertGreater(contiguous, scattered) |
| 66 | |
| 67 | def test_english_phrase_bonus_stays_space_sensitive(self): |
| 68 | # has_cjk gate: English must NOT gain a bonus from space-stripped |
| 69 | # concatenation (no "reacthooks" false phrase match). |
| 70 | q = relevance.PreparedQuery("react hooks") |
| 71 | # "reacthooks" contiguous-without-space should not trigger a CJK-style retry |
| 72 | score = relevance.token_overlap_relevance(q, "myreacthooks bundle") |
| 73 | self.assertLessEqual(score, 1.0) # sanity; behavior identical to pre-change |
| 74 | |
| 75 | |
| 76 | class TestChineseDedupe(_BigramBase): |
| 77 | def test_reordered_chinese_is_near_duplicate(self): |
| 78 | sim = dedupe.hybrid_similarity("国产大模型最新测评对比", "国产大模型测评对比最新") |
| 79 | self.assertGreater(sim, 0.5) |
| 80 | |
| 81 | def test_distinct_chinese_is_not_duplicate(self): |
| 82 | sim = dedupe.hybrid_similarity("国产大模型测评", "今天天气很好出门散步") |
| 83 | self.assertLess(sim, 0.3) |
| 84 | |
| 85 | |
| 86 | class TestJiebaBinding(unittest.TestCase): |
| 87 | def test_jieba_global_is_bound_at_import(self): |
| 88 | # Eager import binds the module global once (None when jieba absent). |
| 89 | # No lazy initializer => no per-call race in the pipeline thread pool. |
| 90 | self.assertTrue(hasattr(cjk, "_jieba")) |
| 91 | self.assertFalse(hasattr(cjk, "_get_jieba")) |
| 92 | |
| 93 | |
| 94 | if __name__ == "__main__": |
| 95 | unittest.main() |
| 96 |