| 1 | """CJK-aware tokenization for relevance scoring and near-duplicate detection. |
| 2 | |
| 3 | The skill ships with zero hard dependencies (pyproject ``dependencies = []``) |
| 4 | so it installs across 50+ Agent Skills hosts as plain Python. Chinese text has |
| 5 | no whitespace word boundaries, so the original ``str.split()`` tokenizers in |
| 6 | relevance.py / dedupe.py collapse a whole sentence into a single token and |
| 7 | break token-overlap scoring and Jaccard de-duplication for Chinese sources |
| 8 | (Xiaohongshu, Bilibili). |
| 9 | |
| 10 | ``segment(text)`` fixes that. It splits text into maximal CJK and non-CJK runs: |
| 11 | |
| 12 | - Non-CJK (ASCII / Latin) runs keep the original ``\\w+`` word behaviour. |
| 13 | - CJK runs are routed through jieba when it is installed (best quality), and |
| 14 | fall back to character bigrams when jieba is absent. Bigrams are a |
| 15 | dictionary-free segmentation that still gives robust overlap signal — e.g. |
| 16 | query "大模型" -> {大模, 模型} overlaps text "国产大模型评测" -> {..大模, 模型..}. |
| 17 | |
| 18 | jieba stays OPTIONAL: present -> used; absent -> bigram fallback. We never add |
| 19 | it to the hard dependency set, preserving the install-anywhere property. |
| 20 | """ |
| 21 | |
| 22 | from __future__ import annotations |
| 23 | |
| 24 | import re |
| 25 | from typing import List |
| 26 | |
| 27 | # CJK ideographs + Japanese kana + Korean hangul. The Chinese ideograph block |
| 28 | # (一-鿿) and its extension-A (㐀-䶿) cover the cases we care |
| 29 | # about; kana/hangul are included so mixed-language text degrades gracefully. |
| 30 | _CJK_CHARS = r"㐀-䶿一-鿿豈--ヿ가-" |
| 31 | _CJK_RE = re.compile(f"[{_CJK_CHARS}]") |
| 32 | _CJK_RUN_RE = re.compile(f"[{_CJK_CHARS}]+") |
| 33 | _LATIN_RE = re.compile(r"\w+") |
| 34 | |
| 35 | # High-frequency Chinese function words that dilute overlap signal, mirroring |
| 36 | # the role of the English STOPWORDS sets in relevance.py / dedupe.py. |
| 37 | CHINESE_STOPWORDS = frozenset( |
| 38 | { |
| 39 | "的", "了", "和", "是", "在", "我", "有", "也", "就", "不", "人", "都", |
| 40 | "一", "一个", "上", "很", "到", "说", "要", "去", "你", "会", "着", |
| 41 | "没有", "看", "好", "自己", "这", "那", "这个", "那个", "什么", "怎么", |
| 42 | "为什么", "以及", "或者", "但是", "因为", "所以", "如果", "可以", |
| 43 | "这样", "那样", "他们", "我们", "你们", "它", "她", "他", "吗", "呢", |
| 44 | "吧", "啊", "哦", "嗯", "与", "及", "等", "被", "把", "让", "给", "向", |
| 45 | "还", "再", "又", "从", "对", "为", "以", "之", "其", "中", |
| 46 | } |
| 47 | ) |
| 48 | |
| 49 | # Optional jieba, resolved once at import time. Binding it here (rather than |
| 50 | # lazily on first use) avoids a race: the pipeline scores relevance inside a |
| 51 | # ThreadPoolExecutor, so a lazy initializer with mutable globals could have two |
| 52 | # threads import concurrently and observe a half-initialized state. Doing it at |
| 53 | # module load means the binding is settled before any worker thread runs. |
| 54 | # |
| 55 | # The BROAD `except Exception` is intentional: jieba is an optional enhancement, |
| 56 | # so ANY failure to load it — package absent, corrupted install, missing data |
| 57 | # files, or a setLogLevel signature change across versions — must degrade to the |
| 58 | # bigram fallback, never crash the skill. jieba guards its own first-call |
| 59 | # dictionary build with an internal lock, so concurrent `cut()` is safe once the |
| 60 | # module object is bound. |
| 61 | try: |
| 62 | import jieba as _jieba # type: ignore |
| 63 | |
| 64 | _jieba.setLogLevel(60) # silence dictionary-build chatter on stderr |
| 65 | except Exception: |
| 66 | _jieba = None |
| 67 | |
| 68 | |
| 69 | def has_cjk(text: str) -> bool: |
| 70 | """True if the text contains any CJK / kana / hangul character.""" |
| 71 | return bool(text) and _CJK_RE.search(text) is not None |
| 72 | |
| 73 | |
| 74 | def _cjk_tokens(run: str) -> List[str]: |
| 75 | # Reads the module-global _jieba at call time, so tests can force the bigram |
| 76 | # path deterministically by setting cjk._jieba = None regardless of whether |
| 77 | # jieba is installed in the environment. |
| 78 | if _jieba is not None: |
| 79 | return [w for w in _jieba.cut(run) if w.strip() and _CJK_RE.search(w)] |
| 80 | # Dictionary-free fallback: character bigrams (single char if run length 1). |
| 81 | if len(run) <= 1: |
| 82 | return [run] if run else [] |
| 83 | return [run[i:i + 2] for i in range(len(run) - 1)] |
| 84 | |
| 85 | |
| 86 | def segment(text: str) -> List[str]: |
| 87 | """Tokenize mixed CJK / Latin text into a flat list of lowercased tokens. |
| 88 | |
| 89 | CJK runs -> jieba words or character bigrams. Latin runs -> ``\\w+`` words. |
| 90 | Order is preserved; callers that want a set can wrap the result. |
| 91 | """ |
| 92 | if not text: |
| 93 | return [] |
| 94 | text = text.lower() |
| 95 | if not has_cjk(text): |
| 96 | return _LATIN_RE.findall(text) |
| 97 | |
| 98 | out: List[str] = [] |
| 99 | pos = 0 |
| 100 | for match in _CJK_RUN_RE.finditer(text): |
| 101 | if match.start() > pos: |
| 102 | out.extend(_LATIN_RE.findall(text[pos:match.start()])) |
| 103 | out.extend(_cjk_tokens(match.group())) |
| 104 | pos = match.end() |
| 105 | if pos < len(text): |
| 106 | out.extend(_LATIN_RE.findall(text[pos:])) |
| 107 | return out |
| 108 |