| 1 | /** |
| 2 | * 字符 n-gram 文本相似度工具 |
| 3 | * 基于字符 n-gram 余弦相似度，天然支持多语言 |
| 4 | */ |
| 5 | |
| 6 | function charNgrams(text: string, n: number): Map<string, number> { |
| 7 | const freq = new Map<string, number>() |
| 8 | const normalized = text.replace(/\s+/g, ' ').trim().toLowerCase() |
| 9 | for (let i = 0; i <= normalized.length - n; i++) { |
| 10 | const gram = normalized.slice(i, i + n) |
| 11 | freq.set(gram, (freq.get(gram) ?? 0) + 1) |
| 12 | } |
| 13 | return freq |
| 14 | } |
| 15 | |
| 16 | function cosineSimilarity( |
| 17 | a: Map<string, number>, |
| 18 | b: Map<string, number>, |
| 19 | ): number { |
| 20 | let dot = 0 |
| 21 | let normA = 0 |
| 22 | let normB = 0 |
| 23 | |
| 24 | for (const [key, val] of a) { |
| 25 | normA += val * val |
| 26 | const bVal = b.get(key) |
| 27 | if (bVal !== undefined) { |
| 28 | dot += val * bVal |
| 29 | } |
| 30 | } |
| 31 | |
| 32 | for (const val of b.values()) { |
| 33 | normB += val * val |
| 34 | } |
| 35 | |
| 36 | if (normA === 0 || normB === 0) |
| 37 | return 0 |
| 38 | return dot / (Math.sqrt(normA) * Math.sqrt(normB)) |
| 39 | } |
| 40 | |
| 41 | export function textSimilarity(a: string, b: string): number { |
| 42 | if (!a && !b) |
| 43 | return 1 |
| 44 | if (!a || !b) |
| 45 | return 0 |
| 46 | if (a === b) |
| 47 | return 1 |
| 48 | |
| 49 | const bigram = cosineSimilarity(charNgrams(a, 2), charNgrams(b, 2)) |
| 50 | const trigram = cosineSimilarity(charNgrams(a, 3), charNgrams(b, 3)) |
| 51 | return bigram * 0.5 + trigram * 0.5 |
| 52 | } |
| 53 | |
| 54 | export function draftWorkSimilarity( |
| 55 | draft: { title?: string, desc?: string }, |
| 56 | work: { title?: string, desc?: string }, |
| 57 | ): { score: number, titleScore: number, descScore: number } { |
| 58 | const titleScore = textSimilarity(draft.title ?? '', work.title ?? '') |
| 59 | const descScore = textSimilarity(draft.desc ?? '', work.desc ?? '') |
| 60 | |
| 61 | const hasTitle = !!(draft.title || work.title) |
| 62 | const hasDesc = !!(draft.desc || work.desc) |
| 63 | |
| 64 | let score: number |
| 65 | if (hasTitle && hasDesc) { |
| 66 | score = titleScore * 0.3 + descScore * 0.7 |
| 67 | } |
| 68 | else if (hasTitle) { |
| 69 | score = titleScore |
| 70 | } |
| 71 | else if (hasDesc) { |
| 72 | score = descScore |
| 73 | } |
| 74 | else { |
| 75 | score = 1 |
| 76 | } |
| 77 | |
| 78 | return { score, titleScore, descScore } |
| 79 | } |
| 80 |