返回 CodeWhale
unicode.py
根目录 / patches / unicode-width-0.2.2 / scripts / unicode.py
1 #!/usr/bin/env python3
2 #
3 # Copyright 2011-2025 The Rust Project Developers. See the COPYRIGHT
4 # file at the top-level directory of this distribution and at
5 # http://rust-lang.org/COPYRIGHT.
6 #
7 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
8 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
9 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
10 # option. This file may not be copied, modified, or distributed
11 # except according to those terms.
12
13 # This script uses the following Unicode tables:
14 #
15 # - DerivedCoreProperties.txt
16 # - EastAsianWidth.txt
17 # - HangulSyllableType.txt
18 # - LineBreak.txt
19 # - NormalizationTest.txt (for tests only)
20 # - PropList.txt
21 # - ReadMe.txt
22 # - UnicodeData.txt
23 # - auxiliary/GraphemeBreakProperty.txt
24 # - emoji/emoji-data.txt
25 # - emoji/emoji-test.txt (for tests only)
26 # - emoji/emoji-variation-sequences.txt
27 # - extracted/DerivedCombiningClass.txt
28 # - extracted/DerivedGeneralCategory.txt
29 # - extracted/DerivedJoiningGroup.txt
30 # - extracted/DerivedJoiningType.txt
31 #
32 # Since this should not require frequent updates, we just store this
33 # out-of-line and check the generated module into git.
34
35 import enum
36 import math
37 import operator
38 import os
39 import re
40 import sys
41 import urllib.request
42 from collections import defaultdict
43 from itertools import batched
44 from typing import Callable, Iterable
45
46 UNICODE_VERSION = "17.0.0"
47 """The version of the Unicode data files to download."""
48
49 NUM_CODEPOINTS = 0x110000
50 """An upper bound for which `range(0, NUM_CODEPOINTS)` contains Unicode's codespace."""
51
52 MAX_CODEPOINT_BITS = math.ceil(math.log2(NUM_CODEPOINTS - 1))
53 """The maximum number of bits required to represent a Unicode codepoint."""
54
55
56 class OffsetType(enum.IntEnum):
57 """Represents the data type of a lookup table's offsets. Each variant's value represents the
58 number of bits required to represent that variant's type."""
59
60 U2 = 2
61 """Offsets are 2-bit unsigned integers, packed four-per-byte."""
62 U4 = 4
63 """Offsets are 4-bit unsigned integers, packed two-per-byte."""
64 U8 = 8
65 """Each offset is a single byte (u8)."""
66
67
68 MODULE_PATH = "../src/tables.rs"
69 """The path of the emitted Rust module (relative to the working directory)"""
70
71 TABLE_SPLITS = [7, 13]
72 """The splits between the bits of the codepoint used to index each subtable.
73 Adjust these values to change the sizes of the subtables"""
74
75 Codepoint = int
76 BitPos = int
77
78
79 def fetch_open(filename: str, local_prefix: str = "", emoji: bool = False):
80 """Opens `filename` and return its corresponding file object. If `filename` isn't on disk,
81 fetches it from `https://www.unicode.org/Public/`. Exits with code 1 on failure.
82 """
83 basename = os.path.basename(filename)
84 localname = os.path.join(local_prefix, basename)
85 if not os.path.exists(localname):
86 if emoji:
87 prefix = "emoji"
88 else:
89 prefix = "ucd"
90 urllib.request.urlretrieve(
91 f"https://www.unicode.org/Public/{UNICODE_VERSION}/{prefix}/{filename}",
92 localname,
93 )
94 try:
95 return open(localname, encoding="utf-8")
96 except OSError:
97 sys.stderr.write(f"cannot load {localname}")
98 sys.exit(1)
99
100
101 def load_unicode_version() -> tuple[int, int, int]:
102 """Returns the current Unicode version by fetching and processing `ReadMe.txt`."""
103 with fetch_open("ReadMe.txt") as readme:
104 pattern = r"for Version (\d+)\.(\d+)\.(\d+) of the Unicode"
105 return tuple(map(int, re.search(pattern, readme.read()).groups())) # type: ignore
106
107
108 def load_property(filename: str, pattern: str, action: Callable[[int], None]):
109 with fetch_open(filename) as properties:
110 single = re.compile(rf"^([0-9A-F]+)\s*;\s*{pattern}\s+")
111 multiple = re.compile(rf"^([0-9A-F]+)\.\.([0-9A-F]+)\s*;\s*{pattern}\s+")
112
113 for line in properties.readlines():
114 raw_data = None # (low, high)
115 if match := single.match(line):
116 raw_data = (match.group(1), match.group(1))
117 elif match := multiple.match(line):
118 raw_data = (match.group(1), match.group(2))
119 else:
120 continue
121 low = int(raw_data[0], 16)
122 high = int(raw_data[1], 16)
123 for cp in range(low, high + 1):
124 action(cp)
125
126
127 def to_sorted_ranges(iter: Iterable[Codepoint]) -> list[tuple[Codepoint, Codepoint]]:
128 "Creates a sorted list of ranges from an iterable of codepoints"
129 lst = [c for c in iter]
130 lst.sort()
131 ret = []
132 for cp in lst:
133 if len(ret) > 0 and ret[-1][1] == cp - 1:
134 ret[-1] = (ret[-1][0], cp)
135 else:
136 ret.append((cp, cp))
137 return ret
138
139
140 class EastAsianWidth(enum.IntEnum):
141 """Represents the width of a Unicode character according to UAX 16.
142 All East Asian Width classes resolve into either
143 `EffectiveWidth.NARROW`, `EffectiveWidth.WIDE`, or `EffectiveWidth.AMBIGUOUS`.
144 """
145
146 NARROW = 1
147 """ One column wide. """
148 WIDE = 2
149 """ Two columns wide. """
150 AMBIGUOUS = 3
151 """ Two columns wide in a CJK context. One column wide in all other contexts. """
152
153
154 class CharWidthInTable(enum.IntEnum):
155 """Represents the width of a Unicode character
156 as stored in the tables."""
157
158 ZERO = 0
159 ONE = 1
160 TWO = 2
161 SPECIAL = 3
162
163
164 class WidthState(enum.IntEnum):
165 """
166 Width calculation proceeds according to a state machine.
167 We iterate over the characters of the string from back to front;
168 the next character encountered determines the transition to take.
169
170 The integer values of these variants have special meaning:
171 - Top bit: whether this is Vs16
172 - 2nd from top: whether this is Vs15
173 - 3rd bit from top: whether this is transparent to emoji/text presentation
174 (if set, should also set 4th)
175 - 4th bit: whether to set top bit on emoji presentation.
176 If this is set but 3rd is not, the width mode is related to zwj sequences
177 - 5th from top: whether this is unaffected by ligature-transparent
178 (if set, should also set 3rd and 4th)
179 - 6th bit: if 4th is set but this one is not, then this is a ZWJ ligature state
180 where no ZWJ has been encountered yet; encountering one flips this on
181 - Seventh bit:
182 - CJK mode: is VS1 or VS3
183 - Not CJK: is VS2
184 """
185
186 # BASIC WIDTHS
187
188 ZERO = 0x1_0000
189 "Zero columns wide."
190
191 NARROW = 0x1_0001
192 "One column wide."
193
194 WIDE = 0x1_0002
195 "Two columns wide."
196
197 THREE = 0x1_0003
198 "Three columns wide."
199
200 # \r\n
201 LINE_FEED = 0b0000_0000_0000_0001
202 "\\n (CRLF has width 1)"
203
204 # EMOJI
205
206 # Emoji skintone modifiers
207 EMOJI_MODIFIER = 0b0000_0000_0000_0010
208 "`Emoji_Modifier`"
209
210 # Emoji ZWJ sequences
211
212 REGIONAL_INDICATOR = 0b0000_0000_0000_0011
213 "`Regional_Indicator`"
214
215 SEVERAL_REGIONAL_INDICATOR = 0b0000_0000_0000_0100
216 "At least two `Regional_Indicator`in sequence"
217
218 EMOJI_PRESENTATION = 0b0000_0000_0000_0101
219 "`Emoji_Presentation`"
220
221 ZWJ_EMOJI_PRESENTATION = 0b0001_0000_0000_0110
222 "\\u200D `Emoji_Presentation`"
223
224 VS16_ZWJ_EMOJI_PRESENTATION = 0b1001_0000_0000_0110
225 "\\uFE0F \\u200D `Emoji_Presentation`"
226
227 KEYCAP_ZWJ_EMOJI_PRESENTATION = 0b0001_0000_0000_0111
228 "\\u20E3 \\u200D `Emoji_Presentation`"
229
230 VS16_KEYCAP_ZWJ_EMOJI_PRESENTATION = 0b1001_0000_0000_0111
231 "\\uFE0F \\u20E3 \\u200D `Emoji_Presentation`"
232
233 REGIONAL_INDICATOR_ZWJ_PRESENTATION = 0b0000_0000_0000_1001
234 "`Regional_Indicator` \\u200D `Emoji_Presentation`"
235
236 EVEN_REGIONAL_INDICATOR_ZWJ_PRESENTATION = 0b0000_0000_0000_1010
237 "(`Regional_Indicator` `Regional_Indicator`)+ \\u200D `Emoji_Presentation`"
238
239 ODD_REGIONAL_INDICATOR_ZWJ_PRESENTATION = 0b0000_0000_0000_1011
240 "(`Regional_Indicator` `Regional_Indicator`)+ `Regional_Indicator` \\u200D `Emoji_Presentation`"
241
242 TAG_END_ZWJ_EMOJI_PRESENTATION = 0b0000_0000_0001_0000
243 "\\uE007F \\u200D `Emoji_Presentation`"
244
245 TAG_D1_END_ZWJ_EMOJI_PRESENTATION = 0b0000_0000_0001_0001
246 "\\uE0030..=\\uE0039 \\uE007F \\u200D `Emoji_Presentation`"
247
248 TAG_D2_END_ZWJ_EMOJI_PRESENTATION = 0b0000_0000_0001_0010
249 "(\\uE0030..=\\uE0039){2} \\uE007F \\u200D `Emoji_Presentation`"
250
251 TAG_D3_END_ZWJ_EMOJI_PRESENTATION = 0b0000_0000_0001_0011
252 "(\\uE0030..=\\uE0039){3} \\uE007F \\u200D `Emoji_Presentation`"
253
254 TAG_A1_END_ZWJ_EMOJI_PRESENTATION = 0b0000_0000_0001_1001
255 "\\uE0061..=\\uE007A \\uE007F \\u200D `Emoji_Presentation`"
256
257 TAG_A2_END_ZWJ_EMOJI_PRESENTATION = 0b0000_0000_0001_1010
258 "(\\uE0061..=\\uE007A){2} \\uE007F \\u200D `Emoji_Presentation`"
259
260 TAG_A3_END_ZWJ_EMOJI_PRESENTATION = 0b0000_0000_0001_1011
261 "(\\uE0061..=\\uE007A){3} \\uE007F \\u200D `Emoji_Presentation`"
262
263 TAG_A4_END_ZWJ_EMOJI_PRESENTATION = 0b0000_0000_0001_1100
264 "(\\uE0061..=\\uE007A){4} \\uE007F \\u200D `Emoji_Presentation`"
265
266 TAG_A5_END_ZWJ_EMOJI_PRESENTATION = 0b0000_0000_0001_1101
267 "(\\uE0061..=\\uE007A){35} \\uE007F \\u200D `Emoji_Presentation`"
268
269 TAG_A6_END_ZWJ_EMOJI_PRESENTATION = 0b0000_0000_0001_1110
270 "(\\uE0061..=\\uE007A){6} \\uE007F \\u200D `Emoji_Presentation`"
271
272 # Kirat Rai
273 KIRAT_RAI_VOWEL_SIGN_E = 0b0000_0000_0010_0000
274 "\\u16D67 (\\u16D67 \\u16D67)+ and canonical equivalents"
275 KIRAT_RAI_VOWEL_SIGN_AI = 0b0000_0000_0010_0001
276 "(\\u16D68)+ and canonical equivalents"
277
278 # VARIATION SELECTORS
279
280 VARIATION_SELECTOR_1_2_OR_3 = 0b0000_0010_0000_0000
281 "\\uFE00 or \\uFE02 if CJK, or \\uFE01 otherwise"
282
283 # Text presentation sequences (not CJK)
284 VARIATION_SELECTOR_15 = 0b0100_0000_0000_0000
285 "\\uFE0E (text presentation sequences)"
286
287 # Emoji presentation sequences
288 VARIATION_SELECTOR_16 = 0b1000_0000_0000_0000
289 "\\uFE0F (emoji presentation sequences)"
290
291 # ARABIC LAM ALEF
292
293 JOINING_GROUP_ALEF = 0b0011_0000_1111_1111
294 "Joining_Group=Alef (Arabic Lam-Alef ligature)"
295
296 # COMBINING SOLIDUS (CJK only)
297
298 COMBINING_LONG_SOLIDUS_OVERLAY = 0b0011_1100_1111_1111
299 "\\u0338 (CJK only, makes <, =, > width 2)"
300
301 # SOLIDUS + ALEF (solidus is Joining_Type=Transparent)
302 SOLIDUS_OVERLAY_ALEF = 0b0011_1000_1111_1111
303 "\\u0338 followed by Joining_Group=Alef"
304
305 # SCRIPT ZWJ LIGATURES
306
307 # Hebrew alef lamed
308
309 HEBREW_LETTER_LAMED = 0b0011_1000_0000_0000
310 "\\u05DC (Alef-ZWJ-Lamed ligature)"
311
312 ZWJ_HEBREW_LETTER_LAMED = 0b0011_1100_0000_0000
313 "\\u200D\\u05DC (Alef-ZWJ-Lamed ligature)"
314
315 # Buginese <a -i> ya
316
317 BUGINESE_LETTER_YA = 0b0011_1000_0000_0001
318 "\\u1A10 (<a, -i> + ya ligature)"
319
320 ZWJ_BUGINESE_LETTER_YA = 0b0011_1100_0000_0001
321 "\\u200D\\u1A10 (<a, -i> + ya ligature)"
322
323 BUGINESE_VOWEL_SIGN_I_ZWJ_LETTER_YA = 0b0011_1100_0000_0010
324 "\\u1A17\\u200D\\u1A10 (<a, -i> + ya ligature)"
325
326 # Tifinagh bi-consonants
327
328 TIFINAGH_CONSONANT = 0b0011_1000_0000_0011
329 "\\u2D31..=\\u2D65 or \\u2D6F (joined by ZWJ or \\u2D7F TIFINAGH CONSONANT JOINER)"
330
331 ZWJ_TIFINAGH_CONSONANT = 0b0011_1100_0000_0011
332 "ZWJ then \\u2D31..=\\u2D65 or \\u2D6F"
333
334 TIFINAGH_JOINER_CONSONANT = 0b0011_1100_0000_0100
335 "\\u2D7F then \\u2D31..=\\u2D65 or \\u2D6F"
336
337 # Lisu tone letters
338 LISU_TONE_LETTER_MYA_NA_JEU = 0b0011_1100_0000_0101
339 "\\uA4FC or \\uA4FD (https://www.unicode.org/versions/Unicode15.0.0/ch18.pdf#G42078)"
340
341 # Old Turkic orkhon ec - orkhon i
342
343 OLD_TURKIC_LETTER_ORKHON_I = 0b0011_1000_0000_0110
344 "\\u10C03 (ORKHON EC-ZWJ-ORKHON I ligature)"
345
346 ZWJ_OLD_TURKIC_LETTER_ORKHON_I = 0b0011_1100_0000_0110
347 "\\u10C03 (ORKHON EC-ZWJ-ORKHON I ligature)"
348
349 # Khmer coeng signs
350
351 KHMER_COENG_ELIGIBLE_LETTER = 0b0011_1100_0000_0111
352 "\\u1780..=\\u17A2 | \\u17A7 | \\u17AB | \\u17AC | \\u17AF"
353
354 def table_width(self) -> CharWidthInTable:
355 "The width of a character as stored in the lookup tables."
356 match self:
357 case WidthState.ZERO:
358 return CharWidthInTable.ZERO
359 case WidthState.NARROW:
360 return CharWidthInTable.ONE
361 case WidthState.WIDE:
362 return CharWidthInTable.TWO
363 case _:
364 return CharWidthInTable.SPECIAL
365
366 def is_carried(self) -> bool:
367 "Whether this corresponds to a non-default `WidthInfo`."
368 return int(self) <= 0xFFFF
369
370 def width_alone(self) -> int:
371 "The width of a character with this type when it appears alone."
372 match self:
373 case (
374 WidthState.ZERO
375 | WidthState.COMBINING_LONG_SOLIDUS_OVERLAY
376 | WidthState.VARIATION_SELECTOR_15
377 | WidthState.VARIATION_SELECTOR_16
378 | WidthState.VARIATION_SELECTOR_1_2_OR_3
379 ):
380 return 0
381 case (
382 WidthState.WIDE
383 | WidthState.EMOJI_MODIFIER
384 | WidthState.EMOJI_PRESENTATION
385 ):
386 return 2
387 case WidthState.THREE:
388 return 3
389 case _:
390 return 1
391
392 def is_cjk_only(self) -> bool:
393 return self in [
394 WidthState.COMBINING_LONG_SOLIDUS_OVERLAY,
395 WidthState.SOLIDUS_OVERLAY_ALEF,
396 ]
397
398 def is_non_cjk_only(self) -> bool:
399 return self == WidthState.VARIATION_SELECTOR_15
400
401
402 assert len(set([v.value for v in WidthState])) == len([v.value for v in WidthState])
403
404
405 def load_east_asian_widths() -> list[EastAsianWidth]:
406 """Return a list of effective widths, indexed by codepoint.
407 Widths are determined by fetching and parsing `EastAsianWidth.txt`.
408
409 `Neutral`, `Narrow`, and `Halfwidth` characters are assigned `EffectiveWidth.NARROW`.
410
411 `Wide` and `Fullwidth` characters are assigned `EffectiveWidth.WIDE`.
412
413 `Ambiguous` characters are assigned `EffectiveWidth.AMBIGUOUS`."""
414
415 with fetch_open("EastAsianWidth.txt") as eaw:
416 # matches a width assignment for a single codepoint, i.e. "1F336;N # ..."
417 single = re.compile(r"^([0-9A-F]+)\s*;\s*(\w+) +# (\w+)")
418 # matches a width assignment for a range of codepoints, i.e. "3001..3003;W # ..."
419 multiple = re.compile(r"^([0-9A-F]+)\.\.([0-9A-F]+)\s*;\s*(\w+) +# (\w+)")
420 # map between width category code and condensed width
421 width_codes = {
422 **{c: EastAsianWidth.NARROW for c in ["N", "Na", "H"]},
423 **{c: EastAsianWidth.WIDE for c in ["W", "F"]},
424 "A": EastAsianWidth.AMBIGUOUS,
425 }
426
427 width_map = []
428 current = 0
429 for line in eaw.readlines():
430 raw_data = None # (low, high, width)
431 if match := single.match(line):
432 raw_data = (match.group(1), match.group(1), match.group(2))
433 elif match := multiple.match(line):
434 raw_data = (match.group(1), match.group(2), match.group(3))
435 else:
436 continue
437 low = int(raw_data[0], 16)
438 high = int(raw_data[1], 16)
439 width = width_codes[raw_data[2]]
440
441 assert current <= high
442 while current <= high:
443 # Some codepoints don't fall into any of the ranges in EastAsianWidth.txt.
444 # All such codepoints are implicitly given Neural width (resolves to narrow)
445 width_map.append(EastAsianWidth.NARROW if current < low else width)
446 current += 1
447
448 while len(width_map) < NUM_CODEPOINTS:
449 # Catch any leftover codepoints and assign them implicit Neutral/narrow width.
450 width_map.append(EastAsianWidth.NARROW)
451
452 # Characters with ambiguous line breaking are ambiguous
453 load_property(
454 "LineBreak.txt",
455 "AI",
456 lambda cp: (operator.setitem(width_map, cp, EastAsianWidth.AMBIGUOUS)),
457 )
458
459 # Ambiguous `Letter`s and `Modifier_Symbol`s are narrow
460 load_property(
461 "extracted/DerivedGeneralCategory.txt",
462 r"(:?Lu|Ll|Lt|Lm|Lo|Sk)",
463 lambda cp: (
464 operator.setitem(width_map, cp, EastAsianWidth.NARROW)
465 if width_map[cp] == EastAsianWidth.AMBIGUOUS
466 else None
467 ),
468 )
469
470 # GREEK ANO TELEIA: NFC decomposes to U+00B7 MIDDLE DOT
471 width_map[0x0387] = EastAsianWidth.AMBIGUOUS
472
473 # Canonical equivalence for symbols with stroke
474 with fetch_open("UnicodeData.txt") as udata:
475 single = re.compile(r"([0-9A-Z]+);.*?;.*?;.*?;.*?;([0-9A-Z]+) 0338;")
476 for line in udata.readlines():
477 if match := single.match(line):
478 composed = int(match.group(1), 16)
479 decomposed = int(match.group(2), 16)
480 if width_map[decomposed] == EastAsianWidth.AMBIGUOUS:
481 width_map[composed] = EastAsianWidth.AMBIGUOUS
482
483 return width_map
484
485
486 def load_zero_widths() -> list[bool]:
487 """Returns a list `l` where `l[c]` is true if codepoint `c` is considered a zero-width
488 character. `c` is considered a zero-width character if
489
490 - it has the `Default_Ignorable_Code_Point` property (determined from `DerivedCoreProperties.txt`),
491 - or if it has the `Grapheme_Extend` property (determined from `DerivedCoreProperties.txt`),
492 - or if it one of eight characters that should be `Grapheme_Extend` but aren't due to a Unicode spec bug,
493 - or if it has a `Hangul_Syllable_Type` of `Vowel_Jamo` or `Trailing_Jamo` (determined from `HangulSyllableType.txt`).
494 """
495
496 zw_map = [False] * NUM_CODEPOINTS
497
498 # `Default_Ignorable_Code_Point`s also have 0 width:
499 # https://www.unicode.org/faq/unsup_char.html#3
500 # https://www.unicode.org/versions/Unicode15.1.0/ch05.pdf#G40095
501 #
502 # `Grapheme_Extend` includes characters with general category `Mn` or `Me`,
503 # as well as a few `Mc` characters that need to be included so that
504 # canonically equivalent sequences have the same width.
505 load_property(
506 "DerivedCoreProperties.txt",
507 r"(?:Default_Ignorable_Code_Point|Grapheme_Extend)",
508 lambda cp: operator.setitem(zw_map, cp, True),
509 )
510
511 # Treat `Hangul_Syllable_Type`s of `Vowel_Jamo` and `Trailing_Jamo`
512 # as zero-width. This matches the behavior of glibc `wcwidth`.
513 #
514 # Decomposed Hangul characters consist of 3 parts: a `Leading_Jamo`,
515 # a `Vowel_Jamo`, and an optional `Trailing_Jamo`. Together these combine
516 # into a single wide grapheme. So we treat vowel and trailing jamo as
517 # 0-width, such that only the width of the leading jamo is counted
518 # and the resulting grapheme has width 2.
519 #
520 # (See the Unicode Standard sections 3.12 and 18.6 for more on Hangul)
521 load_property(
522 "HangulSyllableType.txt",
523 r"(?:V|T)",
524 lambda cp: operator.setitem(zw_map, cp, True),
525 )
526
527 # Syriac abbreviation mark:
528 # Zero-width `Prepended_Concatenation_Mark`
529 zw_map[0x070F] = True
530
531 # Some Arabic Prepended_Concatenation_Mark`s
532 # https://www.unicode.org/versions/Unicode15.0.0/ch09.pdf#G27820
533 zw_map[0x0605] = True
534 zw_map[0x0890] = True
535 zw_map[0x0891] = True
536 zw_map[0x08E2] = True
537
538 # `[:Grapheme_Cluster_Break=Prepend:]-[:Prepended_Concatenation_Mark:]`
539 gcb_prepend = set()
540 load_property(
541 "auxiliary/GraphemeBreakProperty.txt",
542 "Prepend",
543 lambda cp: gcb_prepend.add(cp),
544 )
545 load_property(
546 "PropList.txt",
547 "Prepended_Concatenation_Mark",
548 lambda cp: gcb_prepend.remove(cp),
549 )
550 for cp in gcb_prepend:
551 zw_map[cp] = True
552
553 # HANGUL CHOSEONG FILLER
554 # U+115F is a `Default_Ignorable_Code_Point`, and therefore would normally have
555 # zero width. However, the expected usage is to combine it with vowel or trailing jamo
556 # (which are considered 0-width on their own) to form a composed Hangul syllable with
557 # width 2. Therefore, we treat it as having width 2.
558 zw_map[0x115F] = False
559
560 # TIFINAGH CONSONANT JOINER
561 # (invisible only when used to join two Tifinagh consonants
562 zw_map[0x2D7F] = False
563
564 # DEVANAGARI CARET
565 # https://www.unicode.org/versions/Unicode15.0.0/ch12.pdf#G667447
566 zw_map[0xA8FA] = True
567
568 return zw_map
569
570
571 def load_width_maps() -> tuple[list[WidthState], list[WidthState]]:
572 """Load complete width table, including characters needing special handling.
573 (Returns 2 tables, one for East Asian and one for not.)"""
574
575 eaws = load_east_asian_widths()
576 zws = load_zero_widths()
577
578 not_ea = []
579 ea = []
580
581 for eaw, zw in zip(eaws, zws):
582 if zw:
583 not_ea.append(WidthState.ZERO)
584 ea.append(WidthState.ZERO)
585 else:
586 if eaw == EastAsianWidth.WIDE:
587 not_ea.append(WidthState.WIDE)
588 else:
589 not_ea.append(WidthState.NARROW)
590
591 if eaw == EastAsianWidth.NARROW:
592 ea.append(WidthState.NARROW)
593 else:
594 ea.append(WidthState.WIDE)
595
596 # Joining_Group=Alef (Arabic Lam-Alef ligature)
597 alef_joining = []
598 load_property(
599 "extracted/DerivedJoiningGroup.txt",
600 "Alef",
601 lambda cp: alef_joining.append(cp),
602 )
603
604 # Regional indicators
605 regional_indicators = []
606 load_property(
607 "PropList.txt",
608 "Regional_Indicator",
609 lambda cp: regional_indicators.append(cp),
610 )
611
612 # Emoji modifiers
613 emoji_modifiers = []
614 load_property(
615 "emoji/emoji-data.txt",
616 "Emoji_Modifier",
617 lambda cp: emoji_modifiers.append(cp),
618 )
619
620 # Default emoji presentation (for ZWJ sequences)
621 emoji_presentation = []
622 load_property(
623 "emoji/emoji-data.txt",
624 "Emoji_Presentation",
625 lambda cp: emoji_presentation.append(cp),
626 )
627
628 for cps, width in [
629 ([0x0A], WidthState.LINE_FEED),
630 ([0x05DC], WidthState.HEBREW_LETTER_LAMED),
631 (alef_joining, WidthState.JOINING_GROUP_ALEF),
632 (range(0x1780, 0x1783), WidthState.KHMER_COENG_ELIGIBLE_LETTER),
633 (range(0x1784, 0x1788), WidthState.KHMER_COENG_ELIGIBLE_LETTER),
634 (range(0x1789, 0x178D), WidthState.KHMER_COENG_ELIGIBLE_LETTER),
635 (range(0x178E, 0x1794), WidthState.KHMER_COENG_ELIGIBLE_LETTER),
636 (range(0x1795, 0x1799), WidthState.KHMER_COENG_ELIGIBLE_LETTER),
637 (range(0x179B, 0x179E), WidthState.KHMER_COENG_ELIGIBLE_LETTER),
638 (
639 [0x17A0, 0x17A2, 0x17A7, 0x17AB, 0x17AC, 0x17AF],
640 WidthState.KHMER_COENG_ELIGIBLE_LETTER,
641 ),
642 ([0x17A4], WidthState.WIDE),
643 ([0x17D8], WidthState.THREE),
644 ([0x1A10], WidthState.BUGINESE_LETTER_YA),
645 (range(0x2D31, 0x2D66), WidthState.TIFINAGH_CONSONANT),
646 ([0x2D6F], WidthState.TIFINAGH_CONSONANT),
647 ([0xA4FC], WidthState.LISU_TONE_LETTER_MYA_NA_JEU),
648 ([0xA4FD], WidthState.LISU_TONE_LETTER_MYA_NA_JEU),
649 ([0xFE0F], WidthState.VARIATION_SELECTOR_16),
650 ([0x10C03], WidthState.OLD_TURKIC_LETTER_ORKHON_I),
651 ([0x16D67], WidthState.KIRAT_RAI_VOWEL_SIGN_E),
652 ([0x16D68], WidthState.KIRAT_RAI_VOWEL_SIGN_AI),
653 (emoji_presentation, WidthState.EMOJI_PRESENTATION),
654 (emoji_modifiers, WidthState.EMOJI_MODIFIER),
655 (regional_indicators, WidthState.REGIONAL_INDICATOR),
656 ]:
657 for cp in cps:
658 not_ea[cp] = width
659 ea[cp] = width
660
661 # East-Asian only
662 ea[0x0338] = WidthState.COMBINING_LONG_SOLIDUS_OVERLAY
663 ea[0xFE00] = WidthState.VARIATION_SELECTOR_1_2_OR_3
664 ea[0xFE02] = WidthState.VARIATION_SELECTOR_1_2_OR_3
665
666 # Not East Asian only
667 not_ea[0xFE01] = WidthState.VARIATION_SELECTOR_1_2_OR_3
668 not_ea[0xFE0E] = WidthState.VARIATION_SELECTOR_15
669
670 return (not_ea, ea)
671
672
673 def load_joining_group_lam() -> list[tuple[Codepoint, Codepoint]]:
674 "Returns a list of character ranges with Joining_Group=Lam"
675 lam_joining = []
676 load_property(
677 "extracted/DerivedJoiningGroup.txt",
678 "Lam",
679 lambda cp: lam_joining.append(cp),
680 )
681
682 return to_sorted_ranges(lam_joining)
683
684
685 def load_non_transparent_zero_widths(
686 width_map: list[WidthState],
687 ) -> list[tuple[Codepoint, Codepoint]]:
688 "Returns a list of characters with zero width but not 'Joining_Type=Transparent'"
689
690 zero_widths = set()
691 for cp, width in enumerate(width_map):
692 if width.width_alone() == 0:
693 zero_widths.add(cp)
694 transparent = set()
695 load_property(
696 "extracted/DerivedJoiningType.txt",
697 "T",
698 lambda cp: transparent.add(cp),
699 )
700
701 return to_sorted_ranges(zero_widths - transparent)
702
703
704 def load_ligature_transparent() -> list[tuple[Codepoint, Codepoint]]:
705 """Returns a list of character ranges corresponding to all combining marks that are also
706 `Default_Ignorable_Code_Point`s, plus ZWJ. This is the set of characters that won't interrupt
707 a ligature."""
708 default_ignorables = set()
709 load_property(
710 "DerivedCoreProperties.txt",
711 "Default_Ignorable_Code_Point",
712 lambda cp: default_ignorables.add(cp),
713 )
714
715 combining_marks = set()
716 load_property(
717 "extracted/DerivedGeneralCategory.txt",
718 "(?:Mc|Mn|Me)",
719 lambda cp: combining_marks.add(cp),
720 )
721
722 default_ignorable_combinings = default_ignorables.intersection(combining_marks)
723 default_ignorable_combinings.add(0x200D) # ZWJ
724
725 return to_sorted_ranges(default_ignorable_combinings)
726
727
728 def load_solidus_transparent(
729 ligature_transparents: list[tuple[Codepoint, Codepoint]],
730 cjk_width_map: list[WidthState],
731 ) -> list[tuple[Codepoint, Codepoint]]:
732 """Characters expanding to a canonical combining class above 1, plus `ligature_transparent`s from above.
733 Ranges matching ones in `ligature_transparent` exactly are excluded (for compression), so it needs to be checked also.
734 """
735
736 ccc_above_1 = set()
737 load_property(
738 "extracted/DerivedCombiningClass.txt",
739 "(?:[2-9]|(?:[1-9][0-9]+))",
740 lambda cp: ccc_above_1.add(cp),
741 )
742
743 for lo, hi in ligature_transparents:
744 for cp in range(lo, hi + 1):
745 ccc_above_1.add(cp)
746
747 num_chars = len(ccc_above_1)
748
749 # Recursive decompositions
750 while True:
751 with fetch_open("UnicodeData.txt") as udata:
752 single = re.compile(r"([0-9A-Z]+);.*?;.*?;.*?;.*?;([0-9A-F ]+);")
753 for line in udata.readlines():
754 if match := single.match(line):
755 composed = int(match.group(1), 16)
756 decomposed = [int(c, 16) for c in match.group(2).split(" ")]
757 if all([c in ccc_above_1 for c in decomposed]):
758 ccc_above_1.add(composed)
759 if len(ccc_above_1) == num_chars:
760 break
761 else:
762 num_chars = len(ccc_above_1)
763
764 for cp in ccc_above_1:
765 if cp not in [0xFE00, 0xFE02, 0xFE0F]:
766 assert (
767 cjk_width_map[cp].table_width() != CharWidthInTable.SPECIAL
768 ), f"U+{cp:X}"
769
770 sorted = to_sorted_ranges(ccc_above_1)
771 return list(filter(lambda range: range not in ligature_transparents, sorted))
772
773
774 def load_normalization_tests() -> list[tuple[str, str, str, str, str]]:
775 def parse_codepoints(cps: str) -> str:
776 return "".join(map(lambda cp: chr(int(cp, 16)), cps.split(" ")))
777
778 with fetch_open("NormalizationTest.txt") as normtests:
779 ret = []
780 single = re.compile(
781 r"^([0-9A-F ]+);([0-9A-F ]+);([0-9A-F ]+);([0-9A-F ]+);([0-9A-F ]+);"
782 )
783 for line in normtests.readlines():
784 if match := single.match(line):
785 ret.append(
786 (
787 parse_codepoints(match.group(1)),
788 parse_codepoints(match.group(2)),
789 parse_codepoints(match.group(3)),
790 parse_codepoints(match.group(4)),
791 parse_codepoints(match.group(5)),
792 )
793 )
794 return ret
795
796
797 def make_special_ranges(
798 width_map: list[WidthState],
799 ) -> list[tuple[tuple[Codepoint, Codepoint], WidthState]]:
800 "Assign ranges of characters to their special behavior (used in match)"
801 ret = []
802 can_merge_with_prev = False
803 for cp, width in enumerate(width_map):
804 if width == WidthState.EMOJI_PRESENTATION:
805 can_merge_with_prev = False
806 elif width.table_width() == CharWidthInTable.SPECIAL:
807 if can_merge_with_prev and ret[-1][1] == width:
808 ret[-1] = ((ret[-1][0][0], cp), width)
809 else:
810 ret.append(((cp, cp), width))
811 can_merge_with_prev = True
812 return ret
813
814
815 class Bucket:
816 """A bucket contains a group of codepoints and an ordered width list. If one bucket's width
817 list overlaps with another's width list, those buckets can be merged via `try_extend`.
818 """
819
820 def __init__(self):
821 """Creates an empty bucket."""
822 self.entry_set = set()
823 self.widths = []
824
825 def append(self, codepoint: Codepoint, width: CharWidthInTable):
826 """Adds a codepoint/width pair to the bucket, and appends `width` to the width list."""
827 self.entry_set.add((codepoint, width))
828 self.widths.append(width)
829
830 def try_extend(self, attempt: "Bucket") -> bool:
831 """If either `self` or `attempt`'s width list starts with the other bucket's width list,
832 set `self`'s width list to the longer of the two, add all of `attempt`'s codepoints
833 into `self`, and return `True`. Otherwise, return `False`."""
834 (less, more) = (self.widths, attempt.widths)
835 if len(self.widths) > len(attempt.widths):
836 (less, more) = (attempt.widths, self.widths)
837 if less != more[: len(less)]:
838 return False
839 self.entry_set |= attempt.entry_set
840 self.widths = more
841 return True
842
843 def entries(self) -> list[tuple[Codepoint, CharWidthInTable]]:
844 """Return a list of the codepoint/width pairs in this bucket, sorted by codepoint."""
845 result = list(self.entry_set)
846 result.sort()
847 return result
848
849 def width(self) -> CharWidthInTable | None:
850 """If all codepoints in this bucket have the same width, return that width; otherwise,
851 return `None`."""
852 if len(self.widths) == 0:
853 return None
854 potential_width = self.widths[0]
855 for width in self.widths[1:]:
856 if potential_width != width:
857 return None
858 return potential_width
859
860
861 def make_buckets(
862 entries: Iterable[tuple[int, CharWidthInTable]], low_bit: BitPos, cap_bit: BitPos
863 ) -> list[Bucket]:
864 """Partitions the `(Codepoint, EffectiveWidth)` tuples in `entries` into `Bucket`s. All
865 codepoints with identical bits from `low_bit` to `cap_bit` (exclusive) are placed in the
866 same bucket. Returns a list of the buckets in increasing order of those bits."""
867 num_bits = cap_bit - low_bit
868 assert num_bits > 0
869 buckets = [Bucket() for _ in range(0, 2**num_bits)]
870 mask = (1 << num_bits) - 1
871 for codepoint, width in entries:
872 buckets[(codepoint >> low_bit) & mask].append(codepoint, width)
873 return buckets
874
875
876 class Table:
877 """Represents a lookup table. Each table contains a certain number of subtables; each
878 subtable is indexed by a contiguous bit range of the codepoint and contains a list
879 of `2**(number of bits in bit range)` entries. (The bit range is the same for all subtables.)
880
881 Typically, tables contain a list of buckets of codepoints. Bucket `i`'s codepoints should
882 be indexed by sub-table `i` in the next-level lookup table. The entries of this table are
883 indexes into the bucket list (~= indexes into the sub-tables of the next-level table.) The
884 key to compression is that two different buckets in two different sub-tables may have the
885 same width list, which means that they can be merged into the same bucket.
886
887 If no bucket contains two codepoints with different widths, calling `indices_to_widths` will
888 discard the buckets and convert the entries into `EffectiveWidth` values."""
889
890 def __init__(
891 self,
892 name: str,
893 entry_groups: Iterable[Iterable[tuple[int, CharWidthInTable]]],
894 secondary_entry_groups: Iterable[Iterable[tuple[int, CharWidthInTable]]],
895 low_bit: BitPos,
896 cap_bit: BitPos,
897 offset_type: OffsetType,
898 align: int,
899 bytes_per_row: int | None = None,
900 starting_indexed: list[Bucket] = [],
901 cfged: bool = False,
902 ):
903 """Create a lookup table with a sub-table for each `(Codepoint, EffectiveWidth)` iterator
904 in `entry_groups`. Each sub-table is indexed by codepoint bits in `low_bit..cap_bit`,
905 and each table entry is represented in the format specified by `offset_type`. Asserts
906 that this table is actually representable with `offset_type`."""
907 starting_indexed_len = len(starting_indexed)
908 self.name = name
909 self.low_bit = low_bit
910 self.cap_bit = cap_bit
911 self.offset_type = offset_type
912 self.entries: list[int] = []
913 self.indexed: list[Bucket] = list(starting_indexed)
914 self.align = align
915 self.bytes_per_row = bytes_per_row
916 self.cfged = cfged
917
918 buckets: list[Bucket] = []
919 for entries in entry_groups:
920 buckets.extend(make_buckets(entries, self.low_bit, self.cap_bit))
921
922 for bucket in buckets:
923 for i, existing in enumerate(self.indexed):
924 if existing.try_extend(bucket):
925 self.entries.append(i)
926 break
927 else:
928 self.entries.append(len(self.indexed))
929 self.indexed.append(bucket)
930
931 self.primary_len = len(self.entries)
932 self.primary_bucket_len = len(self.indexed)
933
934 buckets = []
935 for entries in secondary_entry_groups:
936 buckets.extend(make_buckets(entries, self.low_bit, self.cap_bit))
937
938 for bucket in buckets:
939 for i, existing in enumerate(self.indexed):
940 if existing.try_extend(bucket):
941 self.entries.append(i)
942 break
943 else:
944 self.entries.append(len(self.indexed))
945 self.indexed.append(bucket)
946
947 # Validate offset type
948 max_index = 1 << int(self.offset_type)
949 for index in self.entries:
950 assert index < max_index, f"{index} <= {max_index}"
951
952 self.indexed = self.indexed[starting_indexed_len:]
953
954 def indices_to_widths(self):
955 """Destructively converts the indices in this table to the `EffectiveWidth` values of
956 their buckets. Assumes that no bucket contains codepoints with different widths.
957 """
958 self.entries = list(map(lambda i: int(self.indexed[i].width()), self.entries)) # type: ignore
959 del self.indexed
960
961 def buckets(self):
962 """Returns an iterator over this table's buckets."""
963 return self.indexed
964
965 def to_bytes(self) -> list[int]:
966 """Returns this table's entries as a list of bytes. The bytes are formatted according to
967 the `OffsetType` which the table was created with, converting any `EffectiveWidth` entries
968 to their enum variant's integer value. For example, with `OffsetType.U2`, each byte will
969 contain four packed 2-bit entries."""
970 entries_per_byte = 8 // int(self.offset_type)
971 byte_array = []
972 for i in range(0, len(self.entries), entries_per_byte):
973 byte = 0
974 for j in range(0, entries_per_byte):
975 byte |= self.entries[i + j] << (j * int(self.offset_type))
976 byte_array.append(byte)
977 return byte_array
978
979
980 def make_tables(
981 width_map: list[WidthState],
982 cjk_width_map: list[WidthState],
983 ) -> list[Table]:
984 """Creates a table for each configuration in `table_cfgs`, with the first config corresponding
985 to the top-level lookup table, the second config corresponding to the second-level lookup
986 table, and so forth. `entries` is an iterator over the `(Codepoint, EffectiveWidth)` pairs
987 to include in the top-level table."""
988
989 entries = enumerate([w.table_width() for w in width_map])
990 cjk_entries = enumerate([w.table_width() for w in cjk_width_map])
991
992 root_table = Table(
993 "WIDTH_ROOT",
994 [entries],
995 [],
996 TABLE_SPLITS[1],
997 MAX_CODEPOINT_BITS,
998 OffsetType.U8,
999 128,
1000 )
1001
1002 cjk_root_table = Table(
1003 "WIDTH_ROOT_CJK",
1004 [cjk_entries],
1005 [],
1006 TABLE_SPLITS[1],
1007 MAX_CODEPOINT_BITS,
1008 OffsetType.U8,
1009 128,
1010 starting_indexed=root_table.indexed,
1011 cfged=True,
1012 )
1013
1014 middle_table = Table(
1015 "WIDTH_MIDDLE",
1016 map(lambda bucket: bucket.entries(), root_table.buckets()),
1017 map(lambda bucket: bucket.entries(), cjk_root_table.buckets()),
1018 TABLE_SPLITS[0],
1019 TABLE_SPLITS[1],
1020 OffsetType.U8,
1021 2 ** (TABLE_SPLITS[1] - TABLE_SPLITS[0]),
1022 bytes_per_row=2 ** (TABLE_SPLITS[1] - TABLE_SPLITS[0]),
1023 )
1024
1025 leaves_table = Table(
1026 "WIDTH_LEAVES",
1027 map(
1028 lambda bucket: bucket.entries(),
1029 middle_table.buckets()[: middle_table.primary_bucket_len],
1030 ),
1031 map(
1032 lambda bucket: bucket.entries(),
1033 middle_table.buckets()[middle_table.primary_bucket_len :],
1034 ),
1035 0,
1036 TABLE_SPLITS[0],
1037 OffsetType.U2,
1038 2 ** (TABLE_SPLITS[0] - 2),
1039 bytes_per_row=2 ** (TABLE_SPLITS[0] - 2),
1040 )
1041
1042 return [root_table, cjk_root_table, middle_table, leaves_table]
1043
1044
1045 def load_emoji_presentation_sequences() -> list[Codepoint]:
1046 """Outputs a list of cpodepoints, corresponding to all the valid characters for starting
1047 an emoji presentation sequence."""
1048
1049 with fetch_open("emoji/emoji-variation-sequences.txt") as sequences:
1050 # Match all emoji presentation sequences
1051 # (one codepoint followed by U+FE0F, and labeled "emoji style")
1052 sequence = re.compile(r"^([0-9A-F]+)\s+FE0F\s*;\s*emoji style")
1053 codepoints = []
1054 for line in sequences.readlines():
1055 if match := sequence.match(line):
1056 cp = int(match.group(1), 16)
1057 codepoints.append(cp)
1058 return codepoints
1059
1060
1061 def load_text_presentation_sequences() -> list[Codepoint]:
1062 """Outputs a list of codepoints, corresponding to all the valid characters
1063 whose widths change with a text presentation sequence."""
1064
1065 text_presentation_seq_codepoints = set()
1066 with fetch_open("emoji/emoji-variation-sequences.txt") as sequences:
1067 # Match all text presentation sequences
1068 # (one codepoint followed by U+FE0E, and labeled "text style")
1069 sequence = re.compile(r"^([0-9A-F]+)\s+FE0E\s*;\s*text style")
1070 for line in sequences.readlines():
1071 if match := sequence.match(line):
1072 cp = int(match.group(1), 16)
1073 text_presentation_seq_codepoints.add(cp)
1074
1075 default_emoji_codepoints = set()
1076
1077 load_property(
1078 "emoji/emoji-data.txt",
1079 "Emoji_Presentation",
1080 lambda cp: default_emoji_codepoints.add(cp),
1081 )
1082
1083 codepoints = []
1084 for cp in text_presentation_seq_codepoints.intersection(default_emoji_codepoints):
1085 # "Enclosed Ideographic Supplement" block;
1086 # wide even in text presentation
1087 if not cp in range(0x1F200, 0x1F300):
1088 codepoints.append(cp)
1089
1090 codepoints.sort()
1091 return codepoints
1092
1093
1094 def load_emoji_modifier_bases() -> list[Codepoint]:
1095 """Outputs a list of codepoints, corresponding to all the valid characters
1096 whose widths change with a text presentation sequence."""
1097
1098 ret = []
1099 load_property(
1100 "emoji/emoji-data.txt",
1101 "Emoji_Modifier_Base",
1102 lambda cp: ret.append(cp),
1103 )
1104 ret.sort()
1105 return ret
1106
1107
1108 def make_presentation_sequence_table(
1109 seqs: list[Codepoint],
1110 lsb: int = 10,
1111 ) -> tuple[list[tuple[int, int]], list[list[int]]]:
1112 """Generates 2-level lookup table for whether a codepoint might start an emoji variation sequence.
1113 The first level is a match on all but the 10 LSB, the second level is a 1024-bit bitmap for those 10 LSB.
1114 """
1115
1116 prefixes_dict = defaultdict(set)
1117 for cp in seqs:
1118 prefixes_dict[cp >> lsb].add(cp & (2**lsb - 1))
1119
1120 msbs: list[int] = list(prefixes_dict.keys())
1121
1122 leaves: list[list[int]] = []
1123 for cps in prefixes_dict.values():
1124 leaf = [0] * (2 ** (lsb - 3))
1125 for cp in cps:
1126 idx_in_leaf, bit_shift = divmod(cp, 8)
1127 leaf[idx_in_leaf] |= 1 << bit_shift
1128 leaves.append(leaf)
1129
1130 indexes = [(msb, index) for (index, msb) in enumerate(msbs)]
1131
1132 # Cull duplicate leaves
1133 i = 0
1134 while i < len(leaves):
1135 first_idx = leaves.index(leaves[i])
1136 if first_idx == i:
1137 i += 1
1138 else:
1139 for j in range(0, len(indexes)):
1140 if indexes[j][1] == i:
1141 indexes[j] = (indexes[j][0], first_idx)
1142 elif indexes[j][1] > i:
1143 indexes[j] = (indexes[j][0], indexes[j][1] - 1)
1144
1145 leaves.pop(i)
1146
1147 return (indexes, leaves)
1148
1149
1150 def make_ranges_table(
1151 seqs: list[Codepoint],
1152 ) -> tuple[list[tuple[int, int]], list[list[tuple[int, int]]]]:
1153 """Generates 2-level lookup table for a binary property of a codepoint.
1154 First level is all but the last byte, second level is ranges for last byte
1155 """
1156
1157 prefixes_dict = defaultdict(list)
1158 for cp in seqs:
1159 prefixes_dict[cp >> 8].append(cp & 0xFF)
1160
1161 msbs: list[int] = list(prefixes_dict.keys())
1162
1163 leaves: list[list[tuple[int, int]]] = []
1164 for cps in prefixes_dict.values():
1165 leaf = []
1166 for cp in cps:
1167 if len(leaf) > 0 and leaf[-1][1] == cp - 1:
1168 leaf[-1] = (leaf[-1][0], cp)
1169 else:
1170 leaf.append((cp, cp))
1171 leaves.append(leaf)
1172
1173 indexes = [(msb, index) for (index, msb) in enumerate(msbs)]
1174
1175 # Cull duplicate leaves
1176 i = 0
1177 while i < len(leaves):
1178 first_idx = leaves.index(leaves[i])
1179 if first_idx == i:
1180 i += 1
1181 else:
1182 for j in range(0, len(indexes)):
1183 if indexes[j][1] == i:
1184 indexes[j] = (indexes[j][0], first_idx)
1185 elif indexes[j][1] > i:
1186 indexes[j] = (indexes[j][0], indexes[j][1] - 1)
1187
1188 leaves.pop(i)
1189
1190 return (indexes, leaves)
1191
1192
1193 def lookup_fns(
1194 is_cjk: bool,
1195 special_ranges: list[tuple[tuple[Codepoint, Codepoint], WidthState]],
1196 joining_group_lam: list[tuple[Codepoint, Codepoint]],
1197 ) -> str:
1198 if is_cjk:
1199 cfg = '#[cfg(feature = "cjk")]\n'
1200 cjk_lo = "_cjk"
1201 cjk_cap = "_CJK"
1202 ambig = "wide"
1203 else:
1204 cfg = ""
1205 cjk_lo = ""
1206 cjk_cap = ""
1207 ambig = "narrow"
1208 s = f"""
1209 /// Returns the [UAX #11](https://www.unicode.org/reports/tr11/) based width of `c` by
1210 /// consulting a multi-level lookup table.
1211 ///
1212 /// # Maintenance
1213 /// The tables themselves are autogenerated but this function is hardcoded. You should have
1214 /// nothing to worry about if you re-run `unicode.py` (for example, when updating Unicode.)
1215 /// However, if you change the *actual structure* of the lookup tables (perhaps by editing the
1216 /// `make_tables` function in `unicode.py`) you must ensure that this code reflects those changes.
1217 {cfg}#[inline]
1218 fn lookup_width{cjk_lo}(c: char) -> (u8, WidthInfo) {{
1219 let cp = c as usize;
1220
1221 let t1_offset = WIDTH_ROOT{cjk_cap}.0[cp >> {TABLE_SPLITS[1]}];
1222
1223 // Each sub-table in WIDTH_MIDDLE is 7 bits, and each stored entry is a byte,
1224 // so each sub-table is 128 bytes in size.
1225 // (Sub-tables are selected using the computed offset from the previous table.)
1226 let t2_offset = WIDTH_MIDDLE.0[usize::from(t1_offset)][cp >> {TABLE_SPLITS[0]} & 0x{(2 ** (TABLE_SPLITS[1] - TABLE_SPLITS[0]) - 1):X}];
1227
1228 // Each sub-table in WIDTH_LEAVES is 6 bits, but each stored entry is 2 bits.
1229 // This is accomplished by packing four stored entries into one byte.
1230 // So each sub-table is 2**(7-2) == 32 bytes in size.
1231 // Since this is the last table, each entry represents an encoded width.
1232 let packed_widths = WIDTH_LEAVES.0[usize::from(t2_offset)][cp >> 2 & 0x{(2 ** (TABLE_SPLITS[0] - 2) - 1):X}];
1233
1234 // Extract the packed width
1235 let width = packed_widths >> (2 * (cp & 0b11)) & 0b11;
1236
1237 if width < 3 {{
1238 (width, WidthInfo::DEFAULT)
1239 }} else {{
1240 match c {{
1241 """
1242
1243 for (lo, hi), width in special_ranges:
1244 s += f" '\\u{{{lo:X}}}'"
1245 if hi != lo:
1246 s += f"..='\\u{{{hi:X}}}'"
1247 if width.is_carried():
1248 width_info = width.name
1249 else:
1250 width_info = "DEFAULT"
1251 s += f" => ({width.width_alone()}, WidthInfo::{width_info}),\n"
1252
1253 s += f""" _ => (2, WidthInfo::EMOJI_PRESENTATION),
1254 }}
1255 }}
1256 }}
1257
1258 /// Returns the [UAX #11](https://www.unicode.org/reports/tr11/) based width of `c`, or
1259 /// `None` if `c` is a control character.
1260 /// Ambiguous width characters are treated as {ambig}.
1261 {cfg}#[inline]
1262 pub fn single_char_width{cjk_lo}(c: char) -> Option<usize> {{
1263 if c < '\\u{{7F}}' {{
1264 if c >= '\\u{{20}}' {{
1265 // U+0020 to U+007F (exclusive) are single-width ASCII codepoints
1266 Some(1)
1267 }} else {{
1268 // U+0000 to U+0020 (exclusive) are control codes
1269 None
1270 }}
1271 }} else if c >= '\\u{{A0}}' {{
1272 // No characters >= U+00A0 are control codes, so we can consult the lookup tables
1273 Some(lookup_width{cjk_lo}(c).0.into())
1274 }} else {{
1275 // U+007F to U+00A0 (exclusive) are control codes
1276 None
1277 }}
1278 }}
1279
1280 /// Returns the [UAX #11](https://www.unicode.org/reports/tr11/) based width of `c`.
1281 /// Ambiguous width characters are treated as {ambig}.
1282 {cfg}#[inline]
1283 fn width_in_str{cjk_lo}(c: char, mut next_info: WidthInfo) -> (i8, WidthInfo) {{
1284 if next_info.is_emoji_presentation() {{
1285 if starts_emoji_presentation_seq(c) {{
1286 let width = if next_info.is_zwj_emoji_presentation() {{
1287 0
1288 }} else {{
1289 2
1290 }};
1291 return (width, WidthInfo::EMOJI_PRESENTATION);
1292 }} else {{
1293 next_info = next_info.unset_emoji_presentation();
1294 }}
1295 }}"""
1296
1297 if is_cjk:
1298 s += """
1299 if (matches!(
1300 next_info,
1301 WidthInfo::COMBINING_LONG_SOLIDUS_OVERLAY | WidthInfo::SOLIDUS_OVERLAY_ALEF
1302 ) && matches!(c, '<' | '=' | '>'))
1303 {
1304 return (2, WidthInfo::DEFAULT);
1305 }"""
1306
1307 s += """
1308 if c <= '\\u{A0}' {
1309 match c {
1310 '\\n' => (1, WidthInfo::LINE_FEED),
1311 '\\r' if next_info == WidthInfo::LINE_FEED => (0, WidthInfo::DEFAULT),
1312 _ => (1, WidthInfo::DEFAULT),
1313 }
1314 } else {
1315 // Fast path
1316 if next_info != WidthInfo::DEFAULT {
1317 if c == '\\u{FE0F}' {
1318 return (0, next_info.set_emoji_presentation());
1319 }"""
1320
1321 if is_cjk:
1322 s += """
1323 if matches!(c, '\\u{FE00}' | '\\u{FE02}') {
1324 return (0, next_info.set_vs1_2_3());
1325 }
1326 """
1327 else:
1328 s += """
1329 if c == '\\u{FE01}' {
1330 return (0, next_info.set_vs1_2_3());
1331 }
1332 if c == '\\u{FE0E}' {
1333 return (0, next_info.set_text_presentation());
1334 }
1335 if next_info.is_text_presentation() {
1336 if starts_non_ideographic_text_presentation_seq(c) {
1337 return (1, WidthInfo::DEFAULT);
1338 } else {
1339 next_info = next_info.unset_text_presentation();
1340 }
1341 } else """
1342
1343 s += """if next_info.is_vs1_2_3() {
1344 if matches!(c, '\\u{2018}' | '\\u{2019}' | '\\u{201C}' | '\\u{201D}') {
1345 return ("""
1346
1347 s += str(2 - is_cjk)
1348
1349 s += """, WidthInfo::DEFAULT);
1350 } else {
1351 next_info = next_info.unset_vs1_2_3();
1352 }
1353 }
1354 if next_info.is_ligature_transparent() {
1355 if c == '\\u{200D}' {
1356 return (0, next_info.set_zwj_bit());
1357 } else if is_ligature_transparent(c) {
1358 return (0, next_info);
1359 }
1360 }
1361
1362 match (next_info, c) {"""
1363 if is_cjk:
1364 s += """
1365 (WidthInfo::COMBINING_LONG_SOLIDUS_OVERLAY, _) if is_solidus_transparent(c) => {
1366 return (
1367 lookup_width_cjk(c).0 as i8,
1368 WidthInfo::COMBINING_LONG_SOLIDUS_OVERLAY,
1369 );
1370 }
1371 (WidthInfo::JOINING_GROUP_ALEF, '\\u{0338}') => {
1372 return (0, WidthInfo::SOLIDUS_OVERLAY_ALEF);
1373 }
1374 // Arabic Lam-Alef ligature
1375 (
1376 WidthInfo::JOINING_GROUP_ALEF | WidthInfo::SOLIDUS_OVERLAY_ALEF,
1377 """
1378 else:
1379 s += """
1380 // Arabic Lam-Alef ligature
1381 (
1382 WidthInfo::JOINING_GROUP_ALEF,
1383 """
1384
1385 tail = False
1386 for lo, hi in joining_group_lam:
1387 if tail:
1388 s += " | "
1389 tail = True
1390 s += f"'\\u{{{lo:X}}}'"
1391 if hi != lo:
1392 s += f"..='\\u{{{hi:X}}}'"
1393 s += """,
1394 ) => return (0, WidthInfo::DEFAULT),
1395 (WidthInfo::JOINING_GROUP_ALEF, _) if is_transparent_zero_width(c) => {
1396 return (0, WidthInfo::JOINING_GROUP_ALEF);
1397 }
1398
1399 // Hebrew Alef-ZWJ-Lamed ligature
1400 (WidthInfo::ZWJ_HEBREW_LETTER_LAMED, '\\u{05D0}') => {
1401 return (0, WidthInfo::DEFAULT);
1402 }
1403
1404 // Khmer coeng signs
1405 (WidthInfo::KHMER_COENG_ELIGIBLE_LETTER, '\\u{17D2}') => {
1406 return (-1, WidthInfo::DEFAULT);
1407 }
1408
1409 // Buginese <a, -i> ZWJ ya ligature
1410 (WidthInfo::ZWJ_BUGINESE_LETTER_YA, '\\u{1A17}') => {
1411 return (0, WidthInfo::BUGINESE_VOWEL_SIGN_I_ZWJ_LETTER_YA)
1412 }
1413 (WidthInfo::BUGINESE_VOWEL_SIGN_I_ZWJ_LETTER_YA, '\\u{1A15}') => {
1414 return (0, WidthInfo::DEFAULT)
1415 }
1416
1417 // Tifinagh bi-consonants
1418 (WidthInfo::TIFINAGH_CONSONANT | WidthInfo::ZWJ_TIFINAGH_CONSONANT, '\\u{2D7F}') => {
1419 return (1, WidthInfo::TIFINAGH_JOINER_CONSONANT);
1420 }
1421 (WidthInfo::ZWJ_TIFINAGH_CONSONANT, '\\u{2D31}'..='\\u{2D65}' | '\\u{2D6F}') => {
1422 return (0, WidthInfo::DEFAULT);
1423 }
1424 (WidthInfo::TIFINAGH_JOINER_CONSONANT, '\\u{2D31}'..='\\u{2D65}' | '\\u{2D6F}') => {
1425 return (-1, WidthInfo::DEFAULT);
1426 }
1427
1428 // Lisu tone letter combinations
1429 (WidthInfo::LISU_TONE_LETTER_MYA_NA_JEU, '\\u{A4F8}'..='\\u{A4FB}') => {
1430 return (0, WidthInfo::DEFAULT);
1431 }
1432
1433 // Old Turkic ligature
1434 (WidthInfo::ZWJ_OLD_TURKIC_LETTER_ORKHON_I, '\\u{10C32}') => {
1435 return (0, WidthInfo::DEFAULT);
1436 }"""
1437
1438 s += f"""
1439 // Emoji modifier
1440 (WidthInfo::EMOJI_MODIFIER, _) if is_emoji_modifier_base(c) => {{
1441 return (0, WidthInfo::EMOJI_PRESENTATION);
1442 }}
1443
1444 // Regional indicator
1445 (
1446 WidthInfo::REGIONAL_INDICATOR | WidthInfo::SEVERAL_REGIONAL_INDICATOR,
1447 '\\u{{1F1E6}}'..='\\u{{1F1FF}}',
1448 ) => return (1, WidthInfo::SEVERAL_REGIONAL_INDICATOR),
1449
1450 // ZWJ emoji
1451 (
1452 WidthInfo::EMOJI_PRESENTATION
1453 | WidthInfo::SEVERAL_REGIONAL_INDICATOR
1454 | WidthInfo::EVEN_REGIONAL_INDICATOR_ZWJ_PRESENTATION
1455 | WidthInfo::ODD_REGIONAL_INDICATOR_ZWJ_PRESENTATION
1456 | WidthInfo::EMOJI_MODIFIER,
1457 '\\u{{200D}}',
1458 ) => return (0, WidthInfo::ZWJ_EMOJI_PRESENTATION),
1459 (WidthInfo::ZWJ_EMOJI_PRESENTATION, '\\u{{20E3}}') => {{
1460 return (0, WidthInfo::KEYCAP_ZWJ_EMOJI_PRESENTATION);
1461 }}
1462 (WidthInfo::VS16_ZWJ_EMOJI_PRESENTATION, _) if starts_emoji_presentation_seq(c) => {{
1463 return (0, WidthInfo::EMOJI_PRESENTATION)
1464 }}
1465 (WidthInfo::VS16_KEYCAP_ZWJ_EMOJI_PRESENTATION, '0'..='9' | '#' | '*') => {{
1466 return (0, WidthInfo::EMOJI_PRESENTATION)
1467 }}
1468 (WidthInfo::ZWJ_EMOJI_PRESENTATION, '\\u{{1F1E6}}'..='\\u{{1F1FF}}') => {{
1469 return (1, WidthInfo::REGIONAL_INDICATOR_ZWJ_PRESENTATION);
1470 }}
1471 (
1472 WidthInfo::REGIONAL_INDICATOR_ZWJ_PRESENTATION
1473 | WidthInfo::ODD_REGIONAL_INDICATOR_ZWJ_PRESENTATION,
1474 '\\u{{1F1E6}}'..='\\u{{1F1FF}}',
1475 ) => return (-1, WidthInfo::EVEN_REGIONAL_INDICATOR_ZWJ_PRESENTATION),
1476 (
1477 WidthInfo::EVEN_REGIONAL_INDICATOR_ZWJ_PRESENTATION,
1478 '\\u{{1F1E6}}'..='\\u{{1F1FF}}',
1479 ) => return (3, WidthInfo::ODD_REGIONAL_INDICATOR_ZWJ_PRESENTATION),
1480 (WidthInfo::ZWJ_EMOJI_PRESENTATION, '\\u{{1F3FB}}'..='\\u{{1F3FF}}') => {{
1481 return (0, WidthInfo::EMOJI_MODIFIER);
1482 }}
1483 (WidthInfo::ZWJ_EMOJI_PRESENTATION, '\\u{{E007F}}') => {{
1484 return (0, WidthInfo::TAG_END_ZWJ_EMOJI_PRESENTATION);
1485 }}
1486 (WidthInfo::TAG_END_ZWJ_EMOJI_PRESENTATION, '\\u{{E0061}}'..='\\u{{E007A}}') => {{
1487 return (0, WidthInfo::TAG_A1_END_ZWJ_EMOJI_PRESENTATION);
1488 }}
1489 (WidthInfo::TAG_A1_END_ZWJ_EMOJI_PRESENTATION, '\\u{{E0061}}'..='\\u{{E007A}}') => {{
1490 return (0, WidthInfo::TAG_A2_END_ZWJ_EMOJI_PRESENTATION)
1491 }}
1492 (WidthInfo::TAG_A2_END_ZWJ_EMOJI_PRESENTATION, '\\u{{E0061}}'..='\\u{{E007A}}') => {{
1493 return (0, WidthInfo::TAG_A3_END_ZWJ_EMOJI_PRESENTATION)
1494 }}
1495 (WidthInfo::TAG_A3_END_ZWJ_EMOJI_PRESENTATION, '\\u{{E0061}}'..='\\u{{E007A}}') => {{
1496 return (0, WidthInfo::TAG_A4_END_ZWJ_EMOJI_PRESENTATION)
1497 }}
1498 (WidthInfo::TAG_A4_END_ZWJ_EMOJI_PRESENTATION, '\\u{{E0061}}'..='\\u{{E007A}}') => {{
1499 return (0, WidthInfo::TAG_A5_END_ZWJ_EMOJI_PRESENTATION)
1500 }}
1501 (WidthInfo::TAG_A5_END_ZWJ_EMOJI_PRESENTATION, '\\u{{E0061}}'..='\\u{{E007A}}') => {{
1502 return (0, WidthInfo::TAG_A6_END_ZWJ_EMOJI_PRESENTATION)
1503 }}
1504 (
1505 WidthInfo::TAG_END_ZWJ_EMOJI_PRESENTATION
1506 | WidthInfo::TAG_A1_END_ZWJ_EMOJI_PRESENTATION
1507 | WidthInfo::TAG_A2_END_ZWJ_EMOJI_PRESENTATION
1508 | WidthInfo::TAG_A3_END_ZWJ_EMOJI_PRESENTATION
1509 | WidthInfo::TAG_A4_END_ZWJ_EMOJI_PRESENTATION,
1510 '\\u{{E0030}}'..='\\u{{E0039}}',
1511 ) => return (0, WidthInfo::TAG_D1_END_ZWJ_EMOJI_PRESENTATION),
1512 (WidthInfo::TAG_D1_END_ZWJ_EMOJI_PRESENTATION, '\\u{{E0030}}'..='\\u{{E0039}}') => {{
1513 return (0, WidthInfo::TAG_D2_END_ZWJ_EMOJI_PRESENTATION);
1514 }}
1515 (WidthInfo::TAG_D2_END_ZWJ_EMOJI_PRESENTATION, '\\u{{E0030}}'..='\\u{{E0039}}') => {{
1516 return (0, WidthInfo::TAG_D3_END_ZWJ_EMOJI_PRESENTATION);
1517 }}
1518 (
1519 WidthInfo::TAG_A3_END_ZWJ_EMOJI_PRESENTATION
1520 | WidthInfo::TAG_A4_END_ZWJ_EMOJI_PRESENTATION
1521 | WidthInfo::TAG_A5_END_ZWJ_EMOJI_PRESENTATION
1522 | WidthInfo::TAG_A6_END_ZWJ_EMOJI_PRESENTATION
1523 | WidthInfo::TAG_D3_END_ZWJ_EMOJI_PRESENTATION,
1524 '\\u{{1F3F4}}',
1525 ) => return (0, WidthInfo::EMOJI_PRESENTATION),
1526 (WidthInfo::ZWJ_EMOJI_PRESENTATION, _)
1527 if lookup_width{cjk_lo}(c).1 == WidthInfo::EMOJI_PRESENTATION =>
1528 {{
1529 return (0, WidthInfo::EMOJI_PRESENTATION)
1530 }}
1531
1532 (WidthInfo::KIRAT_RAI_VOWEL_SIGN_E, '\\u{{16D63}}') => {{
1533 return (0, WidthInfo::DEFAULT);
1534 }}
1535 (WidthInfo::KIRAT_RAI_VOWEL_SIGN_E, '\\u{{16D67}}') => {{
1536 return (0, WidthInfo::KIRAT_RAI_VOWEL_SIGN_AI);
1537 }}
1538 (WidthInfo::KIRAT_RAI_VOWEL_SIGN_E, '\\u{{16D68}}') => {{
1539 return (1, WidthInfo::KIRAT_RAI_VOWEL_SIGN_E);
1540 }}
1541 (WidthInfo::KIRAT_RAI_VOWEL_SIGN_E, '\\u{{16D69}}') => {{
1542 return (0, WidthInfo::DEFAULT);
1543 }}
1544 (WidthInfo::KIRAT_RAI_VOWEL_SIGN_AI, '\\u{{16D63}}') => {{
1545 return (0, WidthInfo::DEFAULT);
1546 }}
1547
1548 // Fallback
1549 _ => {{}}
1550 }}
1551 }}
1552
1553 let ret = lookup_width{cjk_lo}(c);
1554 (ret.0 as i8, ret.1)
1555 }}
1556 }}
1557
1558 {cfg}#[inline]
1559 pub fn str_width{cjk_lo}(s: &str) -> usize {{
1560 s.chars()
1561 .rfold(
1562 (0, WidthInfo::DEFAULT),
1563 |(sum, next_info), c| -> (usize, WidthInfo) {{
1564 let (add, info) = width_in_str{cjk_lo}(c, next_info);
1565 (sum.wrapping_add_signed(isize::from(add)), info)
1566 }},
1567 )
1568 .0
1569 }}
1570 """
1571
1572 return s
1573
1574
1575 def emit_module(
1576 out_name: str,
1577 unicode_version: tuple[int, int, int],
1578 tables: list[Table],
1579 special_ranges: list[tuple[tuple[Codepoint, Codepoint], WidthState]],
1580 special_ranges_cjk: list[tuple[tuple[Codepoint, Codepoint], WidthState]],
1581 emoji_presentation_table: tuple[list[tuple[int, int]], list[list[int]]],
1582 text_presentation_table: tuple[list[tuple[int, int]], list[list[tuple[int, int]]]],
1583 emoji_modifier_table: tuple[list[tuple[int, int]], list[list[tuple[int, int]]]],
1584 joining_group_lam: list[tuple[Codepoint, Codepoint]],
1585 non_transparent_zero_widths: list[tuple[Codepoint, Codepoint]],
1586 ligature_transparent: list[tuple[Codepoint, Codepoint]],
1587 solidus_transparent: list[tuple[Codepoint, Codepoint]],
1588 normalization_tests: list[tuple[str, str, str, str, str]],
1589 ):
1590 """Outputs a Rust module to `out_name` using table data from `tables`.
1591 If `TABLE_CFGS` is edited, you may need to edit the included code for `lookup_width`.
1592 """
1593 if os.path.exists(out_name):
1594 os.remove(out_name)
1595 with open(out_name, "w", newline="\n", encoding="utf-8") as module:
1596 module.write(
1597 """// Copyright 2012-2025 The Rust Project Developers. See the COPYRIGHT
1598 // file at the top-level directory of this distribution and at
1599 // http://rust-lang.org/COPYRIGHT.
1600 //
1601 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
1602 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
1603 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
1604 // option. This file may not be copied, modified, or distributed
1605 // except according to those terms.
1606
1607 // NOTE: The following code was generated by "scripts/unicode.py", do not edit directly
1608
1609 use core::cmp::Ordering;
1610
1611 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
1612 struct WidthInfo(u16);
1613
1614 const LIGATURE_TRANSPARENT_MASK: u16 = 0b0010_0000_0000_0000;
1615
1616 impl WidthInfo {
1617 /// No special handling necessary
1618 const DEFAULT: Self = Self(0);
1619 """
1620 )
1621
1622 for variant in WidthState:
1623 if variant.is_carried():
1624 if variant.is_cjk_only():
1625 module.write(' #[cfg(feature = "cjk")]\n')
1626 module.write(
1627 f" const {variant.name}: Self = Self(0b{variant.value:016b});\n"
1628 )
1629
1630 module.write(
1631 f"""
1632 /// Whether this width mode is ligature_transparent
1633 /// (has 5th MSB set.)
1634 fn is_ligature_transparent(self) -> bool {{
1635 (self.0 & 0b0000_1000_0000_0000) == 0b0000_1000_0000_0000
1636 }}
1637
1638 /// Sets 6th MSB.
1639 fn set_zwj_bit(self) -> Self {{
1640 Self(self.0 | 0b0000_0100_0000_0000)
1641 }}
1642
1643 /// Has top bit set
1644 fn is_emoji_presentation(self) -> bool {{
1645 (self.0 & WidthInfo::VARIATION_SELECTOR_16.0) == WidthInfo::VARIATION_SELECTOR_16.0
1646 }}
1647
1648 fn is_zwj_emoji_presentation(self) -> bool {{
1649 (self.0 & 0b1011_0000_0000_0000) == 0b1001_0000_0000_0000
1650 }}
1651
1652 /// Set top bit
1653 fn set_emoji_presentation(self) -> Self {{
1654 if (self.0 & LIGATURE_TRANSPARENT_MASK) == LIGATURE_TRANSPARENT_MASK
1655 || (self.0 & 0b1001_0000_0000_0000) == 0b0001_0000_0000_0000
1656 {{
1657 Self(
1658 self.0
1659 | WidthInfo::VARIATION_SELECTOR_16.0
1660 & !WidthInfo::VARIATION_SELECTOR_15.0
1661 & !WidthInfo::VARIATION_SELECTOR_1_2_OR_3.0,
1662 )
1663 }} else {{
1664 Self::VARIATION_SELECTOR_16
1665 }}
1666 }}
1667
1668 /// Clear top bit
1669 fn unset_emoji_presentation(self) -> Self {{
1670 if (self.0 & LIGATURE_TRANSPARENT_MASK) == LIGATURE_TRANSPARENT_MASK {{
1671 Self(self.0 & !WidthInfo::VARIATION_SELECTOR_16.0)
1672 }} else {{
1673 Self::DEFAULT
1674 }}
1675 }}
1676
1677 /// Has 2nd bit set
1678 fn is_text_presentation(self) -> bool {{
1679 (self.0 & WidthInfo::VARIATION_SELECTOR_15.0) == WidthInfo::VARIATION_SELECTOR_15.0
1680 }}
1681
1682 /// Set 2nd bit
1683 fn set_text_presentation(self) -> Self {{
1684 if (self.0 & LIGATURE_TRANSPARENT_MASK) == LIGATURE_TRANSPARENT_MASK {{
1685 Self(
1686 self.0
1687 | WidthInfo::VARIATION_SELECTOR_15.0
1688 & !WidthInfo::VARIATION_SELECTOR_16.0
1689 & !WidthInfo::VARIATION_SELECTOR_1_2_OR_3.0,
1690 )
1691 }} else {{
1692 Self(WidthInfo::VARIATION_SELECTOR_15.0)
1693 }}
1694 }}
1695
1696 /// Clear 2nd bit
1697 fn unset_text_presentation(self) -> Self {{
1698 Self(self.0 & !WidthInfo::VARIATION_SELECTOR_15.0)
1699 }}
1700
1701 /// Has 7th bit set
1702 fn is_vs1_2_3(self) -> bool {{
1703 (self.0 & WidthInfo::VARIATION_SELECTOR_1_2_OR_3.0)
1704 == WidthInfo::VARIATION_SELECTOR_1_2_OR_3.0
1705 }}
1706
1707 /// Set 7th bit
1708 fn set_vs1_2_3(self) -> Self {{
1709 if (self.0 & LIGATURE_TRANSPARENT_MASK) == LIGATURE_TRANSPARENT_MASK {{
1710 Self(
1711 self.0
1712 | WidthInfo::VARIATION_SELECTOR_1_2_OR_3.0
1713 & !WidthInfo::VARIATION_SELECTOR_15.0
1714 & !WidthInfo::VARIATION_SELECTOR_16.0,
1715 )
1716 }} else {{
1717 Self(WidthInfo::VARIATION_SELECTOR_1_2_OR_3.0)
1718 }}
1719 }}
1720
1721 /// Clear 7th bit
1722 fn unset_vs1_2_3(self) -> Self {{
1723 Self(self.0 & !WidthInfo::VARIATION_SELECTOR_1_2_OR_3.0)
1724 }}
1725 }}
1726
1727 /// The version of [Unicode](http://www.unicode.org/)
1728 /// that this version of unicode-width is based on.
1729 pub const UNICODE_VERSION: (u8, u8, u8) = {unicode_version};
1730 """
1731 )
1732
1733 module.write(lookup_fns(False, special_ranges, joining_group_lam))
1734 module.write(lookup_fns(True, special_ranges_cjk, joining_group_lam))
1735
1736 emoji_presentation_idx, emoji_presentation_leaves = emoji_presentation_table
1737 text_presentation_idx, text_presentation_leaves = text_presentation_table
1738 emoji_modifier_idx, emoji_modifier_leaves = emoji_modifier_table
1739
1740 module.write(
1741 """
1742 /// Whether this character is a zero-width character with
1743 /// `Joining_Type=Transparent`. Used by the Alef-Lamed ligatures.
1744 /// See also [`is_ligature_transparent`], a near-subset of this (only ZWJ is excepted)
1745 /// which is transparent for non-Arabic ligatures.
1746 fn is_transparent_zero_width(c: char) -> bool {
1747 if lookup_width(c).0 != 0 {
1748 // Not zero-width
1749 false
1750 } else {
1751 let cp: u32 = c.into();
1752 NON_TRANSPARENT_ZERO_WIDTHS
1753 .binary_search_by(|&(lo, hi)| {
1754 let lo = u32::from_le_bytes([lo[0], lo[1], lo[2], 0]);
1755 let hi = u32::from_le_bytes([hi[0], hi[1], hi[2], 0]);
1756 if cp < lo {
1757 Ordering::Greater
1758 } else if cp > hi {
1759 Ordering::Less
1760 } else {
1761 Ordering::Equal
1762 }
1763 })
1764 .is_err()
1765 }
1766 }
1767
1768 /// Whether this character is a default-ignorable combining mark
1769 /// or ZWJ. These characters won't interrupt non-Arabic ligatures.
1770 fn is_ligature_transparent(c: char) -> bool {
1771 matches!(c, """
1772 )
1773
1774 tail = False
1775 for lo, hi in ligature_transparent:
1776 if tail:
1777 module.write(" | ")
1778 tail = True
1779 module.write(f"'\\u{{{lo:X}}}'")
1780 if hi != lo:
1781 module.write(f"..='\\u{{{hi:X}}}'")
1782
1783 module.write(
1784 """)
1785 }
1786
1787 /// Whether this character is transparent wrt the effect of
1788 /// U+0338 COMBINING LONG SOLIDUS OVERLAY
1789 /// on its base character.
1790 #[cfg(feature = "cjk")]
1791 fn is_solidus_transparent(c: char) -> bool {
1792 let cp: u32 = c.into();
1793 is_ligature_transparent(c)
1794 || SOLIDUS_TRANSPARENT
1795 .binary_search_by(|&(lo, hi)| {
1796 let lo = u32::from_le_bytes([lo[0], lo[1], lo[2], 0]);
1797 let hi = u32::from_le_bytes([hi[0], hi[1], hi[2], 0]);
1798 if cp < lo {
1799 Ordering::Greater
1800 } else if cp > hi {
1801 Ordering::Less
1802 } else {
1803 Ordering::Equal
1804 }
1805 })
1806 .is_ok()
1807 }
1808
1809 /// Whether this character forms an [emoji presentation sequence]
1810 /// (https://www.unicode.org/reports/tr51/#def_emoji_presentation_sequence)
1811 /// when followed by `'\\u{FEOF}'`.
1812 /// Emoji presentation sequences are considered to have width 2.
1813 #[inline]
1814 pub fn starts_emoji_presentation_seq(c: char) -> bool {
1815 let cp: u32 = c.into();
1816 // First level of lookup uses all but 10 LSB
1817 let top_bits = cp >> 10;
1818 let idx_of_leaf: usize = match top_bits {
1819 """
1820 )
1821
1822 for msbs, i in emoji_presentation_idx:
1823 module.write(f" 0x{msbs:X} => {i},\n")
1824
1825 module.write(
1826 """ _ => return false,
1827 };
1828 // Extract the 3-9th (0-indexed) least significant bits of `cp`,
1829 // and use them to index into `leaf_row`.
1830 let idx_within_leaf = usize::try_from((cp >> 3) & 0x7F).unwrap();
1831 let leaf_byte = EMOJI_PRESENTATION_LEAVES.0[idx_of_leaf][idx_within_leaf];
1832 // Use the 3 LSB of `cp` to index into `leaf_byte`.
1833 ((leaf_byte >> (cp & 7)) & 1) == 1
1834 }
1835
1836 /// Returns `true` if `c` has default emoji presentation, but forms a [text presentation sequence]
1837 /// (https://www.unicode.org/reports/tr51/#def_text_presentation_sequence)
1838 /// when followed by `'\\u{FEOE}'`, and is not ideographic.
1839 /// Such sequences are considered to have width 1.
1840 #[inline]
1841 pub fn starts_non_ideographic_text_presentation_seq(c: char) -> bool {
1842 let cp: u32 = c.into();
1843 // First level of lookup uses all but 8 LSB
1844 let top_bits = cp >> 8;
1845 let leaf: &[(u8, u8)] = match top_bits {
1846 """
1847 )
1848
1849 for msbs, i in text_presentation_idx:
1850 module.write(f" 0x{msbs:X} => &TEXT_PRESENTATION_LEAF_{i},\n")
1851
1852 module.write(
1853 """ _ => return false,
1854 };
1855
1856 let bottom_bits = (cp & 0xFF) as u8;
1857 leaf.binary_search_by(|&(lo, hi)| {
1858 if bottom_bits < lo {
1859 Ordering::Greater
1860 } else if bottom_bits > hi {
1861 Ordering::Less
1862 } else {
1863 Ordering::Equal
1864 }
1865 })
1866 .is_ok()
1867 }
1868
1869 /// Returns `true` if `c` is an `Emoji_Modifier_Base`.
1870 #[inline]
1871 pub fn is_emoji_modifier_base(c: char) -> bool {
1872 let cp: u32 = c.into();
1873 // First level of lookup uses all but 8 LSB
1874 let top_bits = cp >> 8;
1875 let leaf: &[(u8, u8)] = match top_bits {
1876 """
1877 )
1878
1879 for msbs, i in emoji_modifier_idx:
1880 module.write(f" 0x{msbs:X} => &EMOJI_MODIFIER_LEAF_{i},\n")
1881
1882 module.write(
1883 """ _ => return false,
1884 };
1885
1886 let bottom_bits = (cp & 0xFF) as u8;
1887 leaf.binary_search_by(|&(lo, hi)| {
1888 if bottom_bits < lo {
1889 Ordering::Greater
1890 } else if bottom_bits > hi {
1891 Ordering::Less
1892 } else {
1893 Ordering::Equal
1894 }
1895 })
1896 .is_ok()
1897 }
1898
1899 #[repr(align(32))]
1900 struct Align32<T>(T);
1901
1902 #[repr(align(64))]
1903 struct Align64<T>(T);
1904
1905 #[repr(align(128))]
1906 struct Align128<T>(T);
1907 """
1908 )
1909
1910 subtable_count = 1
1911 for i, table in enumerate(tables):
1912 new_subtable_count = len(table.buckets())
1913 if i == len(tables) - 1:
1914 table.indices_to_widths() # for the last table, indices == widths
1915 byte_array = table.to_bytes()
1916
1917 if table.bytes_per_row is None:
1918 module.write(
1919 f"/// Autogenerated. {subtable_count} sub-table(s). Consult [`lookup_width`] for layout info.)\n"
1920 )
1921 if table.cfged:
1922 module.write('#[cfg(feature = "cjk")]\n')
1923 module.write(
1924 f"static {table.name}: Align{table.align}<[u8; {len(byte_array)}]> = Align{table.align}(["
1925 )
1926 for j, byte in enumerate(byte_array):
1927 # Add line breaks for every 15th entry (chosen to match what rustfmt does)
1928 if j % 16 == 0:
1929 module.write("\n ")
1930 module.write(f" 0x{byte:02X},")
1931 module.write("\n")
1932 else:
1933 num_rows = len(byte_array) // table.bytes_per_row
1934 num_primary_rows = (
1935 table.primary_len
1936 // (8 // int(table.offset_type))
1937 // table.bytes_per_row
1938 )
1939 module.write(
1940 f"""
1941 #[cfg(feature = "cjk")]
1942 const {table.name}_LEN: usize = {num_rows};
1943 #[cfg(not(feature = "cjk"))]
1944 const {table.name}_LEN: usize = {num_primary_rows};
1945 /// Autogenerated. {subtable_count} sub-table(s). Consult [`lookup_width`] for layout info.
1946 static {table.name}: Align{table.align}<[[u8; {table.bytes_per_row}]; {table.name}_LEN]> = Align{table.align}([\n"""
1947 )
1948 for row_num in range(0, num_rows):
1949 if row_num >= num_primary_rows:
1950 module.write(' #[cfg(feature = "cjk")]\n')
1951 module.write(" [\n")
1952 row = byte_array[
1953 row_num
1954 * table.bytes_per_row : (row_num + 1)
1955 * table.bytes_per_row
1956 ]
1957 for subrow in batched(row, 15):
1958 module.write(" ")
1959 for entry in subrow:
1960 module.write(f" 0x{entry:02X},")
1961 module.write("\n")
1962 module.write(" ],\n")
1963 module.write("]);\n")
1964 subtable_count = new_subtable_count
1965
1966 # non transparent zero width table
1967
1968 module.write(
1969 f"""
1970 /// Sorted list of codepoint ranges (inclusive)
1971 /// that are zero-width but not `Joining_Type=Transparent`
1972 /// FIXME: can we get better compression?
1973 static NON_TRANSPARENT_ZERO_WIDTHS: [([u8; 3], [u8; 3]); {len(non_transparent_zero_widths)}] = [
1974 """
1975 )
1976
1977 for lo, hi in non_transparent_zero_widths:
1978 module.write(
1979 f" ([0x{lo & 0xFF:02X}, 0x{lo >> 8 & 0xFF:02X}, 0x{lo >> 16:02X}], [0x{hi & 0xFF:02X}, 0x{hi >> 8 & 0xFF:02X}, 0x{hi >> 16:02X}]),\n"
1980 )
1981
1982 # solidus transparent table
1983
1984 module.write(
1985 f"""];
1986
1987 /// Sorted list of codepoint ranges (inclusive)
1988 /// that don't affect how the combining solidus applies
1989 /// (mostly ccc > 1).
1990 /// FIXME: can we get better compression?
1991 #[cfg(feature = "cjk")]
1992 static SOLIDUS_TRANSPARENT: [([u8; 3], [u8; 3]); {len(solidus_transparent)}] = [
1993 """
1994 )
1995
1996 for lo, hi in solidus_transparent:
1997 module.write(
1998 f" ([0x{lo & 0xFF:02X}, 0x{lo >> 8 & 0xFF:02X}, 0x{lo >> 16:02X}], [0x{hi & 0xFF:02X}, 0x{hi >> 8 & 0xFF:02X}, 0x{hi >> 16:02X}]),\n"
1999 )
2000
2001 # emoji table
2002
2003 module.write(
2004 f"""];
2005
2006 /// Array of 1024-bit bitmaps. Index into the correct bitmap with the 10 LSB of your codepoint
2007 /// to get whether it can start an emoji presentation sequence.
2008 static EMOJI_PRESENTATION_LEAVES: Align128<[[u8; 128]; {len(emoji_presentation_leaves)}]> = Align128([
2009 """
2010 )
2011 for leaf in emoji_presentation_leaves:
2012 module.write(" [\n")
2013 for row in batched(leaf, 15):
2014 module.write(" ")
2015 for entry in row:
2016 module.write(f" 0x{entry:02X},")
2017 module.write("\n")
2018 module.write(" ],\n")
2019
2020 module.write("]);\n")
2021
2022 # text table
2023
2024 for leaf_idx, leaf in enumerate(text_presentation_leaves):
2025 module.write(
2026 f"""
2027 #[rustfmt::skip]
2028 static TEXT_PRESENTATION_LEAF_{leaf_idx}: [(u8, u8); {len(leaf)}] = [
2029 """
2030 )
2031 for lo, hi in leaf:
2032 module.write(f" (0x{lo:02X}, 0x{hi:02X}),\n")
2033 module.write(f"];\n")
2034
2035 # emoji modifier table
2036
2037 for leaf_idx, leaf in enumerate(emoji_modifier_leaves):
2038 module.write(
2039 f"""
2040 #[rustfmt::skip]
2041 static EMOJI_MODIFIER_LEAF_{leaf_idx}: [(u8, u8); {len(leaf)}] = [
2042 """
2043 )
2044 for lo, hi in leaf:
2045 module.write(f" (0x{lo:02X}, 0x{hi:02X}),\n")
2046 module.write(f"];\n")
2047
2048 test_width_variants = []
2049 test_width_variants_cjk = []
2050 for variant in WidthState:
2051 if variant.is_carried():
2052 if not variant.is_cjk_only():
2053 test_width_variants.append(variant)
2054 if not variant.is_non_cjk_only():
2055 test_width_variants_cjk.append(variant)
2056
2057 module.write(
2058 f"""
2059 #[cfg(test)]
2060 mod tests {{
2061 use super::*;
2062
2063 fn str_width_test(s: &str, init: WidthInfo) -> isize {{
2064 s.chars()
2065 .rfold((0, init), |(sum, next_info), c| -> (isize, WidthInfo) {{
2066 let (add, info) = width_in_str(c, next_info);
2067 (sum.checked_add(isize::from(add)).unwrap(), info)
2068 }})
2069 .0
2070 }}
2071
2072 #[cfg(feature = "cjk")]
2073 fn str_width_test_cjk(s: &str, init: WidthInfo) -> isize {{
2074 s.chars()
2075 .rfold((0, init), |(sum, next_info), c| -> (isize, WidthInfo) {{
2076 let (add, info) = width_in_str_cjk(c, next_info);
2077 (sum.checked_add(isize::from(add)).unwrap(), info)
2078 }})
2079 .0
2080 }}
2081
2082 #[test]
2083 fn test_normalization() {{
2084 for &(orig, nfc, nfd, nfkc, nfkd) in &NORMALIZATION_TEST {{
2085 for init in NORMALIZATION_TEST_WIDTHS {{
2086 assert_eq!(
2087 str_width_test(orig, init),
2088 str_width_test(nfc, init),
2089 "width of X = {{orig:?}} differs from toNFC(X) = {{nfc:?}} with mode {{init:X?}}",
2090 );
2091 assert_eq!(
2092 str_width_test(orig, init),
2093 str_width_test(nfd, init),
2094 "width of X = {{orig:?}} differs from toNFD(X) = {{nfd:?}} with mode {{init:X?}}",
2095 );
2096 assert_eq!(
2097 str_width_test(nfkc, init),
2098 str_width_test(nfkd, init),
2099 "width of toNFKC(X) = {{nfkc:?}} differs from toNFKD(X) = {{nfkd:?}} with mode {{init:X?}}",
2100 );
2101 }}
2102
2103 #[cfg(feature = "cjk")]
2104 for init in NORMALIZATION_TEST_WIDTHS_CJK {{
2105 assert_eq!(
2106 str_width_test_cjk(orig, init),
2107 str_width_test_cjk(nfc, init),
2108 "CJK width of X = {{orig:?}} differs from toNFC(X) = {{nfc:?}} with mode {{init:X?}}",
2109 );
2110 assert_eq!(
2111 str_width_test_cjk(orig, init),
2112 str_width_test_cjk(nfd, init),
2113 "CJK width of X = {{orig:?}} differs from toNFD(X) = {{nfd:?}} with mode {{init:X?}}",
2114 );
2115 assert_eq!(
2116 str_width_test_cjk(nfkc, init),
2117 str_width_test_cjk(nfkd, init),
2118 "CJK width of toNFKC(X) = {{nfkc:?}} differs from toNFKD(X) = {{nfkd:?}} with mode {{init:?}}",
2119 );
2120 }}
2121 }}
2122 }}
2123
2124 static NORMALIZATION_TEST_WIDTHS: [WidthInfo; {len(test_width_variants) + 1}] = [
2125 WidthInfo::DEFAULT,\n"""
2126 )
2127
2128 for variant in WidthState:
2129 if variant.is_carried() and not variant.is_cjk_only():
2130 module.write(f" WidthInfo::{variant.name},\n")
2131
2132 module.write(
2133 f""" ];
2134
2135 #[cfg(feature = "cjk")]
2136 static NORMALIZATION_TEST_WIDTHS_CJK: [WidthInfo; {len(test_width_variants_cjk) + 1}] = [
2137 WidthInfo::DEFAULT,\n"""
2138 )
2139
2140 for variant in WidthState:
2141 if variant.is_carried() and not variant.is_non_cjk_only():
2142 module.write(f" WidthInfo::{variant.name},\n")
2143
2144 module.write(
2145 f""" ];
2146
2147 #[rustfmt::skip]
2148 static NORMALIZATION_TEST: [(&str, &str, &str, &str, &str); {len(normalization_tests)}] = [\n"""
2149 )
2150 for orig, nfc, nfd, nfkc, nfkd in normalization_tests:
2151 module.write(
2152 f' (r#"{orig}"#, r#"{nfc}"#, r#"{nfd}"#, r#"{nfkc}"#, r#"{nfkd}"#),\n'
2153 )
2154
2155 module.write(" ];\n}\n")
2156
2157
2158 def main(module_path: str):
2159 """Obtain character data from the latest version of Unicode, transform it into a multi-level
2160 lookup table for character width, and write a Rust module utilizing that table to
2161 `module_filename`.
2162
2163 See `lib.rs` for documentation of the exact width rules.
2164 """
2165 version = load_unicode_version()
2166 print(f"Generating module for Unicode {version[0]}.{version[1]}.{version[2]}")
2167
2168 (width_map, cjk_width_map) = load_width_maps()
2169
2170 tables = make_tables(width_map, cjk_width_map)
2171
2172 special_ranges = make_special_ranges(width_map)
2173 cjk_special_ranges = make_special_ranges(cjk_width_map)
2174
2175 emoji_presentations = load_emoji_presentation_sequences()
2176 emoji_presentation_table = make_presentation_sequence_table(emoji_presentations)
2177
2178 text_presentations = load_text_presentation_sequences()
2179 text_presentation_table = make_ranges_table(text_presentations)
2180
2181 emoji_modifier_bases = load_emoji_modifier_bases()
2182 emoji_modifier_table = make_ranges_table(emoji_modifier_bases)
2183
2184 joining_group_lam = load_joining_group_lam()
2185 non_transparent_zero_widths = load_non_transparent_zero_widths(width_map)
2186 ligature_transparent = load_ligature_transparent()
2187 solidus_transparent = load_solidus_transparent(ligature_transparent, cjk_width_map)
2188
2189 normalization_tests = load_normalization_tests()
2190
2191 fetch_open("emoji-test.txt", "../tests", emoji=True)
2192
2193 print("------------------------")
2194 total_size = 0
2195 for i, table in enumerate(tables):
2196 size_bytes = len(table.to_bytes())
2197 print(f"Table {i} size: {size_bytes} bytes")
2198 total_size += size_bytes
2199
2200 for s, table in [
2201 ("Emoji presentation", emoji_presentation_table),
2202 ]:
2203 index_size = len(table[0]) * (math.ceil(math.log(table[0][-1][0], 256)) + 8)
2204 print(f"{s} index size: {index_size} bytes")
2205 total_size += index_size
2206 leaves_size = len(table[1]) * len(table[1][0])
2207 print(f"{s} leaves size: {leaves_size} bytes")
2208 total_size += leaves_size
2209
2210 for s, table in [
2211 ("Text presentation", text_presentation_table),
2212 ("Emoji modifier", emoji_modifier_table),
2213 ]:
2214 index_size = len(table[0]) * (math.ceil(math.log(table[0][-1][0], 256)) + 16)
2215 print(f"{s} index size: {index_size} bytes")
2216 total_size += index_size
2217 leaves_size = 2 * sum(map(len, table[1]))
2218 print(f"{s} leaves size: {leaves_size} bytes")
2219 total_size += leaves_size
2220
2221 for s, table in [
2222 ("Non transparent zero width", non_transparent_zero_widths),
2223 ("Solidus transparent", solidus_transparent),
2224 ]:
2225 table_size = 6 * len(table)
2226 print(f"{s} table size: {table_size} bytes")
2227 total_size += table_size
2228 print("------------------------")
2229 print(f" Total size: {total_size} bytes")
2230
2231 emit_module(
2232 out_name=module_path,
2233 unicode_version=version,
2234 tables=tables,
2235 special_ranges=special_ranges,
2236 special_ranges_cjk=cjk_special_ranges,
2237 emoji_presentation_table=emoji_presentation_table,
2238 text_presentation_table=text_presentation_table,
2239 emoji_modifier_table=emoji_modifier_table,
2240 joining_group_lam=joining_group_lam,
2241 non_transparent_zero_widths=non_transparent_zero_widths,
2242 ligature_transparent=ligature_transparent,
2243 solidus_transparent=solidus_transparent,
2244 normalization_tests=normalization_tests,
2245 )
2246 print(f'Wrote to "{module_path}"')
2247
2248
2249 if __name__ == "__main__":
2250 main(MODULE_PATH)
2251
2251 lines PYTHON