fork/rust
git clone https://git.julia.blue/fork/rust
4560ea788cb
1.39.0
1#!/usr/bin/env python 2 3""" 4Regenerate Unicode tables (tables.rs). 5""" 6 7# This script uses the Unicode tables as defined 8# in the UnicodeFiles class. 9 10# Since this should not require frequent updates, we just store this 11# out-of-line and check the tables.rs file into git. 12 13# Note that the "curl" program is required for operation. 14# This script is compatible with Python 2.7 and 3.x. 15 16import argparse 17import datetime 18import fileinput 19import itertools 20import os 21import re 22import textwrap 23import subprocess 24 25from collections import defaultdict, namedtuple 26 27try: 28 # Python 3 29 from itertools import zip_longest 30 from io import StringIO 31except ImportError: 32 # Python 2 compatibility 33 zip_longest = itertools.izip_longest 34 from StringIO import StringIO 35 36try: 37 # Completely optional type hinting 38 # (Python 2 compatible using comments, 39 # see: https://mypy.readthedocs.io/en/latest/python2.html) 40 # This is very helpful in typing-aware IDE like PyCharm. 41 from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Set, Tuple 42except ImportError: 43 pass 44 45 46# We don't use enum.Enum because of Python 2.7 compatibility. 47class UnicodeFiles(object): 48 # ReadMe does not contain any Unicode data, we 49 # only use it to extract versions. 50 README = "ReadMe.txt" 51 52 DERIVED_CORE_PROPERTIES = "DerivedCoreProperties.txt" 53 DERIVED_NORMALIZATION_PROPS = "DerivedNormalizationProps.txt" 54 PROPS = "PropList.txt" 55 SCRIPTS = "Scripts.txt" 56 SPECIAL_CASING = "SpecialCasing.txt" 57 UNICODE_DATA = "UnicodeData.txt" 58 59 60# The order doesn't really matter (Python < 3.6 won't preserve it), 61# we only want to aggregate all the file names. 62ALL_UNICODE_FILES = tuple( 63 value for name, value in UnicodeFiles.__dict__.items() 64 if not name.startswith("_") 65) 66 67assert len(ALL_UNICODE_FILES) == 7, "Unexpected number of unicode files" 68 69# The directory this file is located in. 70THIS_DIR = os.path.dirname(os.path.realpath(__file__)) 71 72# Where to download the Unicode data. The downloaded files 73# will be placed in sub-directories named after Unicode version. 74FETCH_DIR = os.path.join(THIS_DIR, "downloaded") 75 76FETCH_URL_LATEST = "ftp://ftp.unicode.org/Public/UNIDATA/{filename}" 77FETCH_URL_VERSION = "ftp://ftp.unicode.org/Public/{version}/ucd/{filename}" 78 79PREAMBLE = """\ 80// NOTE: The following code was generated by "./unicode.py", do not edit directly 81 82#![allow(missing_docs, non_upper_case_globals, non_snake_case, clippy::unreadable_literal)] 83 84use crate::unicode::version::UnicodeVersion; 85use crate::unicode::bool_trie::{{BoolTrie, SmallBoolTrie}}; 86""".format(year=datetime.datetime.now().year) 87 88# Mapping taken from Table 12 from: 89# http://www.unicode.org/reports/tr44/#General_Category_Values 90EXPANDED_CATEGORIES = { 91 "Lu": ["LC", "L"], "Ll": ["LC", "L"], "Lt": ["LC", "L"], 92 "Lm": ["L"], "Lo": ["L"], 93 "Mn": ["M"], "Mc": ["M"], "Me": ["M"], 94 "Nd": ["N"], "Nl": ["N"], "No": ["N"], 95 "Pc": ["P"], "Pd": ["P"], "Ps": ["P"], "Pe": ["P"], 96 "Pi": ["P"], "Pf": ["P"], "Po": ["P"], 97 "Sm": ["S"], "Sc": ["S"], "Sk": ["S"], "So": ["S"], 98 "Zs": ["Z"], "Zl": ["Z"], "Zp": ["Z"], 99 "Cc": ["C"], "Cf": ["C"], "Cs": ["C"], "Co": ["C"], "Cn": ["C"], 100} 101 102# This is the (inclusive) range of surrogate codepoints. 103# These are not valid Rust characters. 104SURROGATE_CODEPOINTS_RANGE = (0xd800, 0xdfff) 105 106UnicodeData = namedtuple( 107 "UnicodeData", ( 108 # Conversions: 109 "to_upper", "to_lower", "to_title", 110 111 # Decompositions: canonical decompositions, compatibility decomp 112 "canon_decomp", "compat_decomp", 113 114 # Grouped: general categories and combining characters 115 "general_categories", "combines", 116 ) 117) 118 119UnicodeVersion = namedtuple( 120 "UnicodeVersion", ("major", "minor", "micro", "as_str") 121) 122 123 124def fetch_files(version=None): 125 # type: (str) -> UnicodeVersion 126 """ 127 Fetch all the Unicode files from unicode.org. 128 129 This will use cached files (stored in `FETCH_DIR`) if they exist, 130 creating them if they don't. In any case, the Unicode version 131 is always returned. 132 133 :param version: The desired Unicode version, as string. 134 (If None, defaults to latest final release available, 135 querying the unicode.org service). 136 """ 137 have_version = check_stored_version(version) 138 if have_version: 139 return have_version 140 141 if version: 142 # Check if the desired version exists on the server. 143 get_fetch_url = lambda name: FETCH_URL_VERSION.format(version=version, filename=name) 144 else: 145 # Extract the latest version. 146 get_fetch_url = lambda name: FETCH_URL_LATEST.format(filename=name) 147 148 readme_url = get_fetch_url(UnicodeFiles.README) 149 150 print("Fetching: {}".format(readme_url)) 151 readme_content = subprocess.check_output(("curl", readme_url)) 152 153 unicode_version = parse_readme_unicode_version( 154 readme_content.decode("utf8") 155 ) 156 157 download_dir = get_unicode_dir(unicode_version) 158 if not os.path.exists(download_dir): 159 # For 2.7 compat, we don't use `exist_ok=True`. 160 os.makedirs(download_dir) 161 162 for filename in ALL_UNICODE_FILES: 163 file_path = get_unicode_file_path(unicode_version, filename) 164 165 if os.path.exists(file_path): 166 # Assume file on the server didn't change if it's been saved before. 167 continue 168 169 if filename == UnicodeFiles.README: 170 with open(file_path, "wb") as fd: 171 fd.write(readme_content) 172 else: 173 url = get_fetch_url(filename) 174 print("Fetching: {}".format(url)) 175 subprocess.check_call(("curl", "-o", file_path, url)) 176 177 return unicode_version 178 179 180def check_stored_version(version): 181 # type: (Optional[str]) -> Optional[UnicodeVersion] 182 """ 183 Given desired Unicode version, return the version 184 if stored files are all present, and `None` otherwise. 185 """ 186 if not version: 187 # If no desired version specified, we should check what's the latest 188 # version, skipping stored version checks. 189 return None 190 191 fetch_dir = os.path.join(FETCH_DIR, version) 192 193 for filename in ALL_UNICODE_FILES: 194 file_path = os.path.join(fetch_dir, filename) 195 196 if not os.path.exists(file_path): 197 return None 198 199 with open(os.path.join(fetch_dir, UnicodeFiles.README)) as fd: 200 return parse_readme_unicode_version(fd.read()) 201 202 203def parse_readme_unicode_version(readme_content): 204 # type: (str) -> UnicodeVersion 205 """ 206 Parse the Unicode version contained in their `ReadMe.txt` file. 207 """ 208 # "Raw string" is necessary for \d not being treated as escape char 209 # (for the sake of compat with future Python versions). 210 # See: https://docs.python.org/3.6/whatsnew/3.6.html#deprecated-python-behavior 211 pattern = r"for Version (\d+)\.(\d+)\.(\d+) of the Unicode" 212 groups = re.search(pattern, readme_content).groups() 213 214 return UnicodeVersion(*map(int, groups), as_str=".".join(groups)) 215 216 217def get_unicode_dir(unicode_version): 218 # type: (UnicodeVersion) -> str 219 """ 220 Indicate in which parent dir the Unicode data files should be stored. 221 222 This returns a full, absolute path. 223 """ 224 return os.path.join(FETCH_DIR, unicode_version.as_str) 225 226 227def get_unicode_file_path(unicode_version, filename): 228 # type: (UnicodeVersion, str) -> str 229 """ 230 Indicate where the Unicode data file should be stored. 231 """ 232 return os.path.join(get_unicode_dir(unicode_version), filename) 233 234 235def is_surrogate(n): 236 # type: (int) -> bool 237 """ 238 Tell if given codepoint is a surrogate (not a valid Rust character). 239 """ 240 return SURROGATE_CODEPOINTS_RANGE[0] <= n <= SURROGATE_CODEPOINTS_RANGE[1] 241 242 243def load_unicode_data(file_path): 244 # type: (str) -> UnicodeData 245 """ 246 Load main Unicode data. 247 """ 248 # Conversions 249 to_lower = {} # type: Dict[int, Tuple[int, int, int]] 250 to_upper = {} # type: Dict[int, Tuple[int, int, int]] 251 to_title = {} # type: Dict[int, Tuple[int, int, int]] 252 253 # Decompositions 254 compat_decomp = {} # type: Dict[int, List[int]] 255 canon_decomp = {} # type: Dict[int, List[int]] 256 257 # Combining characters 258 # FIXME: combines are not used 259 combines = defaultdict(set) # type: Dict[str, Set[int]] 260 261 # Categories 262 general_categories = defaultdict(set) # type: Dict[str, Set[int]] 263 category_assigned_codepoints = set() # type: Set[int] 264 265 all_codepoints = {} 266 267 range_start = -1 268 269 for line in fileinput.input(file_path): 270 data = line.split(";") 271 if len(data) != 15: 272 continue 273 codepoint = int(data[0], 16) 274 if is_surrogate(codepoint): 275 continue 276 if range_start >= 0: 277 for i in range(range_start, codepoint): 278 all_codepoints[i] = data 279 range_start = -1 280 if data[1].endswith(", First>"): 281 range_start = codepoint 282 continue 283 all_codepoints[codepoint] = data 284 285 for code, data in all_codepoints.items(): 286 (code_org, name, gencat, combine, bidi, 287 decomp, deci, digit, num, mirror, 288 old, iso, upcase, lowcase, titlecase) = data 289 290 # Generate char to char direct common and simple conversions: 291 292 # Uppercase to lowercase 293 if lowcase != "" and code_org != lowcase: 294 to_lower[code] = (int(lowcase, 16), 0, 0) 295 296 # Lowercase to uppercase 297 if upcase != "" and code_org != upcase: 298 to_upper[code] = (int(upcase, 16), 0, 0) 299 300 # Title case 301 if titlecase.strip() != "" and code_org != titlecase: 302 to_title[code] = (int(titlecase, 16), 0, 0) 303 304 # Store decomposition, if given 305 if decomp: 306 decompositions = decomp.split()[1:] 307 decomp_code_points = [int(i, 16) for i in decompositions] 308 309 if decomp.startswith("<"): 310 # Compatibility decomposition 311 compat_decomp[code] = decomp_code_points 312 else: 313 # Canonical decomposition 314 canon_decomp[code] = decomp_code_points 315 316 # Place letter in categories as appropriate. 317 for cat in itertools.chain((gencat, ), EXPANDED_CATEGORIES.get(gencat, [])): 318 general_categories[cat].add(code) 319 category_assigned_codepoints.add(code) 320 321 # Record combining class, if any. 322 if combine != "0": 323 combines[combine].add(code) 324 325 # Generate Not_Assigned from Assigned. 326 general_categories["Cn"] = get_unassigned_codepoints(category_assigned_codepoints) 327 328 # Other contains Not_Assigned 329 general_categories["C"].update(general_categories["Cn"]) 330 331 grouped_categories = group_categories(general_categories) 332 333 # FIXME: combines are not used 334 return UnicodeData( 335 to_lower=to_lower, to_upper=to_upper, to_title=to_title, 336 compat_decomp=compat_decomp, canon_decomp=canon_decomp, 337 general_categories=grouped_categories, combines=combines, 338 ) 339 340 341def load_special_casing(file_path, unicode_data): 342 # type: (str, UnicodeData) -> None 343 """ 344 Load special casing data and enrich given Unicode data. 345 """ 346 for line in fileinput.input(file_path): 347 data = line.split("#")[0].split(";") 348 if len(data) == 5: 349 code, lower, title, upper, _comment = data 350 elif len(data) == 6: 351 code, lower, title, upper, condition, _comment = data 352 if condition.strip(): # Only keep unconditional mappins 353 continue 354 else: 355 continue 356 code = code.strip() 357 lower = lower.strip() 358 title = title.strip() 359 upper = upper.strip() 360 key = int(code, 16) 361 for (map_, values) in ((unicode_data.to_lower, lower), 362 (unicode_data.to_upper, upper), 363 (unicode_data.to_title, title)): 364 if values != code: 365 split = values.split() 366 367 codepoints = list(itertools.chain( 368 (int(i, 16) for i in split), 369 (0 for _ in range(len(split), 3)) 370 )) 371 372 assert len(codepoints) == 3 373 map_[key] = codepoints 374 375 376def group_categories(mapping): 377 # type: (Dict[Any, Iterable[int]]) -> Dict[str, List[Tuple[int, int]]] 378 """ 379 Group codepoints mapped in "categories". 380 """ 381 return {category: group_codepoints(codepoints) 382 for category, codepoints in mapping.items()} 383 384 385def group_codepoints(codepoints): 386 # type: (Iterable[int]) -> List[Tuple[int, int]] 387 """ 388 Group integral values into continuous, disjoint value ranges. 389 390 Performs value deduplication. 391 392 :return: sorted list of pairs denoting start and end of codepoint 393 group values, both ends inclusive. 394 395 >>> group_codepoints([1, 2, 10, 11, 12, 3, 4]) 396 [(1, 4), (10, 12)] 397 >>> group_codepoints([1]) 398 [(1, 1)] 399 >>> group_codepoints([1, 5, 6]) 400 [(1, 1), (5, 6)] 401 >>> group_codepoints([]) 402 [] 403 """ 404 sorted_codes = sorted(set(codepoints)) 405 result = [] # type: List[Tuple[int, int]] 406 407 if not sorted_codes: 408 return result 409 410 next_codes = sorted_codes[1:] 411 start_code = sorted_codes[0] 412 413 for code, next_code in zip_longest(sorted_codes, next_codes, fillvalue=None): 414 if next_code is None or next_code - code != 1: 415 result.append((start_code, code)) 416 start_code = next_code 417 418 return result 419 420 421def ungroup_codepoints(codepoint_pairs): 422 # type: (Iterable[Tuple[int, int]]) -> List[int] 423 """ 424 The inverse of group_codepoints -- produce a flat list of values 425 from value range pairs. 426 427 >>> ungroup_codepoints([(1, 4), (10, 12)]) 428 [1, 2, 3, 4, 10, 11, 12] 429 >>> ungroup_codepoints([(1, 1), (5, 6)]) 430 [1, 5, 6] 431 >>> ungroup_codepoints(group_codepoints([1, 2, 7, 8])) 432 [1, 2, 7, 8] 433 >>> ungroup_codepoints([]) 434 [] 435 """ 436 return list(itertools.chain.from_iterable( 437 range(lo, hi + 1) for lo, hi in codepoint_pairs 438 )) 439 440 441def get_unassigned_codepoints(assigned_codepoints): 442 # type: (Set[int]) -> Set[int] 443 """ 444 Given a set of "assigned" codepoints, return a set 445 of these that are not in assigned and not surrogate. 446 """ 447 return {i for i in range(0, 0x110000) 448 if i not in assigned_codepoints and not is_surrogate(i)} 449 450 451def generate_table_lines(items, indent, wrap=98): 452 # type: (Iterable[str], int, int) -> Iterator[str] 453 """ 454 Given table items, generate wrapped lines of text with comma-separated items. 455 456 This is a generator function. 457 458 :param wrap: soft wrap limit (characters per line), integer. 459 """ 460 line = " " * indent 461 first = True 462 for item in items: 463 if len(line) + len(item) < wrap: 464 if first: 465 line += item 466 else: 467 line += ", " + item 468 first = False 469 else: 470 yield line + ",\n" 471 line = " " * indent + item 472 473 yield line 474 475 476def load_properties(file_path, interesting_props): 477 # type: (str, Iterable[str]) -> Dict[str, List[Tuple[int, int]]] 478 """ 479 Load properties data and return in grouped form. 480 """ 481 props = defaultdict(list) # type: Dict[str, List[Tuple[int, int]]] 482 # "Raw string" is necessary for `\.` and `\w` not to be treated as escape chars 483 # (for the sake of compat with future Python versions). 484 # See: https://docs.python.org/3.6/whatsnew/3.6.html#deprecated-python-behavior 485 re1 = re.compile(r"^ *([0-9A-F]+) *; *(\w+)") 486 re2 = re.compile(r"^ *([0-9A-F]+)\.\.([0-9A-F]+) *; *(\w+)") 487 488 for line in fileinput.input(file_path): 489 match = re1.match(line) or re2.match(line) 490 if match: 491 groups = match.groups() 492 493 if len(groups) == 2: 494 # `re1` matched (2 groups). 495 d_lo, prop = groups 496 d_hi = d_lo 497 else: 498 d_lo, d_hi, prop = groups 499 else: 500 continue 501 502 if interesting_props and prop not in interesting_props: 503 continue 504 505 lo_value = int(d_lo, 16) 506 hi_value = int(d_hi, 16) 507 508 props[prop].append((lo_value, hi_value)) 509 510 # Optimize if possible. 511 for prop in props: 512 props[prop] = group_codepoints(ungroup_codepoints(props[prop])) 513 514 return props 515 516 517def escape_char(c): 518 # type: (int) -> str 519 r""" 520 Escape a codepoint for use as Rust char literal. 521 522 Outputs are OK to use as Rust source code as char literals 523 and they also include necessary quotes. 524 525 >>> escape_char(97) 526 "'\\u{61}'" 527 >>> escape_char(0) 528 "'\\0'" 529 """ 530 return r"'\u{%x}'" % c if c != 0 else r"'\0'" 531 532 533def format_char_pair(pair): 534 # type: (Tuple[int, int]) -> str 535 """ 536 Format a pair of two Rust chars. 537 """ 538 return "(%s,%s)" % (escape_char(pair[0]), escape_char(pair[1])) 539 540 541def generate_table( 542 name, # type: str 543 items, # type: List[Tuple[int, int]] 544 decl_type="&[(char, char)]", # type: str 545 is_pub=True, # type: bool 546 format_item=format_char_pair, # type: Callable[[Tuple[int, int]], str] 547): 548 # type: (...) -> Iterator[str] 549 """ 550 Generate a nicely formatted Rust constant "table" array. 551 552 This generates actual Rust code. 553 """ 554 pub_string = "" 555 if is_pub: 556 pub_string = "pub " 557 558 yield " %sconst %s: %s = &[\n" % (pub_string, name, decl_type) 559 560 data = [] 561 first = True 562 for item in items: 563 if not first: 564 data.append(",") 565 first = False 566 data.extend(format_item(item)) 567 568 for table_line in generate_table_lines("".join(data).split(","), 8): 569 yield table_line 570 571 yield "\n ];\n\n" 572 573 574def compute_trie(raw_data, chunk_size): 575 # type: (List[int], int) -> Tuple[List[int], List[int]] 576 """ 577 Compute postfix-compressed trie. 578 579 See: bool_trie.rs for more details. 580 581 >>> compute_trie([1, 2, 3, 1, 2, 3, 4, 5, 6], 3) 582 ([0, 0, 1], [1, 2, 3, 4, 5, 6]) 583 >>> compute_trie([1, 2, 3, 1, 2, 4, 4, 5, 6], 3) 584 ([0, 1, 2], [1, 2, 3, 1, 2, 4, 4, 5, 6]) 585 """ 586 root = [] 587 childmap = {} # type: Dict[Tuple[int, ...], int] 588 child_data = [] 589 590 assert len(raw_data) % chunk_size == 0, "Chunks must be equally sized" 591 592 for i in range(len(raw_data) // chunk_size): 593 data = raw_data[i * chunk_size : (i + 1) * chunk_size] 594 595 # Postfix compression of child nodes (data chunks) 596 # (identical child nodes are shared). 597 598 # Make a tuple out of the list so it's hashable. 599 child = tuple(data) 600 if child not in childmap: 601 childmap[child] = len(childmap) 602 child_data.extend(data) 603 604 root.append(childmap[child]) 605 606 return root, child_data 607 608 609def generate_bool_trie(name, codepoint_ranges, is_pub=False): 610 # type: (str, List[Tuple[int, int]], bool) -> Iterator[str] 611 """ 612 Generate Rust code for BoolTrie struct. 613 614 This yields string fragments that should be joined to produce 615 the final string. 616 617 See: `bool_trie.rs`. 618 """ 619 chunk_size = 64 620 rawdata = [False] * 0x110000 621 for (lo, hi) in codepoint_ranges: 622 for cp in range(lo, hi + 1): 623 rawdata[cp] = True 624 625 # Convert to bitmap chunks of `chunk_size` bits each. 626 chunks = [] 627 for i in range(0x110000 // chunk_size): 628 chunk = 0 629 for j in range(chunk_size): 630 if rawdata[i * chunk_size + j]: 631 chunk |= 1 << j 632 chunks.append(chunk) 633 634 pub_string = "" 635 if is_pub: 636 pub_string = "pub " 637 yield " %sconst %s: &super::BoolTrie = &super::BoolTrie {\n" % (pub_string, name) 638 yield " r1: [\n" 639 data = ("0x%016x" % chunk for chunk in chunks[:0x800 // chunk_size]) 640 for fragment in generate_table_lines(data, 12): 641 yield fragment 642 yield "\n ],\n" 643 644 # 0x800..0x10000 trie 645 (r2, r3) = compute_trie(chunks[0x800 // chunk_size : 0x10000 // chunk_size], 64 // chunk_size) 646 yield " r2: [\n" 647 data = map(str, r2) 648 for fragment in generate_table_lines(data, 12): 649 yield fragment 650 yield "\n ],\n" 651 652 yield " r3: &[\n" 653 data = ("0x%016x" % node for node in r3) 654 for fragment in generate_table_lines(data, 12): 655 yield fragment 656 yield "\n ],\n" 657 658 # 0x10000..0x110000 trie 659 (mid, r6) = compute_trie(chunks[0x10000 // chunk_size : 0x110000 // chunk_size], 660 64 // chunk_size) 661 (r4, r5) = compute_trie(mid, 64) 662 663 yield " r4: [\n" 664 data = map(str, r4) 665 for fragment in generate_table_lines(data, 12): 666 yield fragment 667 yield "\n ],\n" 668 669 yield " r5: &[\n" 670 data = map(str, r5) 671 for fragment in generate_table_lines(data, 12): 672 yield fragment 673 yield "\n ],\n" 674 675 yield " r6: &[\n" 676 data = ("0x%016x" % node for node in r6) 677 for fragment in generate_table_lines(data, 12): 678 yield fragment 679 yield "\n ],\n" 680 681 yield " };\n\n" 682 683 684def generate_small_bool_trie(name, codepoint_ranges, is_pub=False): 685 # type: (str, List[Tuple[int, int]], bool) -> Iterator[str] 686 """ 687 Generate Rust code for `SmallBoolTrie` struct. 688 689 See: `bool_trie.rs`. 690 """ 691 last_chunk = max(hi // 64 for (lo, hi) in codepoint_ranges) 692 n_chunks = last_chunk + 1 693 chunks = [0] * n_chunks 694 for (lo, hi) in codepoint_ranges: 695 for cp in range(lo, hi + 1): 696 assert cp // 64 < len(chunks) 697 chunks[cp // 64] |= 1 << (cp & 63) 698 699 pub_string = "" 700 if is_pub: 701 pub_string = "pub " 702 703 yield (" %sconst %s: &super::SmallBoolTrie = &super::SmallBoolTrie {\n" 704 % (pub_string, name)) 705 706 (r1, r2) = compute_trie(chunks, 1) 707 708 yield " r1: &[\n" 709 data = (str(node) for node in r1) 710 for fragment in generate_table_lines(data, 12): 711 yield fragment 712 yield "\n ],\n" 713 714 yield " r2: &[\n" 715 data = ("0x%016x" % node for node in r2) 716 for fragment in generate_table_lines(data, 12): 717 yield fragment 718 yield "\n ],\n" 719 720 yield " };\n\n" 721 722 723def generate_property_module(mod, grouped_categories, category_subset): 724 # type: (str, Dict[str, List[Tuple[int, int]]], Iterable[str]) -> Iterator[str] 725 """ 726 Generate Rust code for module defining properties. 727 """ 728 729 yield "pub(crate) mod %s {\n" % mod 730 for cat in sorted(category_subset): 731 if cat in ("Cc", "White_Space"): 732 generator = generate_small_bool_trie("%s_table" % cat, grouped_categories[cat]) 733 else: 734 generator = generate_bool_trie("%s_table" % cat, grouped_categories[cat]) 735 736 for fragment in generator: 737 yield fragment 738 739 yield " pub fn %s(c: char) -> bool {\n" % cat 740 yield " %s_table.lookup(c)\n" % cat 741 yield " }\n\n" 742 743 yield "}\n\n" 744 745 746def generate_conversions_module(unicode_data): 747 # type: (UnicodeData) -> Iterator[str] 748 """ 749 Generate Rust code for module defining conversions. 750 """ 751 752 yield "pub(crate) mod conversions {" 753 yield """ 754 pub fn to_lower(c: char) -> [char; 3] { 755 match bsearch_case_table(c, to_lowercase_table) { 756 None => [c, '\\0', '\\0'], 757 Some(index) => to_lowercase_table[index].1, 758 } 759 } 760 761 pub fn to_upper(c: char) -> [char; 3] { 762 match bsearch_case_table(c, to_uppercase_table) { 763 None => [c, '\\0', '\\0'], 764 Some(index) => to_uppercase_table[index].1, 765 } 766 } 767 768 fn bsearch_case_table(c: char, table: &[(char, [char; 3])]) -> Option<usize> { 769 table.binary_search_by(|&(key, _)| key.cmp(&c)).ok() 770 }\n\n""" 771 772 decl_type = "&[(char, [char; 3])]" 773 format_conversion = lambda x: "({},[{},{},{}])".format(*( 774 escape_char(c) for c in (x[0], x[1][0], x[1][1], x[1][2]) 775 )) 776 777 for fragment in generate_table( 778 name="to_lowercase_table", 779 items=sorted(unicode_data.to_lower.items(), key=lambda x: x[0]), 780 decl_type=decl_type, 781 is_pub=False, 782 format_item=format_conversion 783 ): 784 yield fragment 785 786 for fragment in generate_table( 787 name="to_uppercase_table", 788 items=sorted(unicode_data.to_upper.items(), key=lambda x: x[0]), 789 decl_type=decl_type, 790 is_pub=False, 791 format_item=format_conversion 792 ): 793 yield fragment 794 795 yield "}\n" 796 797 798def parse_args(): 799 # type: () -> argparse.Namespace 800 """ 801 Parse command line arguments. 802 """ 803 parser = argparse.ArgumentParser(description=__doc__) 804 parser.add_argument("-v", "--version", default=None, type=str, 805 help="Unicode version to use (if not specified," 806 " defaults to latest release).") 807 808 return parser.parse_args() 809 810 811def main(): 812 # type: () -> None 813 """ 814 Script entry point. 815 """ 816 args = parse_args() 817 818 unicode_version = fetch_files(args.version) 819 print("Using Unicode version: {}".format(unicode_version.as_str)) 820 821 # All the writing happens entirely in memory, we only write to file 822 # once we have generated the file content (it's not very large, <1 MB). 823 buf = StringIO() 824 buf.write(PREAMBLE) 825 826 unicode_version_notice = textwrap.dedent(""" 827 /// The version of [Unicode](http://www.unicode.org/) that the Unicode parts of 828 /// `char` and `str` methods are based on. 829 #[unstable(feature = "unicode_version", issue = "49726")] 830 pub const UNICODE_VERSION: UnicodeVersion = UnicodeVersion {{ 831 major: {version.major}, 832 minor: {version.minor}, 833 micro: {version.micro}, 834 _priv: (), 835 }}; 836 """).format(version=unicode_version) 837 buf.write(unicode_version_notice) 838 839 get_path = lambda f: get_unicode_file_path(unicode_version, f) 840 841 unicode_data = load_unicode_data(get_path(UnicodeFiles.UNICODE_DATA)) 842 load_special_casing(get_path(UnicodeFiles.SPECIAL_CASING), unicode_data) 843 844 want_derived = {"Alphabetic", "Lowercase", "Uppercase", 845 "Cased", "Case_Ignorable", "Grapheme_Extend"} 846 derived = load_properties(get_path(UnicodeFiles.DERIVED_CORE_PROPERTIES), want_derived) 847 848 props = load_properties(get_path(UnicodeFiles.PROPS), 849 {"White_Space", "Join_Control", "Noncharacter_Code_Point"}) 850 851 # Category tables 852 for (name, categories, category_subset) in ( 853 ("general_category", unicode_data.general_categories, ["N", "Cc"]), 854 ("derived_property", derived, want_derived), 855 ("property", props, ["White_Space"]) 856 ): 857 for fragment in generate_property_module(name, categories, category_subset): 858 buf.write(fragment) 859 860 for fragment in generate_conversions_module(unicode_data): 861 buf.write(fragment) 862 863 tables_rs_path = os.path.join(THIS_DIR, "tables.rs") 864 865 # Actually write out the file content. 866 # Will overwrite the file if it exists. 867 with open(tables_rs_path, "w") as fd: 868 fd.write(buf.getvalue()) 869 870 print("Regenerated tables.rs.") 871 872 873if __name__ == "__main__": 874 main()