fork/rust
git clone https://git.julia.blue/fork/rust
5e1a799842b
1.41.0
1#!/usr/bin/env python 2 3""" 4Regenerate Unicode tables (tables.rs). 5""" 6 7# This script uses the Unicode tables as defined 8# in the UnicodeFiles class. 9 10# Since this should not require frequent updates, we just store this 11# out-of-line and check the tables.rs file into git. 12 13# Note that the "curl" program is required for operation. 14# This script is compatible with Python 2.7 and 3.x. 15 16import argparse 17import datetime 18import fileinput 19import itertools 20import os 21import re 22import textwrap 23import subprocess 24 25from collections import defaultdict, namedtuple 26 27try: 28 # Python 3 29 from itertools import zip_longest 30 from io import StringIO 31except ImportError: 32 # Python 2 compatibility 33 zip_longest = itertools.izip_longest 34 from StringIO import StringIO 35 36try: 37 # Completely optional type hinting 38 # (Python 2 compatible using comments, 39 # see: https://mypy.readthedocs.io/en/latest/python2.html) 40 # This is very helpful in typing-aware IDE like PyCharm. 41 from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Set, Tuple 42except ImportError: 43 pass 44 45 46# We don't use enum.Enum because of Python 2.7 compatibility. 47class UnicodeFiles(object): 48 # ReadMe does not contain any Unicode data, we 49 # only use it to extract versions. 50 README = "ReadMe.txt" 51 52 DERIVED_CORE_PROPERTIES = "DerivedCoreProperties.txt" 53 DERIVED_NORMALIZATION_PROPS = "DerivedNormalizationProps.txt" 54 PROPS = "PropList.txt" 55 SCRIPTS = "Scripts.txt" 56 SPECIAL_CASING = "SpecialCasing.txt" 57 UNICODE_DATA = "UnicodeData.txt" 58 59 60# The order doesn't really matter (Python < 3.6 won't preserve it), 61# we only want to aggregate all the file names. 62ALL_UNICODE_FILES = tuple( 63 value for name, value in UnicodeFiles.__dict__.items() 64 if not name.startswith("_") 65) 66 67assert len(ALL_UNICODE_FILES) == 7, "Unexpected number of unicode files" 68 69# The directory this file is located in. 70THIS_DIR = os.path.dirname(os.path.realpath(__file__)) 71 72# Where to download the Unicode data. The downloaded files 73# will be placed in sub-directories named after Unicode version. 74FETCH_DIR = os.path.join(THIS_DIR, "downloaded") 75 76FETCH_URL_LATEST = "ftp://ftp.unicode.org/Public/UNIDATA/{filename}" 77FETCH_URL_VERSION = "ftp://ftp.unicode.org/Public/{version}/ucd/{filename}" 78 79PREAMBLE = """\ 80// NOTE: The following code was generated by "./unicode.py", do not edit directly 81 82#![allow(missing_docs, non_upper_case_globals, non_snake_case, clippy::unreadable_literal)] 83 84use crate::unicode::bool_trie::{{BoolTrie, SmallBoolTrie}}; 85use crate::unicode::version::UnicodeVersion; 86""".format(year=datetime.datetime.now().year) 87 88# Mapping taken from Table 12 from: 89# http://www.unicode.org/reports/tr44/#General_Category_Values 90EXPANDED_CATEGORIES = { 91 "Lu": ["LC", "L"], "Ll": ["LC", "L"], "Lt": ["LC", "L"], 92 "Lm": ["L"], "Lo": ["L"], 93 "Mn": ["M"], "Mc": ["M"], "Me": ["M"], 94 "Nd": ["N"], "Nl": ["N"], "No": ["N"], 95 "Pc": ["P"], "Pd": ["P"], "Ps": ["P"], "Pe": ["P"], 96 "Pi": ["P"], "Pf": ["P"], "Po": ["P"], 97 "Sm": ["S"], "Sc": ["S"], "Sk": ["S"], "So": ["S"], 98 "Zs": ["Z"], "Zl": ["Z"], "Zp": ["Z"], 99 "Cc": ["C"], "Cf": ["C"], "Cs": ["C"], "Co": ["C"], "Cn": ["C"], 100} 101 102# This is the (inclusive) range of surrogate codepoints. 103# These are not valid Rust characters. 104SURROGATE_CODEPOINTS_RANGE = (0xd800, 0xdfff) 105 106UnicodeData = namedtuple( 107 "UnicodeData", ( 108 # Conversions: 109 "to_upper", "to_lower", "to_title", 110 111 # Decompositions: canonical decompositions, compatibility decomp 112 "canon_decomp", "compat_decomp", 113 114 # Grouped: general categories and combining characters 115 "general_categories", "combines", 116 ) 117) 118 119UnicodeVersion = namedtuple( 120 "UnicodeVersion", ("major", "minor", "micro", "as_str") 121) 122 123 124def fetch_files(version=None): 125 # type: (str) -> UnicodeVersion 126 """ 127 Fetch all the Unicode files from unicode.org. 128 129 This will use cached files (stored in `FETCH_DIR`) if they exist, 130 creating them if they don't. In any case, the Unicode version 131 is always returned. 132 133 :param version: The desired Unicode version, as string. 134 (If None, defaults to latest final release available, 135 querying the unicode.org service). 136 """ 137 have_version = check_stored_version(version) 138 if have_version: 139 return have_version 140 141 if version: 142 # Check if the desired version exists on the server. 143 get_fetch_url = lambda name: FETCH_URL_VERSION.format(version=version, filename=name) 144 else: 145 # Extract the latest version. 146 get_fetch_url = lambda name: FETCH_URL_LATEST.format(filename=name) 147 148 readme_url = get_fetch_url(UnicodeFiles.README) 149 150 print("Fetching: {}".format(readme_url)) 151 readme_content = subprocess.check_output(("curl", readme_url)) 152 153 unicode_version = parse_readme_unicode_version( 154 readme_content.decode("utf8") 155 ) 156 157 download_dir = get_unicode_dir(unicode_version) 158 if not os.path.exists(download_dir): 159 # For 2.7 compat, we don't use `exist_ok=True`. 160 os.makedirs(download_dir) 161 162 for filename in ALL_UNICODE_FILES: 163 file_path = get_unicode_file_path(unicode_version, filename) 164 165 if os.path.exists(file_path): 166 # Assume file on the server didn't change if it's been saved before. 167 continue 168 169 if filename == UnicodeFiles.README: 170 with open(file_path, "wb") as fd: 171 fd.write(readme_content) 172 else: 173 url = get_fetch_url(filename) 174 print("Fetching: {}".format(url)) 175 subprocess.check_call(("curl", "-o", file_path, url)) 176 177 return unicode_version 178 179 180def check_stored_version(version): 181 # type: (Optional[str]) -> Optional[UnicodeVersion] 182 """ 183 Given desired Unicode version, return the version 184 if stored files are all present, and `None` otherwise. 185 """ 186 if not version: 187 # If no desired version specified, we should check what's the latest 188 # version, skipping stored version checks. 189 return None 190 191 fetch_dir = os.path.join(FETCH_DIR, version) 192 193 for filename in ALL_UNICODE_FILES: 194 file_path = os.path.join(fetch_dir, filename) 195 196 if not os.path.exists(file_path): 197 return None 198 199 with open(os.path.join(fetch_dir, UnicodeFiles.README)) as fd: 200 return parse_readme_unicode_version(fd.read()) 201 202 203def parse_readme_unicode_version(readme_content): 204 # type: (str) -> UnicodeVersion 205 """ 206 Parse the Unicode version contained in their `ReadMe.txt` file. 207 """ 208 # "Raw string" is necessary for \d not being treated as escape char 209 # (for the sake of compat with future Python versions). 210 # See: https://docs.python.org/3.6/whatsnew/3.6.html#deprecated-python-behavior 211 pattern = r"for Version (\d+)\.(\d+)\.(\d+) of the Unicode" 212 groups = re.search(pattern, readme_content).groups() 213 214 return UnicodeVersion(*map(int, groups), as_str=".".join(groups)) 215 216 217def get_unicode_dir(unicode_version): 218 # type: (UnicodeVersion) -> str 219 """ 220 Indicate in which parent dir the Unicode data files should be stored. 221 222 This returns a full, absolute path. 223 """ 224 return os.path.join(FETCH_DIR, unicode_version.as_str) 225 226 227def get_unicode_file_path(unicode_version, filename): 228 # type: (UnicodeVersion, str) -> str 229 """ 230 Indicate where the Unicode data file should be stored. 231 """ 232 return os.path.join(get_unicode_dir(unicode_version), filename) 233 234 235def is_surrogate(n): 236 # type: (int) -> bool 237 """ 238 Tell if given codepoint is a surrogate (not a valid Rust character). 239 """ 240 return SURROGATE_CODEPOINTS_RANGE[0] <= n <= SURROGATE_CODEPOINTS_RANGE[1] 241 242 243def load_unicode_data(file_path): 244 # type: (str) -> UnicodeData 245 """ 246 Load main Unicode data. 247 """ 248 # Conversions 249 to_lower = {} # type: Dict[int, Tuple[int, int, int]] 250 to_upper = {} # type: Dict[int, Tuple[int, int, int]] 251 to_title = {} # type: Dict[int, Tuple[int, int, int]] 252 253 # Decompositions 254 compat_decomp = {} # type: Dict[int, List[int]] 255 canon_decomp = {} # type: Dict[int, List[int]] 256 257 # Combining characters 258 # FIXME: combines are not used 259 combines = defaultdict(set) # type: Dict[str, Set[int]] 260 261 # Categories 262 general_categories = defaultdict(set) # type: Dict[str, Set[int]] 263 category_assigned_codepoints = set() # type: Set[int] 264 265 all_codepoints = {} 266 267 range_start = -1 268 269 for line in fileinput.input(file_path): 270 data = line.split(";") 271 if len(data) != 15: 272 continue 273 codepoint = int(data[0], 16) 274 if is_surrogate(codepoint): 275 continue 276 if range_start >= 0: 277 for i in range(range_start, codepoint): 278 all_codepoints[i] = data 279 range_start = -1 280 if data[1].endswith(", First>"): 281 range_start = codepoint 282 continue 283 all_codepoints[codepoint] = data 284 285 for code, data in all_codepoints.items(): 286 (code_org, name, gencat, combine, bidi, 287 decomp, deci, digit, num, mirror, 288 old, iso, upcase, lowcase, titlecase) = data 289 290 # Generate char to char direct common and simple conversions: 291 292 # Uppercase to lowercase 293 if lowcase != "" and code_org != lowcase: 294 to_lower[code] = (int(lowcase, 16), 0, 0) 295 296 # Lowercase to uppercase 297 if upcase != "" and code_org != upcase: 298 to_upper[code] = (int(upcase, 16), 0, 0) 299 300 # Title case 301 if titlecase.strip() != "" and code_org != titlecase: 302 to_title[code] = (int(titlecase, 16), 0, 0) 303 304 # Store decomposition, if given 305 if decomp: 306 decompositions = decomp.split()[1:] 307 decomp_code_points = [int(i, 16) for i in decompositions] 308 309 if decomp.startswith("<"): 310 # Compatibility decomposition 311 compat_decomp[code] = decomp_code_points 312 else: 313 # Canonical decomposition 314 canon_decomp[code] = decomp_code_points 315 316 # Place letter in categories as appropriate. 317 for cat in itertools.chain((gencat, ), EXPANDED_CATEGORIES.get(gencat, [])): 318 general_categories[cat].add(code) 319 category_assigned_codepoints.add(code) 320 321 # Record combining class, if any. 322 if combine != "0": 323 combines[combine].add(code) 324 325 # Generate Not_Assigned from Assigned. 326 general_categories["Cn"] = get_unassigned_codepoints(category_assigned_codepoints) 327 328 # Other contains Not_Assigned 329 general_categories["C"].update(general_categories["Cn"]) 330 331 grouped_categories = group_categories(general_categories) 332 333 # FIXME: combines are not used 334 return UnicodeData( 335 to_lower=to_lower, to_upper=to_upper, to_title=to_title, 336 compat_decomp=compat_decomp, canon_decomp=canon_decomp, 337 general_categories=grouped_categories, combines=combines, 338 ) 339 340 341def load_special_casing(file_path, unicode_data): 342 # type: (str, UnicodeData) -> None 343 """ 344 Load special casing data and enrich given Unicode data. 345 """ 346 for line in fileinput.input(file_path): 347 data = line.split("#")[0].split(";") 348 if len(data) == 5: 349 code, lower, title, upper, _comment = data 350 elif len(data) == 6: 351 code, lower, title, upper, condition, _comment = data 352 if condition.strip(): # Only keep unconditional mappins 353 continue 354 else: 355 continue 356 code = code.strip() 357 lower = lower.strip() 358 title = title.strip() 359 upper = upper.strip() 360 key = int(code, 16) 361 for (map_, values) in ((unicode_data.to_lower, lower), 362 (unicode_data.to_upper, upper), 363 (unicode_data.to_title, title)): 364 if values != code: 365 split = values.split() 366 367 codepoints = list(itertools.chain( 368 (int(i, 16) for i in split), 369 (0 for _ in range(len(split), 3)) 370 )) 371 372 assert len(codepoints) == 3 373 map_[key] = codepoints 374 375 376def group_categories(mapping): 377 # type: (Dict[Any, Iterable[int]]) -> Dict[str, List[Tuple[int, int]]] 378 """ 379 Group codepoints mapped in "categories". 380 """ 381 return {category: group_codepoints(codepoints) 382 for category, codepoints in mapping.items()} 383 384 385def group_codepoints(codepoints): 386 # type: (Iterable[int]) -> List[Tuple[int, int]] 387 """ 388 Group integral values into continuous, disjoint value ranges. 389 390 Performs value deduplication. 391 392 :return: sorted list of pairs denoting start and end of codepoint 393 group values, both ends inclusive. 394 395 >>> group_codepoints([1, 2, 10, 11, 12, 3, 4]) 396 [(1, 4), (10, 12)] 397 >>> group_codepoints([1]) 398 [(1, 1)] 399 >>> group_codepoints([1, 5, 6]) 400 [(1, 1), (5, 6)] 401 >>> group_codepoints([]) 402 [] 403 """ 404 sorted_codes = sorted(set(codepoints)) 405 result = [] # type: List[Tuple[int, int]] 406 407 if not sorted_codes: 408 return result 409 410 next_codes = sorted_codes[1:] 411 start_code = sorted_codes[0] 412 413 for code, next_code in zip_longest(sorted_codes, next_codes, fillvalue=None): 414 if next_code is None or next_code - code != 1: 415 result.append((start_code, code)) 416 start_code = next_code 417 418 return result 419 420 421def ungroup_codepoints(codepoint_pairs): 422 # type: (Iterable[Tuple[int, int]]) -> List[int] 423 """ 424 The inverse of group_codepoints -- produce a flat list of values 425 from value range pairs. 426 427 >>> ungroup_codepoints([(1, 4), (10, 12)]) 428 [1, 2, 3, 4, 10, 11, 12] 429 >>> ungroup_codepoints([(1, 1), (5, 6)]) 430 [1, 5, 6] 431 >>> ungroup_codepoints(group_codepoints([1, 2, 7, 8])) 432 [1, 2, 7, 8] 433 >>> ungroup_codepoints([]) 434 [] 435 """ 436 return list(itertools.chain.from_iterable( 437 range(lo, hi + 1) for lo, hi in codepoint_pairs 438 )) 439 440 441def get_unassigned_codepoints(assigned_codepoints): 442 # type: (Set[int]) -> Set[int] 443 """ 444 Given a set of "assigned" codepoints, return a set 445 of these that are not in assigned and not surrogate. 446 """ 447 return {i for i in range(0, 0x110000) 448 if i not in assigned_codepoints and not is_surrogate(i)} 449 450 451def generate_table_lines(items, indent, wrap=98): 452 # type: (Iterable[str], int, int) -> Iterator[str] 453 """ 454 Given table items, generate wrapped lines of text with comma-separated items. 455 456 This is a generator function. 457 458 :param wrap: soft wrap limit (characters per line), integer. 459 """ 460 line = " " * indent 461 first = True 462 for item in items: 463 if len(line) + len(item) < wrap: 464 if first: 465 line += item 466 else: 467 line += ", " + item 468 first = False 469 else: 470 yield line + ",\n" 471 line = " " * indent + item 472 473 yield line 474 475 476def load_properties(file_path, interesting_props): 477 # type: (str, Iterable[str]) -> Dict[str, List[Tuple[int, int]]] 478 """ 479 Load properties data and return in grouped form. 480 """ 481 props = defaultdict(list) # type: Dict[str, List[Tuple[int, int]]] 482 # "Raw string" is necessary for `\.` and `\w` not to be treated as escape chars 483 # (for the sake of compat with future Python versions). 484 # See: https://docs.python.org/3.6/whatsnew/3.6.html#deprecated-python-behavior 485 re1 = re.compile(r"^ *([0-9A-F]+) *; *(\w+)") 486 re2 = re.compile(r"^ *([0-9A-F]+)\.\.([0-9A-F]+) *; *(\w+)") 487 488 for line in fileinput.input(file_path): 489 match = re1.match(line) or re2.match(line) 490 if match: 491 groups = match.groups() 492 493 if len(groups) == 2: 494 # `re1` matched (2 groups). 495 d_lo, prop = groups 496 d_hi = d_lo 497 else: 498 d_lo, d_hi, prop = groups 499 else: 500 continue 501 502 if interesting_props and prop not in interesting_props: 503 continue 504 505 lo_value = int(d_lo, 16) 506 hi_value = int(d_hi, 16) 507 508 props[prop].append((lo_value, hi_value)) 509 510 # Optimize if possible. 511 for prop in props: 512 props[prop] = group_codepoints(ungroup_codepoints(props[prop])) 513 514 return props 515 516 517def escape_char(c): 518 # type: (int) -> str 519 r""" 520 Escape a codepoint for use as Rust char literal. 521 522 Outputs are OK to use as Rust source code as char literals 523 and they also include necessary quotes. 524 525 >>> escape_char(97) 526 "'\\u{61}'" 527 >>> escape_char(0) 528 "'\\0'" 529 """ 530 return r"'\u{%x}'" % c if c != 0 else r"'\0'" 531 532 533def format_char_pair(pair): 534 # type: (Tuple[int, int]) -> str 535 """ 536 Format a pair of two Rust chars. 537 """ 538 return "(%s,%s)" % (escape_char(pair[0]), escape_char(pair[1])) 539 540 541def generate_table( 542 name, # type: str 543 items, # type: List[Tuple[int, int]] 544 decl_type="&[(char, char)]", # type: str 545 is_pub=True, # type: bool 546 format_item=format_char_pair, # type: Callable[[Tuple[int, int]], str] 547): 548 # type: (...) -> Iterator[str] 549 """ 550 Generate a nicely formatted Rust constant "table" array. 551 552 This generates actual Rust code. 553 """ 554 pub_string = "" 555 if is_pub: 556 pub_string = "pub " 557 558 yield "\n" 559 yield " #[rustfmt::skip]\n" 560 yield " %sconst %s: %s = &[\n" % (pub_string, name, decl_type) 561 562 data = [] 563 first = True 564 for item in items: 565 if not first: 566 data.append(",") 567 first = False 568 data.extend(format_item(item)) 569 570 for table_line in generate_table_lines("".join(data).split(","), 8): 571 yield table_line 572 573 yield "\n ];\n" 574 575 576def compute_trie(raw_data, chunk_size): 577 # type: (List[int], int) -> Tuple[List[int], List[int]] 578 """ 579 Compute postfix-compressed trie. 580 581 See: bool_trie.rs for more details. 582 583 >>> compute_trie([1, 2, 3, 1, 2, 3, 4, 5, 6], 3) 584 ([0, 0, 1], [1, 2, 3, 4, 5, 6]) 585 >>> compute_trie([1, 2, 3, 1, 2, 4, 4, 5, 6], 3) 586 ([0, 1, 2], [1, 2, 3, 1, 2, 4, 4, 5, 6]) 587 """ 588 root = [] 589 childmap = {} # type: Dict[Tuple[int, ...], int] 590 child_data = [] 591 592 assert len(raw_data) % chunk_size == 0, "Chunks must be equally sized" 593 594 for i in range(len(raw_data) // chunk_size): 595 data = raw_data[i * chunk_size : (i + 1) * chunk_size] 596 597 # Postfix compression of child nodes (data chunks) 598 # (identical child nodes are shared). 599 600 # Make a tuple out of the list so it's hashable. 601 child = tuple(data) 602 if child not in childmap: 603 childmap[child] = len(childmap) 604 child_data.extend(data) 605 606 root.append(childmap[child]) 607 608 return root, child_data 609 610 611def generate_bool_trie(name, codepoint_ranges, is_pub=False): 612 # type: (str, List[Tuple[int, int]], bool) -> Iterator[str] 613 """ 614 Generate Rust code for BoolTrie struct. 615 616 This yields string fragments that should be joined to produce 617 the final string. 618 619 See: `bool_trie.rs`. 620 """ 621 chunk_size = 64 622 rawdata = [False] * 0x110000 623 for (lo, hi) in codepoint_ranges: 624 for cp in range(lo, hi + 1): 625 rawdata[cp] = True 626 627 # Convert to bitmap chunks of `chunk_size` bits each. 628 chunks = [] 629 for i in range(0x110000 // chunk_size): 630 chunk = 0 631 for j in range(chunk_size): 632 if rawdata[i * chunk_size + j]: 633 chunk |= 1 << j 634 chunks.append(chunk) 635 636 pub_string = "" 637 if is_pub: 638 pub_string = "pub " 639 640 yield "\n" 641 yield " #[rustfmt::skip]\n" 642 yield " %sconst %s: &super::BoolTrie = &super::BoolTrie {\n" % (pub_string, name) 643 yield " r1: [\n" 644 data = ("0x%016x" % chunk for chunk in chunks[:0x800 // chunk_size]) 645 for fragment in generate_table_lines(data, 12): 646 yield fragment 647 yield "\n ],\n" 648 649 # 0x800..0x10000 trie 650 (r2, r3) = compute_trie(chunks[0x800 // chunk_size : 0x10000 // chunk_size], 64 // chunk_size) 651 yield " r2: [\n" 652 data = map(str, r2) 653 for fragment in generate_table_lines(data, 12): 654 yield fragment 655 yield "\n ],\n" 656 657 yield " r3: &[\n" 658 data = ("0x%016x" % node for node in r3) 659 for fragment in generate_table_lines(data, 12): 660 yield fragment 661 yield "\n ],\n" 662 663 # 0x10000..0x110000 trie 664 (mid, r6) = compute_trie(chunks[0x10000 // chunk_size : 0x110000 // chunk_size], 665 64 // chunk_size) 666 (r4, r5) = compute_trie(mid, 64) 667 668 yield " r4: [\n" 669 data = map(str, r4) 670 for fragment in generate_table_lines(data, 12): 671 yield fragment 672 yield "\n ],\n" 673 674 yield " r5: &[\n" 675 data = map(str, r5) 676 for fragment in generate_table_lines(data, 12): 677 yield fragment 678 yield "\n ],\n" 679 680 yield " r6: &[\n" 681 data = ("0x%016x" % node for node in r6) 682 for fragment in generate_table_lines(data, 12): 683 yield fragment 684 yield "\n ],\n" 685 686 yield " };\n" 687 688 689def generate_small_bool_trie(name, codepoint_ranges, is_pub=False): 690 # type: (str, List[Tuple[int, int]], bool) -> Iterator[str] 691 """ 692 Generate Rust code for `SmallBoolTrie` struct. 693 694 See: `bool_trie.rs`. 695 """ 696 last_chunk = max(hi // 64 for (lo, hi) in codepoint_ranges) 697 n_chunks = last_chunk + 1 698 chunks = [0] * n_chunks 699 for (lo, hi) in codepoint_ranges: 700 for cp in range(lo, hi + 1): 701 assert cp // 64 < len(chunks) 702 chunks[cp // 64] |= 1 << (cp & 63) 703 704 pub_string = "" 705 if is_pub: 706 pub_string = "pub " 707 708 yield "\n" 709 yield " #[rustfmt::skip]\n" 710 yield (" %sconst %s: &super::SmallBoolTrie = &super::SmallBoolTrie {\n" 711 % (pub_string, name)) 712 713 (r1, r2) = compute_trie(chunks, 1) 714 715 yield " r1: &[\n" 716 data = (str(node) for node in r1) 717 for fragment in generate_table_lines(data, 12): 718 yield fragment 719 yield "\n ],\n" 720 721 yield " r2: &[\n" 722 data = ("0x%016x" % node for node in r2) 723 for fragment in generate_table_lines(data, 12): 724 yield fragment 725 yield "\n ],\n" 726 727 yield " };\n" 728 729 730def generate_property_module(mod, grouped_categories, category_subset): 731 # type: (str, Dict[str, List[Tuple[int, int]]], Iterable[str]) -> Iterator[str] 732 """ 733 Generate Rust code for module defining properties. 734 """ 735 736 yield "pub(crate) mod %s {" % mod 737 for cat in sorted(category_subset): 738 if cat in ("Cc", "White_Space"): 739 generator = generate_small_bool_trie("%s_table" % cat, grouped_categories[cat]) 740 else: 741 generator = generate_bool_trie("%s_table" % cat, grouped_categories[cat]) 742 743 for fragment in generator: 744 yield fragment 745 746 yield "\n" 747 yield " pub fn %s(c: char) -> bool {\n" % cat 748 yield " %s_table.lookup(c)\n" % cat 749 yield " }\n" 750 751 yield "}\n\n" 752 753 754def generate_conversions_module(unicode_data): 755 # type: (UnicodeData) -> Iterator[str] 756 """ 757 Generate Rust code for module defining conversions. 758 """ 759 760 yield "pub(crate) mod conversions {" 761 yield """ 762 pub fn to_lower(c: char) -> [char; 3] { 763 match bsearch_case_table(c, to_lowercase_table) { 764 None => [c, '\\0', '\\0'], 765 Some(index) => to_lowercase_table[index].1, 766 } 767 } 768 769 pub fn to_upper(c: char) -> [char; 3] { 770 match bsearch_case_table(c, to_uppercase_table) { 771 None => [c, '\\0', '\\0'], 772 Some(index) => to_uppercase_table[index].1, 773 } 774 } 775 776 fn bsearch_case_table(c: char, table: &[(char, [char; 3])]) -> Option<usize> { 777 table.binary_search_by(|&(key, _)| key.cmp(&c)).ok() 778 }\n""" 779 780 decl_type = "&[(char, [char; 3])]" 781 format_conversion = lambda x: "({},[{},{},{}])".format(*( 782 escape_char(c) for c in (x[0], x[1][0], x[1][1], x[1][2]) 783 )) 784 785 for fragment in generate_table( 786 name="to_lowercase_table", 787 items=sorted(unicode_data.to_lower.items(), key=lambda x: x[0]), 788 decl_type=decl_type, 789 is_pub=False, 790 format_item=format_conversion 791 ): 792 yield fragment 793 794 for fragment in generate_table( 795 name="to_uppercase_table", 796 items=sorted(unicode_data.to_upper.items(), key=lambda x: x[0]), 797 decl_type=decl_type, 798 is_pub=False, 799 format_item=format_conversion 800 ): 801 yield fragment 802 803 yield "}\n" 804 805 806def parse_args(): 807 # type: () -> argparse.Namespace 808 """ 809 Parse command line arguments. 810 """ 811 parser = argparse.ArgumentParser(description=__doc__) 812 parser.add_argument("-v", "--version", default=None, type=str, 813 help="Unicode version to use (if not specified," 814 " defaults to latest release).") 815 816 return parser.parse_args() 817 818 819def main(): 820 # type: () -> None 821 """ 822 Script entry point. 823 """ 824 args = parse_args() 825 826 unicode_version = fetch_files(args.version) 827 print("Using Unicode version: {}".format(unicode_version.as_str)) 828 829 # All the writing happens entirely in memory, we only write to file 830 # once we have generated the file content (it's not very large, <1 MB). 831 buf = StringIO() 832 buf.write(PREAMBLE) 833 834 unicode_version_notice = textwrap.dedent(""" 835 /// The version of [Unicode](http://www.unicode.org/) that the Unicode parts of 836 /// `char` and `str` methods are based on. 837 #[unstable(feature = "unicode_version", issue = "49726")] 838 pub const UNICODE_VERSION: UnicodeVersion = 839 UnicodeVersion {{ major: {v.major}, minor: {v.minor}, micro: {v.micro}, _priv: () }}; 840 """).format(v=unicode_version) 841 buf.write(unicode_version_notice) 842 843 get_path = lambda f: get_unicode_file_path(unicode_version, f) 844 845 unicode_data = load_unicode_data(get_path(UnicodeFiles.UNICODE_DATA)) 846 load_special_casing(get_path(UnicodeFiles.SPECIAL_CASING), unicode_data) 847 848 want_derived = {"Alphabetic", "Lowercase", "Uppercase", 849 "Cased", "Case_Ignorable", "Grapheme_Extend"} 850 derived = load_properties(get_path(UnicodeFiles.DERIVED_CORE_PROPERTIES), want_derived) 851 852 props = load_properties(get_path(UnicodeFiles.PROPS), 853 {"White_Space", "Join_Control", "Noncharacter_Code_Point"}) 854 855 # Category tables 856 for (name, categories, category_subset) in ( 857 ("general_category", unicode_data.general_categories, ["N", "Cc"]), 858 ("derived_property", derived, want_derived), 859 ("property", props, ["White_Space"]) 860 ): 861 for fragment in generate_property_module(name, categories, category_subset): 862 buf.write(fragment) 863 864 for fragment in generate_conversions_module(unicode_data): 865 buf.write(fragment) 866 867 tables_rs_path = os.path.join(THIS_DIR, "tables.rs") 868 869 # Actually write out the file content. 870 # Will overwrite the file if it exists. 871 with open(tables_rs_path, "w") as fd: 872 fd.write(buf.getvalue()) 873 874 print("Regenerated tables.rs.") 875 876 877if __name__ == "__main__": 878 main()