fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #66089 - Mark-Simulacrum:stable-next, r=Mark-Simulacrum4560ea788cb

1.39.0
26.5 KiB874 linesraw
1#!/usr/bin/env python
2
3"""
4Regenerate Unicode tables (tables.rs).
5"""
6
7# This script uses the Unicode tables as defined
8# in the UnicodeFiles class.
9
10# Since this should not require frequent updates, we just store this
11# out-of-line and check the tables.rs file into git.
12
13# Note that the "curl" program is required for operation.
14# This script is compatible with Python 2.7 and 3.x.
15
16import argparse
17import datetime
18import fileinput
19import itertools
20import os
21import re
22import textwrap
23import subprocess
24
25from collections import defaultdict, namedtuple
26
27try:
28    # Python 3
29    from itertools import zip_longest
30    from io import StringIO
31except ImportError:
32    # Python 2 compatibility
33    zip_longest = itertools.izip_longest
34    from StringIO import StringIO
35
36try:
37    # Completely optional type hinting
38    # (Python 2 compatible using comments,
39    # see: https://mypy.readthedocs.io/en/latest/python2.html)
40    # This is very helpful in typing-aware IDE like PyCharm.
41    from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Set, Tuple
42except ImportError:
43    pass
44
45
46# We don't use enum.Enum because of Python 2.7 compatibility.
47class UnicodeFiles(object):
48    # ReadMe does not contain any Unicode data, we
49    # only use it to extract versions.
50    README = "ReadMe.txt"
51
52    DERIVED_CORE_PROPERTIES = "DerivedCoreProperties.txt"
53    DERIVED_NORMALIZATION_PROPS = "DerivedNormalizationProps.txt"
54    PROPS = "PropList.txt"
55    SCRIPTS = "Scripts.txt"
56    SPECIAL_CASING = "SpecialCasing.txt"
57    UNICODE_DATA = "UnicodeData.txt"
58
59
60# The order doesn't really matter (Python < 3.6 won't preserve it),
61# we only want to aggregate all the file names.
62ALL_UNICODE_FILES = tuple(
63    value for name, value in UnicodeFiles.__dict__.items()
64    if not name.startswith("_")
65)
66
67assert len(ALL_UNICODE_FILES) == 7, "Unexpected number of unicode files"
68
69# The directory this file is located in.
70THIS_DIR = os.path.dirname(os.path.realpath(__file__))
71
72# Where to download the Unicode data.  The downloaded files
73# will be placed in sub-directories named after Unicode version.
74FETCH_DIR = os.path.join(THIS_DIR, "downloaded")
75
76FETCH_URL_LATEST = "ftp://ftp.unicode.org/Public/UNIDATA/{filename}"
77FETCH_URL_VERSION = "ftp://ftp.unicode.org/Public/{version}/ucd/{filename}"
78
79PREAMBLE = """\
80// NOTE: The following code was generated by "./unicode.py", do not edit directly
81
82#![allow(missing_docs, non_upper_case_globals, non_snake_case, clippy::unreadable_literal)]
83
84use crate::unicode::version::UnicodeVersion;
85use crate::unicode::bool_trie::{{BoolTrie, SmallBoolTrie}};
86""".format(year=datetime.datetime.now().year)
87
88# Mapping taken from Table 12 from:
89# http://www.unicode.org/reports/tr44/#General_Category_Values
90EXPANDED_CATEGORIES = {
91    "Lu": ["LC", "L"], "Ll": ["LC", "L"], "Lt": ["LC", "L"],
92    "Lm": ["L"], "Lo": ["L"],
93    "Mn": ["M"], "Mc": ["M"], "Me": ["M"],
94    "Nd": ["N"], "Nl": ["N"], "No": ["N"],
95    "Pc": ["P"], "Pd": ["P"], "Ps": ["P"], "Pe": ["P"],
96    "Pi": ["P"], "Pf": ["P"], "Po": ["P"],
97    "Sm": ["S"], "Sc": ["S"], "Sk": ["S"], "So": ["S"],
98    "Zs": ["Z"], "Zl": ["Z"], "Zp": ["Z"],
99    "Cc": ["C"], "Cf": ["C"], "Cs": ["C"], "Co": ["C"], "Cn": ["C"],
100}
101
102# This is the (inclusive) range of surrogate codepoints.
103# These are not valid Rust characters.
104SURROGATE_CODEPOINTS_RANGE = (0xd800, 0xdfff)
105
106UnicodeData = namedtuple(
107    "UnicodeData", (
108        # Conversions:
109        "to_upper", "to_lower", "to_title",
110
111        # Decompositions: canonical decompositions, compatibility decomp
112        "canon_decomp", "compat_decomp",
113
114        # Grouped: general categories and combining characters
115        "general_categories", "combines",
116    )
117)
118
119UnicodeVersion = namedtuple(
120    "UnicodeVersion", ("major", "minor", "micro", "as_str")
121)
122
123
124def fetch_files(version=None):
125    # type: (str) -> UnicodeVersion
126    """
127    Fetch all the Unicode files from unicode.org.
128
129    This will use cached files (stored in `FETCH_DIR`) if they exist,
130    creating them if they don't.  In any case, the Unicode version
131    is always returned.
132
133    :param version: The desired Unicode version, as string.
134        (If None, defaults to latest final release available,
135         querying the unicode.org service).
136    """
137    have_version = check_stored_version(version)
138    if have_version:
139        return have_version
140
141    if version:
142        # Check if the desired version exists on the server.
143        get_fetch_url = lambda name: FETCH_URL_VERSION.format(version=version, filename=name)
144    else:
145        # Extract the latest version.
146        get_fetch_url = lambda name: FETCH_URL_LATEST.format(filename=name)
147
148    readme_url = get_fetch_url(UnicodeFiles.README)
149
150    print("Fetching: {}".format(readme_url))
151    readme_content = subprocess.check_output(("curl", readme_url))
152
153    unicode_version = parse_readme_unicode_version(
154        readme_content.decode("utf8")
155    )
156
157    download_dir = get_unicode_dir(unicode_version)
158    if not os.path.exists(download_dir):
159        # For 2.7 compat, we don't use `exist_ok=True`.
160        os.makedirs(download_dir)
161
162    for filename in ALL_UNICODE_FILES:
163        file_path = get_unicode_file_path(unicode_version, filename)
164
165        if os.path.exists(file_path):
166            # Assume file on the server didn't change if it's been saved before.
167            continue
168
169        if filename == UnicodeFiles.README:
170            with open(file_path, "wb") as fd:
171                fd.write(readme_content)
172        else:
173            url = get_fetch_url(filename)
174            print("Fetching: {}".format(url))
175            subprocess.check_call(("curl", "-o", file_path, url))
176
177    return unicode_version
178
179
180def check_stored_version(version):
181    # type: (Optional[str]) -> Optional[UnicodeVersion]
182    """
183    Given desired Unicode version, return the version
184    if stored files are all present, and `None` otherwise.
185    """
186    if not version:
187        # If no desired version specified, we should check what's the latest
188        # version, skipping stored version checks.
189        return None
190
191    fetch_dir = os.path.join(FETCH_DIR, version)
192
193    for filename in ALL_UNICODE_FILES:
194        file_path = os.path.join(fetch_dir, filename)
195
196        if not os.path.exists(file_path):
197            return None
198
199    with open(os.path.join(fetch_dir, UnicodeFiles.README)) as fd:
200        return parse_readme_unicode_version(fd.read())
201
202
203def parse_readme_unicode_version(readme_content):
204    # type: (str) -> UnicodeVersion
205    """
206    Parse the Unicode version contained in their `ReadMe.txt` file.
207    """
208    # "Raw string" is necessary for \d not being treated as escape char
209    # (for the sake of compat with future Python versions).
210    # See: https://docs.python.org/3.6/whatsnew/3.6.html#deprecated-python-behavior
211    pattern = r"for Version (\d+)\.(\d+)\.(\d+) of the Unicode"
212    groups = re.search(pattern, readme_content).groups()
213
214    return UnicodeVersion(*map(int, groups), as_str=".".join(groups))
215
216
217def get_unicode_dir(unicode_version):
218    # type: (UnicodeVersion) -> str
219    """
220    Indicate in which parent dir the Unicode data files should be stored.
221
222    This returns a full, absolute path.
223    """
224    return os.path.join(FETCH_DIR, unicode_version.as_str)
225
226
227def get_unicode_file_path(unicode_version, filename):
228    # type: (UnicodeVersion, str) -> str
229    """
230    Indicate where the Unicode data file should be stored.
231    """
232    return os.path.join(get_unicode_dir(unicode_version), filename)
233
234
235def is_surrogate(n):
236    # type: (int) -> bool
237    """
238    Tell if given codepoint is a surrogate (not a valid Rust character).
239    """
240    return SURROGATE_CODEPOINTS_RANGE[0] <= n <= SURROGATE_CODEPOINTS_RANGE[1]
241
242
243def load_unicode_data(file_path):
244    # type: (str) -> UnicodeData
245    """
246    Load main Unicode data.
247    """
248    # Conversions
249    to_lower = {}   # type: Dict[int, Tuple[int, int, int]]
250    to_upper = {}   # type: Dict[int, Tuple[int, int, int]]
251    to_title = {}   # type: Dict[int, Tuple[int, int, int]]
252
253    # Decompositions
254    compat_decomp = {}   # type: Dict[int, List[int]]
255    canon_decomp = {}    # type: Dict[int, List[int]]
256
257    # Combining characters
258    # FIXME: combines are not used
259    combines = defaultdict(set)   # type: Dict[str, Set[int]]
260
261    # Categories
262    general_categories = defaultdict(set)   # type: Dict[str, Set[int]]
263    category_assigned_codepoints = set()    # type: Set[int]
264
265    all_codepoints = {}
266
267    range_start = -1
268
269    for line in fileinput.input(file_path):
270        data = line.split(";")
271        if len(data) != 15:
272            continue
273        codepoint = int(data[0], 16)
274        if is_surrogate(codepoint):
275            continue
276        if range_start >= 0:
277            for i in range(range_start, codepoint):
278                all_codepoints[i] = data
279            range_start = -1
280        if data[1].endswith(", First>"):
281            range_start = codepoint
282            continue
283        all_codepoints[codepoint] = data
284
285    for code, data in all_codepoints.items():
286        (code_org, name, gencat, combine, bidi,
287         decomp, deci, digit, num, mirror,
288         old, iso, upcase, lowcase, titlecase) = data
289
290        # Generate char to char direct common and simple conversions:
291
292        # Uppercase to lowercase
293        if lowcase != "" and code_org != lowcase:
294            to_lower[code] = (int(lowcase, 16), 0, 0)
295
296        # Lowercase to uppercase
297        if upcase != "" and code_org != upcase:
298            to_upper[code] = (int(upcase, 16), 0, 0)
299
300        # Title case
301        if titlecase.strip() != "" and code_org != titlecase:
302            to_title[code] = (int(titlecase, 16), 0, 0)
303
304        # Store decomposition, if given
305        if decomp:
306            decompositions = decomp.split()[1:]
307            decomp_code_points = [int(i, 16) for i in decompositions]
308
309            if decomp.startswith("<"):
310                # Compatibility decomposition
311                compat_decomp[code] = decomp_code_points
312            else:
313                # Canonical decomposition
314                canon_decomp[code] = decomp_code_points
315
316        # Place letter in categories as appropriate.
317        for cat in itertools.chain((gencat, ), EXPANDED_CATEGORIES.get(gencat, [])):
318            general_categories[cat].add(code)
319            category_assigned_codepoints.add(code)
320
321        # Record combining class, if any.
322        if combine != "0":
323            combines[combine].add(code)
324
325    # Generate Not_Assigned from Assigned.
326    general_categories["Cn"] = get_unassigned_codepoints(category_assigned_codepoints)
327
328    # Other contains Not_Assigned
329    general_categories["C"].update(general_categories["Cn"])
330
331    grouped_categories = group_categories(general_categories)
332
333    # FIXME: combines are not used
334    return UnicodeData(
335        to_lower=to_lower, to_upper=to_upper, to_title=to_title,
336        compat_decomp=compat_decomp, canon_decomp=canon_decomp,
337        general_categories=grouped_categories, combines=combines,
338    )
339
340
341def load_special_casing(file_path, unicode_data):
342    # type: (str, UnicodeData) -> None
343    """
344    Load special casing data and enrich given Unicode data.
345    """
346    for line in fileinput.input(file_path):
347        data = line.split("#")[0].split(";")
348        if len(data) == 5:
349            code, lower, title, upper, _comment = data
350        elif len(data) == 6:
351            code, lower, title, upper, condition, _comment = data
352            if condition.strip():  # Only keep unconditional mappins
353                continue
354        else:
355            continue
356        code = code.strip()
357        lower = lower.strip()
358        title = title.strip()
359        upper = upper.strip()
360        key = int(code, 16)
361        for (map_, values) in ((unicode_data.to_lower, lower),
362                               (unicode_data.to_upper, upper),
363                               (unicode_data.to_title, title)):
364            if values != code:
365                split = values.split()
366
367                codepoints = list(itertools.chain(
368                    (int(i, 16) for i in split),
369                    (0 for _ in range(len(split), 3))
370                ))
371
372                assert len(codepoints) == 3
373                map_[key] = codepoints
374
375
376def group_categories(mapping):
377    # type: (Dict[Any, Iterable[int]]) -> Dict[str, List[Tuple[int, int]]]
378    """
379    Group codepoints mapped in "categories".
380    """
381    return {category: group_codepoints(codepoints)
382            for category, codepoints in mapping.items()}
383
384
385def group_codepoints(codepoints):
386    # type: (Iterable[int]) -> List[Tuple[int, int]]
387    """
388    Group integral values into continuous, disjoint value ranges.
389
390    Performs value deduplication.
391
392    :return: sorted list of pairs denoting start and end of codepoint
393        group values, both ends inclusive.
394
395    >>> group_codepoints([1, 2, 10, 11, 12, 3, 4])
396    [(1, 4), (10, 12)]
397    >>> group_codepoints([1])
398    [(1, 1)]
399    >>> group_codepoints([1, 5, 6])
400    [(1, 1), (5, 6)]
401    >>> group_codepoints([])
402    []
403    """
404    sorted_codes = sorted(set(codepoints))
405    result = []     # type: List[Tuple[int, int]]
406
407    if not sorted_codes:
408        return result
409
410    next_codes = sorted_codes[1:]
411    start_code = sorted_codes[0]
412
413    for code, next_code in zip_longest(sorted_codes, next_codes, fillvalue=None):
414        if next_code is None or next_code - code != 1:
415            result.append((start_code, code))
416            start_code = next_code
417
418    return result
419
420
421def ungroup_codepoints(codepoint_pairs):
422    # type: (Iterable[Tuple[int, int]]) -> List[int]
423    """
424    The inverse of group_codepoints -- produce a flat list of values
425    from value range pairs.
426
427    >>> ungroup_codepoints([(1, 4), (10, 12)])
428    [1, 2, 3, 4, 10, 11, 12]
429    >>> ungroup_codepoints([(1, 1), (5, 6)])
430    [1, 5, 6]
431    >>> ungroup_codepoints(group_codepoints([1, 2, 7, 8]))
432    [1, 2, 7, 8]
433    >>> ungroup_codepoints([])
434    []
435    """
436    return list(itertools.chain.from_iterable(
437        range(lo, hi + 1) for lo, hi in codepoint_pairs
438    ))
439
440
441def get_unassigned_codepoints(assigned_codepoints):
442    # type: (Set[int]) -> Set[int]
443    """
444    Given a set of "assigned" codepoints, return a set
445    of these that are not in assigned and not surrogate.
446    """
447    return {i for i in range(0, 0x110000)
448            if i not in assigned_codepoints and not is_surrogate(i)}
449
450
451def generate_table_lines(items, indent, wrap=98):
452    # type: (Iterable[str], int, int) -> Iterator[str]
453    """
454    Given table items, generate wrapped lines of text with comma-separated items.
455
456    This is a generator function.
457
458    :param wrap: soft wrap limit (characters per line), integer.
459    """
460    line = " " * indent
461    first = True
462    for item in items:
463        if len(line) + len(item) < wrap:
464            if first:
465                line += item
466            else:
467                line += ", " + item
468            first = False
469        else:
470            yield line + ",\n"
471            line = " " * indent + item
472
473    yield line
474
475
476def load_properties(file_path, interesting_props):
477    # type: (str, Iterable[str]) -> Dict[str, List[Tuple[int, int]]]
478    """
479    Load properties data and return in grouped form.
480    """
481    props = defaultdict(list)   # type: Dict[str, List[Tuple[int, int]]]
482    # "Raw string" is necessary for `\.` and `\w` not to be treated as escape chars
483    # (for the sake of compat with future Python versions).
484    # See: https://docs.python.org/3.6/whatsnew/3.6.html#deprecated-python-behavior
485    re1 = re.compile(r"^ *([0-9A-F]+) *; *(\w+)")
486    re2 = re.compile(r"^ *([0-9A-F]+)\.\.([0-9A-F]+) *; *(\w+)")
487
488    for line in fileinput.input(file_path):
489        match = re1.match(line) or re2.match(line)
490        if match:
491            groups = match.groups()
492
493            if len(groups) == 2:
494                # `re1` matched (2 groups).
495                d_lo, prop = groups
496                d_hi = d_lo
497            else:
498                d_lo, d_hi, prop = groups
499        else:
500            continue
501
502        if interesting_props and prop not in interesting_props:
503            continue
504
505        lo_value = int(d_lo, 16)
506        hi_value = int(d_hi, 16)
507
508        props[prop].append((lo_value, hi_value))
509
510    # Optimize if possible.
511    for prop in props:
512        props[prop] = group_codepoints(ungroup_codepoints(props[prop]))
513
514    return props
515
516
517def escape_char(c):
518    # type: (int) -> str
519    r"""
520    Escape a codepoint for use as Rust char literal.
521
522    Outputs are OK to use as Rust source code as char literals
523    and they also include necessary quotes.
524
525    >>> escape_char(97)
526    "'\\u{61}'"
527    >>> escape_char(0)
528    "'\\0'"
529    """
530    return r"'\u{%x}'" % c if c != 0 else r"'\0'"
531
532
533def format_char_pair(pair):
534    # type: (Tuple[int, int]) -> str
535    """
536    Format a pair of two Rust chars.
537    """
538    return "(%s,%s)" % (escape_char(pair[0]), escape_char(pair[1]))
539
540
541def generate_table(
542    name,   # type: str
543    items,  # type: List[Tuple[int, int]]
544    decl_type="&[(char, char)]",    # type: str
545    is_pub=True,                    # type: bool
546    format_item=format_char_pair,   # type: Callable[[Tuple[int, int]], str]
547):
548    # type: (...) -> Iterator[str]
549    """
550    Generate a nicely formatted Rust constant "table" array.
551
552    This generates actual Rust code.
553    """
554    pub_string = ""
555    if is_pub:
556        pub_string = "pub "
557
558    yield "    %sconst %s: %s = &[\n" % (pub_string, name, decl_type)
559
560    data = []
561    first = True
562    for item in items:
563        if not first:
564            data.append(",")
565        first = False
566        data.extend(format_item(item))
567
568    for table_line in generate_table_lines("".join(data).split(","), 8):
569        yield table_line
570
571    yield "\n    ];\n\n"
572
573
574def compute_trie(raw_data, chunk_size):
575    # type: (List[int], int) -> Tuple[List[int], List[int]]
576    """
577    Compute postfix-compressed trie.
578
579    See: bool_trie.rs for more details.
580
581    >>> compute_trie([1, 2, 3, 1, 2, 3, 4, 5, 6], 3)
582    ([0, 0, 1], [1, 2, 3, 4, 5, 6])
583    >>> compute_trie([1, 2, 3, 1, 2, 4, 4, 5, 6], 3)
584    ([0, 1, 2], [1, 2, 3, 1, 2, 4, 4, 5, 6])
585    """
586    root = []
587    childmap = {}       # type: Dict[Tuple[int, ...], int]
588    child_data = []
589
590    assert len(raw_data) % chunk_size == 0, "Chunks must be equally sized"
591
592    for i in range(len(raw_data) // chunk_size):
593        data = raw_data[i * chunk_size : (i + 1) * chunk_size]
594
595        # Postfix compression of child nodes (data chunks)
596        # (identical child nodes are shared).
597
598        # Make a tuple out of the list so it's hashable.
599        child = tuple(data)
600        if child not in childmap:
601            childmap[child] = len(childmap)
602            child_data.extend(data)
603
604        root.append(childmap[child])
605
606    return root, child_data
607
608
609def generate_bool_trie(name, codepoint_ranges, is_pub=False):
610    # type: (str, List[Tuple[int, int]], bool) -> Iterator[str]
611    """
612    Generate Rust code for BoolTrie struct.
613
614    This yields string fragments that should be joined to produce
615    the final string.
616
617    See: `bool_trie.rs`.
618    """
619    chunk_size = 64
620    rawdata = [False] * 0x110000
621    for (lo, hi) in codepoint_ranges:
622        for cp in range(lo, hi + 1):
623            rawdata[cp] = True
624
625    # Convert to bitmap chunks of `chunk_size` bits each.
626    chunks = []
627    for i in range(0x110000 // chunk_size):
628        chunk = 0
629        for j in range(chunk_size):
630            if rawdata[i * chunk_size + j]:
631                chunk |= 1 << j
632        chunks.append(chunk)
633
634    pub_string = ""
635    if is_pub:
636        pub_string = "pub "
637    yield "    %sconst %s: &super::BoolTrie = &super::BoolTrie {\n" % (pub_string, name)
638    yield "        r1: [\n"
639    data = ("0x%016x" % chunk for chunk in chunks[:0x800 // chunk_size])
640    for fragment in generate_table_lines(data, 12):
641        yield fragment
642    yield "\n        ],\n"
643
644    # 0x800..0x10000 trie
645    (r2, r3) = compute_trie(chunks[0x800 // chunk_size : 0x10000 // chunk_size], 64 // chunk_size)
646    yield "        r2: [\n"
647    data = map(str, r2)
648    for fragment in generate_table_lines(data, 12):
649        yield fragment
650    yield "\n        ],\n"
651
652    yield "        r3: &[\n"
653    data = ("0x%016x" % node for node in r3)
654    for fragment in generate_table_lines(data, 12):
655        yield fragment
656    yield "\n        ],\n"
657
658    # 0x10000..0x110000 trie
659    (mid, r6) = compute_trie(chunks[0x10000 // chunk_size : 0x110000 // chunk_size],
660                             64 // chunk_size)
661    (r4, r5) = compute_trie(mid, 64)
662
663    yield "        r4: [\n"
664    data = map(str, r4)
665    for fragment in generate_table_lines(data, 12):
666        yield fragment
667    yield "\n        ],\n"
668
669    yield "        r5: &[\n"
670    data = map(str, r5)
671    for fragment in generate_table_lines(data, 12):
672        yield fragment
673    yield "\n        ],\n"
674
675    yield "        r6: &[\n"
676    data = ("0x%016x" % node for node in r6)
677    for fragment in generate_table_lines(data, 12):
678        yield fragment
679    yield "\n        ],\n"
680
681    yield "    };\n\n"
682
683
684def generate_small_bool_trie(name, codepoint_ranges, is_pub=False):
685    # type: (str, List[Tuple[int, int]], bool) -> Iterator[str]
686    """
687    Generate Rust code for `SmallBoolTrie` struct.
688
689    See: `bool_trie.rs`.
690    """
691    last_chunk = max(hi // 64 for (lo, hi) in codepoint_ranges)
692    n_chunks = last_chunk + 1
693    chunks = [0] * n_chunks
694    for (lo, hi) in codepoint_ranges:
695        for cp in range(lo, hi + 1):
696            assert cp // 64 < len(chunks)
697            chunks[cp // 64] |= 1 << (cp & 63)
698
699    pub_string = ""
700    if is_pub:
701        pub_string = "pub "
702
703    yield ("    %sconst %s: &super::SmallBoolTrie = &super::SmallBoolTrie {\n"
704           % (pub_string, name))
705
706    (r1, r2) = compute_trie(chunks, 1)
707
708    yield "        r1: &[\n"
709    data = (str(node) for node in r1)
710    for fragment in generate_table_lines(data, 12):
711        yield fragment
712    yield "\n        ],\n"
713
714    yield "        r2: &[\n"
715    data = ("0x%016x" % node for node in r2)
716    for fragment in generate_table_lines(data, 12):
717        yield fragment
718    yield "\n        ],\n"
719
720    yield "    };\n\n"
721
722
723def generate_property_module(mod, grouped_categories, category_subset):
724    # type: (str, Dict[str, List[Tuple[int, int]]], Iterable[str]) -> Iterator[str]
725    """
726    Generate Rust code for module defining properties.
727    """
728
729    yield "pub(crate) mod %s {\n" % mod
730    for cat in sorted(category_subset):
731        if cat in ("Cc", "White_Space"):
732            generator = generate_small_bool_trie("%s_table" % cat, grouped_categories[cat])
733        else:
734            generator = generate_bool_trie("%s_table" % cat, grouped_categories[cat])
735
736        for fragment in generator:
737            yield fragment
738
739        yield "    pub fn %s(c: char) -> bool {\n" % cat
740        yield "        %s_table.lookup(c)\n" % cat
741        yield "    }\n\n"
742
743    yield "}\n\n"
744
745
746def generate_conversions_module(unicode_data):
747    # type: (UnicodeData) -> Iterator[str]
748    """
749    Generate Rust code for module defining conversions.
750    """
751
752    yield "pub(crate) mod conversions {"
753    yield """
754    pub fn to_lower(c: char) -> [char; 3] {
755        match bsearch_case_table(c, to_lowercase_table) {
756            None        => [c, '\\0', '\\0'],
757            Some(index) => to_lowercase_table[index].1,
758        }
759    }
760
761    pub fn to_upper(c: char) -> [char; 3] {
762        match bsearch_case_table(c, to_uppercase_table) {
763            None        => [c, '\\0', '\\0'],
764            Some(index) => to_uppercase_table[index].1,
765        }
766    }
767
768    fn bsearch_case_table(c: char, table: &[(char, [char; 3])]) -> Option<usize> {
769        table.binary_search_by(|&(key, _)| key.cmp(&c)).ok()
770    }\n\n"""
771
772    decl_type = "&[(char, [char; 3])]"
773    format_conversion = lambda x: "({},[{},{},{}])".format(*(
774        escape_char(c) for c in (x[0], x[1][0], x[1][1], x[1][2])
775    ))
776
777    for fragment in generate_table(
778        name="to_lowercase_table",
779        items=sorted(unicode_data.to_lower.items(), key=lambda x: x[0]),
780        decl_type=decl_type,
781        is_pub=False,
782        format_item=format_conversion
783    ):
784        yield fragment
785
786    for fragment in generate_table(
787        name="to_uppercase_table",
788        items=sorted(unicode_data.to_upper.items(), key=lambda x: x[0]),
789        decl_type=decl_type,
790        is_pub=False,
791        format_item=format_conversion
792    ):
793        yield fragment
794
795    yield "}\n"
796
797
798def parse_args():
799    # type: () -> argparse.Namespace
800    """
801    Parse command line arguments.
802    """
803    parser = argparse.ArgumentParser(description=__doc__)
804    parser.add_argument("-v", "--version", default=None, type=str,
805                        help="Unicode version to use (if not specified,"
806                             " defaults to latest release).")
807
808    return parser.parse_args()
809
810
811def main():
812    # type: () -> None
813    """
814    Script entry point.
815    """
816    args = parse_args()
817
818    unicode_version = fetch_files(args.version)
819    print("Using Unicode version: {}".format(unicode_version.as_str))
820
821    # All the writing happens entirely in memory, we only write to file
822    # once we have generated the file content (it's not very large, <1 MB).
823    buf = StringIO()
824    buf.write(PREAMBLE)
825
826    unicode_version_notice = textwrap.dedent("""
827    /// The version of [Unicode](http://www.unicode.org/) that the Unicode parts of
828    /// `char` and `str` methods are based on.
829    #[unstable(feature = "unicode_version", issue = "49726")]
830    pub const UNICODE_VERSION: UnicodeVersion = UnicodeVersion {{
831        major: {version.major},
832        minor: {version.minor},
833        micro: {version.micro},
834        _priv: (),
835    }};
836    """).format(version=unicode_version)
837    buf.write(unicode_version_notice)
838
839    get_path = lambda f: get_unicode_file_path(unicode_version, f)
840
841    unicode_data = load_unicode_data(get_path(UnicodeFiles.UNICODE_DATA))
842    load_special_casing(get_path(UnicodeFiles.SPECIAL_CASING), unicode_data)
843
844    want_derived = {"Alphabetic", "Lowercase", "Uppercase",
845                    "Cased", "Case_Ignorable", "Grapheme_Extend"}
846    derived = load_properties(get_path(UnicodeFiles.DERIVED_CORE_PROPERTIES), want_derived)
847
848    props = load_properties(get_path(UnicodeFiles.PROPS),
849                            {"White_Space", "Join_Control", "Noncharacter_Code_Point"})
850
851    # Category tables
852    for (name, categories, category_subset) in (
853            ("general_category", unicode_data.general_categories, ["N", "Cc"]),
854            ("derived_property", derived, want_derived),
855            ("property", props, ["White_Space"])
856    ):
857        for fragment in generate_property_module(name, categories, category_subset):
858            buf.write(fragment)
859
860    for fragment in generate_conversions_module(unicode_data):
861        buf.write(fragment)
862
863    tables_rs_path = os.path.join(THIS_DIR, "tables.rs")
864
865    # Actually write out the file content.
866    # Will overwrite the file if it exists.
867    with open(tables_rs_path, "w") as fd:
868        fd.write(buf.getvalue())
869
870    print("Regenerated tables.rs.")
871
872
873if __name__ == "__main__":
874    main()