fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #68568 - pietroalbini:stable-next, r=pietroalbini5e1a799842b

1.41.0
26.6 KiB878 linesraw
1#!/usr/bin/env python
2
3"""
4Regenerate Unicode tables (tables.rs).
5"""
6
7# This script uses the Unicode tables as defined
8# in the UnicodeFiles class.
9
10# Since this should not require frequent updates, we just store this
11# out-of-line and check the tables.rs file into git.
12
13# Note that the "curl" program is required for operation.
14# This script is compatible with Python 2.7 and 3.x.
15
16import argparse
17import datetime
18import fileinput
19import itertools
20import os
21import re
22import textwrap
23import subprocess
24
25from collections import defaultdict, namedtuple
26
27try:
28    # Python 3
29    from itertools import zip_longest
30    from io import StringIO
31except ImportError:
32    # Python 2 compatibility
33    zip_longest = itertools.izip_longest
34    from StringIO import StringIO
35
36try:
37    # Completely optional type hinting
38    # (Python 2 compatible using comments,
39    # see: https://mypy.readthedocs.io/en/latest/python2.html)
40    # This is very helpful in typing-aware IDE like PyCharm.
41    from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Set, Tuple
42except ImportError:
43    pass
44
45
46# We don't use enum.Enum because of Python 2.7 compatibility.
47class UnicodeFiles(object):
48    # ReadMe does not contain any Unicode data, we
49    # only use it to extract versions.
50    README = "ReadMe.txt"
51
52    DERIVED_CORE_PROPERTIES = "DerivedCoreProperties.txt"
53    DERIVED_NORMALIZATION_PROPS = "DerivedNormalizationProps.txt"
54    PROPS = "PropList.txt"
55    SCRIPTS = "Scripts.txt"
56    SPECIAL_CASING = "SpecialCasing.txt"
57    UNICODE_DATA = "UnicodeData.txt"
58
59
60# The order doesn't really matter (Python < 3.6 won't preserve it),
61# we only want to aggregate all the file names.
62ALL_UNICODE_FILES = tuple(
63    value for name, value in UnicodeFiles.__dict__.items()
64    if not name.startswith("_")
65)
66
67assert len(ALL_UNICODE_FILES) == 7, "Unexpected number of unicode files"
68
69# The directory this file is located in.
70THIS_DIR = os.path.dirname(os.path.realpath(__file__))
71
72# Where to download the Unicode data.  The downloaded files
73# will be placed in sub-directories named after Unicode version.
74FETCH_DIR = os.path.join(THIS_DIR, "downloaded")
75
76FETCH_URL_LATEST = "ftp://ftp.unicode.org/Public/UNIDATA/{filename}"
77FETCH_URL_VERSION = "ftp://ftp.unicode.org/Public/{version}/ucd/{filename}"
78
79PREAMBLE = """\
80// NOTE: The following code was generated by "./unicode.py", do not edit directly
81
82#![allow(missing_docs, non_upper_case_globals, non_snake_case, clippy::unreadable_literal)]
83
84use crate::unicode::bool_trie::{{BoolTrie, SmallBoolTrie}};
85use crate::unicode::version::UnicodeVersion;
86""".format(year=datetime.datetime.now().year)
87
88# Mapping taken from Table 12 from:
89# http://www.unicode.org/reports/tr44/#General_Category_Values
90EXPANDED_CATEGORIES = {
91    "Lu": ["LC", "L"], "Ll": ["LC", "L"], "Lt": ["LC", "L"],
92    "Lm": ["L"], "Lo": ["L"],
93    "Mn": ["M"], "Mc": ["M"], "Me": ["M"],
94    "Nd": ["N"], "Nl": ["N"], "No": ["N"],
95    "Pc": ["P"], "Pd": ["P"], "Ps": ["P"], "Pe": ["P"],
96    "Pi": ["P"], "Pf": ["P"], "Po": ["P"],
97    "Sm": ["S"], "Sc": ["S"], "Sk": ["S"], "So": ["S"],
98    "Zs": ["Z"], "Zl": ["Z"], "Zp": ["Z"],
99    "Cc": ["C"], "Cf": ["C"], "Cs": ["C"], "Co": ["C"], "Cn": ["C"],
100}
101
102# This is the (inclusive) range of surrogate codepoints.
103# These are not valid Rust characters.
104SURROGATE_CODEPOINTS_RANGE = (0xd800, 0xdfff)
105
106UnicodeData = namedtuple(
107    "UnicodeData", (
108        # Conversions:
109        "to_upper", "to_lower", "to_title",
110
111        # Decompositions: canonical decompositions, compatibility decomp
112        "canon_decomp", "compat_decomp",
113
114        # Grouped: general categories and combining characters
115        "general_categories", "combines",
116    )
117)
118
119UnicodeVersion = namedtuple(
120    "UnicodeVersion", ("major", "minor", "micro", "as_str")
121)
122
123
124def fetch_files(version=None):
125    # type: (str) -> UnicodeVersion
126    """
127    Fetch all the Unicode files from unicode.org.
128
129    This will use cached files (stored in `FETCH_DIR`) if they exist,
130    creating them if they don't.  In any case, the Unicode version
131    is always returned.
132
133    :param version: The desired Unicode version, as string.
134        (If None, defaults to latest final release available,
135         querying the unicode.org service).
136    """
137    have_version = check_stored_version(version)
138    if have_version:
139        return have_version
140
141    if version:
142        # Check if the desired version exists on the server.
143        get_fetch_url = lambda name: FETCH_URL_VERSION.format(version=version, filename=name)
144    else:
145        # Extract the latest version.
146        get_fetch_url = lambda name: FETCH_URL_LATEST.format(filename=name)
147
148    readme_url = get_fetch_url(UnicodeFiles.README)
149
150    print("Fetching: {}".format(readme_url))
151    readme_content = subprocess.check_output(("curl", readme_url))
152
153    unicode_version = parse_readme_unicode_version(
154        readme_content.decode("utf8")
155    )
156
157    download_dir = get_unicode_dir(unicode_version)
158    if not os.path.exists(download_dir):
159        # For 2.7 compat, we don't use `exist_ok=True`.
160        os.makedirs(download_dir)
161
162    for filename in ALL_UNICODE_FILES:
163        file_path = get_unicode_file_path(unicode_version, filename)
164
165        if os.path.exists(file_path):
166            # Assume file on the server didn't change if it's been saved before.
167            continue
168
169        if filename == UnicodeFiles.README:
170            with open(file_path, "wb") as fd:
171                fd.write(readme_content)
172        else:
173            url = get_fetch_url(filename)
174            print("Fetching: {}".format(url))
175            subprocess.check_call(("curl", "-o", file_path, url))
176
177    return unicode_version
178
179
180def check_stored_version(version):
181    # type: (Optional[str]) -> Optional[UnicodeVersion]
182    """
183    Given desired Unicode version, return the version
184    if stored files are all present, and `None` otherwise.
185    """
186    if not version:
187        # If no desired version specified, we should check what's the latest
188        # version, skipping stored version checks.
189        return None
190
191    fetch_dir = os.path.join(FETCH_DIR, version)
192
193    for filename in ALL_UNICODE_FILES:
194        file_path = os.path.join(fetch_dir, filename)
195
196        if not os.path.exists(file_path):
197            return None
198
199    with open(os.path.join(fetch_dir, UnicodeFiles.README)) as fd:
200        return parse_readme_unicode_version(fd.read())
201
202
203def parse_readme_unicode_version(readme_content):
204    # type: (str) -> UnicodeVersion
205    """
206    Parse the Unicode version contained in their `ReadMe.txt` file.
207    """
208    # "Raw string" is necessary for \d not being treated as escape char
209    # (for the sake of compat with future Python versions).
210    # See: https://docs.python.org/3.6/whatsnew/3.6.html#deprecated-python-behavior
211    pattern = r"for Version (\d+)\.(\d+)\.(\d+) of the Unicode"
212    groups = re.search(pattern, readme_content).groups()
213
214    return UnicodeVersion(*map(int, groups), as_str=".".join(groups))
215
216
217def get_unicode_dir(unicode_version):
218    # type: (UnicodeVersion) -> str
219    """
220    Indicate in which parent dir the Unicode data files should be stored.
221
222    This returns a full, absolute path.
223    """
224    return os.path.join(FETCH_DIR, unicode_version.as_str)
225
226
227def get_unicode_file_path(unicode_version, filename):
228    # type: (UnicodeVersion, str) -> str
229    """
230    Indicate where the Unicode data file should be stored.
231    """
232    return os.path.join(get_unicode_dir(unicode_version), filename)
233
234
235def is_surrogate(n):
236    # type: (int) -> bool
237    """
238    Tell if given codepoint is a surrogate (not a valid Rust character).
239    """
240    return SURROGATE_CODEPOINTS_RANGE[0] <= n <= SURROGATE_CODEPOINTS_RANGE[1]
241
242
243def load_unicode_data(file_path):
244    # type: (str) -> UnicodeData
245    """
246    Load main Unicode data.
247    """
248    # Conversions
249    to_lower = {}   # type: Dict[int, Tuple[int, int, int]]
250    to_upper = {}   # type: Dict[int, Tuple[int, int, int]]
251    to_title = {}   # type: Dict[int, Tuple[int, int, int]]
252
253    # Decompositions
254    compat_decomp = {}   # type: Dict[int, List[int]]
255    canon_decomp = {}    # type: Dict[int, List[int]]
256
257    # Combining characters
258    # FIXME: combines are not used
259    combines = defaultdict(set)   # type: Dict[str, Set[int]]
260
261    # Categories
262    general_categories = defaultdict(set)   # type: Dict[str, Set[int]]
263    category_assigned_codepoints = set()    # type: Set[int]
264
265    all_codepoints = {}
266
267    range_start = -1
268
269    for line in fileinput.input(file_path):
270        data = line.split(";")
271        if len(data) != 15:
272            continue
273        codepoint = int(data[0], 16)
274        if is_surrogate(codepoint):
275            continue
276        if range_start >= 0:
277            for i in range(range_start, codepoint):
278                all_codepoints[i] = data
279            range_start = -1
280        if data[1].endswith(", First>"):
281            range_start = codepoint
282            continue
283        all_codepoints[codepoint] = data
284
285    for code, data in all_codepoints.items():
286        (code_org, name, gencat, combine, bidi,
287         decomp, deci, digit, num, mirror,
288         old, iso, upcase, lowcase, titlecase) = data
289
290        # Generate char to char direct common and simple conversions:
291
292        # Uppercase to lowercase
293        if lowcase != "" and code_org != lowcase:
294            to_lower[code] = (int(lowcase, 16), 0, 0)
295
296        # Lowercase to uppercase
297        if upcase != "" and code_org != upcase:
298            to_upper[code] = (int(upcase, 16), 0, 0)
299
300        # Title case
301        if titlecase.strip() != "" and code_org != titlecase:
302            to_title[code] = (int(titlecase, 16), 0, 0)
303
304        # Store decomposition, if given
305        if decomp:
306            decompositions = decomp.split()[1:]
307            decomp_code_points = [int(i, 16) for i in decompositions]
308
309            if decomp.startswith("<"):
310                # Compatibility decomposition
311                compat_decomp[code] = decomp_code_points
312            else:
313                # Canonical decomposition
314                canon_decomp[code] = decomp_code_points
315
316        # Place letter in categories as appropriate.
317        for cat in itertools.chain((gencat, ), EXPANDED_CATEGORIES.get(gencat, [])):
318            general_categories[cat].add(code)
319            category_assigned_codepoints.add(code)
320
321        # Record combining class, if any.
322        if combine != "0":
323            combines[combine].add(code)
324
325    # Generate Not_Assigned from Assigned.
326    general_categories["Cn"] = get_unassigned_codepoints(category_assigned_codepoints)
327
328    # Other contains Not_Assigned
329    general_categories["C"].update(general_categories["Cn"])
330
331    grouped_categories = group_categories(general_categories)
332
333    # FIXME: combines are not used
334    return UnicodeData(
335        to_lower=to_lower, to_upper=to_upper, to_title=to_title,
336        compat_decomp=compat_decomp, canon_decomp=canon_decomp,
337        general_categories=grouped_categories, combines=combines,
338    )
339
340
341def load_special_casing(file_path, unicode_data):
342    # type: (str, UnicodeData) -> None
343    """
344    Load special casing data and enrich given Unicode data.
345    """
346    for line in fileinput.input(file_path):
347        data = line.split("#")[0].split(";")
348        if len(data) == 5:
349            code, lower, title, upper, _comment = data
350        elif len(data) == 6:
351            code, lower, title, upper, condition, _comment = data
352            if condition.strip():  # Only keep unconditional mappins
353                continue
354        else:
355            continue
356        code = code.strip()
357        lower = lower.strip()
358        title = title.strip()
359        upper = upper.strip()
360        key = int(code, 16)
361        for (map_, values) in ((unicode_data.to_lower, lower),
362                               (unicode_data.to_upper, upper),
363                               (unicode_data.to_title, title)):
364            if values != code:
365                split = values.split()
366
367                codepoints = list(itertools.chain(
368                    (int(i, 16) for i in split),
369                    (0 for _ in range(len(split), 3))
370                ))
371
372                assert len(codepoints) == 3
373                map_[key] = codepoints
374
375
376def group_categories(mapping):
377    # type: (Dict[Any, Iterable[int]]) -> Dict[str, List[Tuple[int, int]]]
378    """
379    Group codepoints mapped in "categories".
380    """
381    return {category: group_codepoints(codepoints)
382            for category, codepoints in mapping.items()}
383
384
385def group_codepoints(codepoints):
386    # type: (Iterable[int]) -> List[Tuple[int, int]]
387    """
388    Group integral values into continuous, disjoint value ranges.
389
390    Performs value deduplication.
391
392    :return: sorted list of pairs denoting start and end of codepoint
393        group values, both ends inclusive.
394
395    >>> group_codepoints([1, 2, 10, 11, 12, 3, 4])
396    [(1, 4), (10, 12)]
397    >>> group_codepoints([1])
398    [(1, 1)]
399    >>> group_codepoints([1, 5, 6])
400    [(1, 1), (5, 6)]
401    >>> group_codepoints([])
402    []
403    """
404    sorted_codes = sorted(set(codepoints))
405    result = []     # type: List[Tuple[int, int]]
406
407    if not sorted_codes:
408        return result
409
410    next_codes = sorted_codes[1:]
411    start_code = sorted_codes[0]
412
413    for code, next_code in zip_longest(sorted_codes, next_codes, fillvalue=None):
414        if next_code is None or next_code - code != 1:
415            result.append((start_code, code))
416            start_code = next_code
417
418    return result
419
420
421def ungroup_codepoints(codepoint_pairs):
422    # type: (Iterable[Tuple[int, int]]) -> List[int]
423    """
424    The inverse of group_codepoints -- produce a flat list of values
425    from value range pairs.
426
427    >>> ungroup_codepoints([(1, 4), (10, 12)])
428    [1, 2, 3, 4, 10, 11, 12]
429    >>> ungroup_codepoints([(1, 1), (5, 6)])
430    [1, 5, 6]
431    >>> ungroup_codepoints(group_codepoints([1, 2, 7, 8]))
432    [1, 2, 7, 8]
433    >>> ungroup_codepoints([])
434    []
435    """
436    return list(itertools.chain.from_iterable(
437        range(lo, hi + 1) for lo, hi in codepoint_pairs
438    ))
439
440
441def get_unassigned_codepoints(assigned_codepoints):
442    # type: (Set[int]) -> Set[int]
443    """
444    Given a set of "assigned" codepoints, return a set
445    of these that are not in assigned and not surrogate.
446    """
447    return {i for i in range(0, 0x110000)
448            if i not in assigned_codepoints and not is_surrogate(i)}
449
450
451def generate_table_lines(items, indent, wrap=98):
452    # type: (Iterable[str], int, int) -> Iterator[str]
453    """
454    Given table items, generate wrapped lines of text with comma-separated items.
455
456    This is a generator function.
457
458    :param wrap: soft wrap limit (characters per line), integer.
459    """
460    line = " " * indent
461    first = True
462    for item in items:
463        if len(line) + len(item) < wrap:
464            if first:
465                line += item
466            else:
467                line += ", " + item
468            first = False
469        else:
470            yield line + ",\n"
471            line = " " * indent + item
472
473    yield line
474
475
476def load_properties(file_path, interesting_props):
477    # type: (str, Iterable[str]) -> Dict[str, List[Tuple[int, int]]]
478    """
479    Load properties data and return in grouped form.
480    """
481    props = defaultdict(list)   # type: Dict[str, List[Tuple[int, int]]]
482    # "Raw string" is necessary for `\.` and `\w` not to be treated as escape chars
483    # (for the sake of compat with future Python versions).
484    # See: https://docs.python.org/3.6/whatsnew/3.6.html#deprecated-python-behavior
485    re1 = re.compile(r"^ *([0-9A-F]+) *; *(\w+)")
486    re2 = re.compile(r"^ *([0-9A-F]+)\.\.([0-9A-F]+) *; *(\w+)")
487
488    for line in fileinput.input(file_path):
489        match = re1.match(line) or re2.match(line)
490        if match:
491            groups = match.groups()
492
493            if len(groups) == 2:
494                # `re1` matched (2 groups).
495                d_lo, prop = groups
496                d_hi = d_lo
497            else:
498                d_lo, d_hi, prop = groups
499        else:
500            continue
501
502        if interesting_props and prop not in interesting_props:
503            continue
504
505        lo_value = int(d_lo, 16)
506        hi_value = int(d_hi, 16)
507
508        props[prop].append((lo_value, hi_value))
509
510    # Optimize if possible.
511    for prop in props:
512        props[prop] = group_codepoints(ungroup_codepoints(props[prop]))
513
514    return props
515
516
517def escape_char(c):
518    # type: (int) -> str
519    r"""
520    Escape a codepoint for use as Rust char literal.
521
522    Outputs are OK to use as Rust source code as char literals
523    and they also include necessary quotes.
524
525    >>> escape_char(97)
526    "'\\u{61}'"
527    >>> escape_char(0)
528    "'\\0'"
529    """
530    return r"'\u{%x}'" % c if c != 0 else r"'\0'"
531
532
533def format_char_pair(pair):
534    # type: (Tuple[int, int]) -> str
535    """
536    Format a pair of two Rust chars.
537    """
538    return "(%s,%s)" % (escape_char(pair[0]), escape_char(pair[1]))
539
540
541def generate_table(
542    name,   # type: str
543    items,  # type: List[Tuple[int, int]]
544    decl_type="&[(char, char)]",    # type: str
545    is_pub=True,                    # type: bool
546    format_item=format_char_pair,   # type: Callable[[Tuple[int, int]], str]
547):
548    # type: (...) -> Iterator[str]
549    """
550    Generate a nicely formatted Rust constant "table" array.
551
552    This generates actual Rust code.
553    """
554    pub_string = ""
555    if is_pub:
556        pub_string = "pub "
557
558    yield "\n"
559    yield "    #[rustfmt::skip]\n"
560    yield "    %sconst %s: %s = &[\n" % (pub_string, name, decl_type)
561
562    data = []
563    first = True
564    for item in items:
565        if not first:
566            data.append(",")
567        first = False
568        data.extend(format_item(item))
569
570    for table_line in generate_table_lines("".join(data).split(","), 8):
571        yield table_line
572
573    yield "\n    ];\n"
574
575
576def compute_trie(raw_data, chunk_size):
577    # type: (List[int], int) -> Tuple[List[int], List[int]]
578    """
579    Compute postfix-compressed trie.
580
581    See: bool_trie.rs for more details.
582
583    >>> compute_trie([1, 2, 3, 1, 2, 3, 4, 5, 6], 3)
584    ([0, 0, 1], [1, 2, 3, 4, 5, 6])
585    >>> compute_trie([1, 2, 3, 1, 2, 4, 4, 5, 6], 3)
586    ([0, 1, 2], [1, 2, 3, 1, 2, 4, 4, 5, 6])
587    """
588    root = []
589    childmap = {}       # type: Dict[Tuple[int, ...], int]
590    child_data = []
591
592    assert len(raw_data) % chunk_size == 0, "Chunks must be equally sized"
593
594    for i in range(len(raw_data) // chunk_size):
595        data = raw_data[i * chunk_size : (i + 1) * chunk_size]
596
597        # Postfix compression of child nodes (data chunks)
598        # (identical child nodes are shared).
599
600        # Make a tuple out of the list so it's hashable.
601        child = tuple(data)
602        if child not in childmap:
603            childmap[child] = len(childmap)
604            child_data.extend(data)
605
606        root.append(childmap[child])
607
608    return root, child_data
609
610
611def generate_bool_trie(name, codepoint_ranges, is_pub=False):
612    # type: (str, List[Tuple[int, int]], bool) -> Iterator[str]
613    """
614    Generate Rust code for BoolTrie struct.
615
616    This yields string fragments that should be joined to produce
617    the final string.
618
619    See: `bool_trie.rs`.
620    """
621    chunk_size = 64
622    rawdata = [False] * 0x110000
623    for (lo, hi) in codepoint_ranges:
624        for cp in range(lo, hi + 1):
625            rawdata[cp] = True
626
627    # Convert to bitmap chunks of `chunk_size` bits each.
628    chunks = []
629    for i in range(0x110000 // chunk_size):
630        chunk = 0
631        for j in range(chunk_size):
632            if rawdata[i * chunk_size + j]:
633                chunk |= 1 << j
634        chunks.append(chunk)
635
636    pub_string = ""
637    if is_pub:
638        pub_string = "pub "
639
640    yield "\n"
641    yield "    #[rustfmt::skip]\n"
642    yield "    %sconst %s: &super::BoolTrie = &super::BoolTrie {\n" % (pub_string, name)
643    yield "        r1: [\n"
644    data = ("0x%016x" % chunk for chunk in chunks[:0x800 // chunk_size])
645    for fragment in generate_table_lines(data, 12):
646        yield fragment
647    yield "\n        ],\n"
648
649    # 0x800..0x10000 trie
650    (r2, r3) = compute_trie(chunks[0x800 // chunk_size : 0x10000 // chunk_size], 64 // chunk_size)
651    yield "        r2: [\n"
652    data = map(str, r2)
653    for fragment in generate_table_lines(data, 12):
654        yield fragment
655    yield "\n        ],\n"
656
657    yield "        r3: &[\n"
658    data = ("0x%016x" % node for node in r3)
659    for fragment in generate_table_lines(data, 12):
660        yield fragment
661    yield "\n        ],\n"
662
663    # 0x10000..0x110000 trie
664    (mid, r6) = compute_trie(chunks[0x10000 // chunk_size : 0x110000 // chunk_size],
665                             64 // chunk_size)
666    (r4, r5) = compute_trie(mid, 64)
667
668    yield "        r4: [\n"
669    data = map(str, r4)
670    for fragment in generate_table_lines(data, 12):
671        yield fragment
672    yield "\n        ],\n"
673
674    yield "        r5: &[\n"
675    data = map(str, r5)
676    for fragment in generate_table_lines(data, 12):
677        yield fragment
678    yield "\n        ],\n"
679
680    yield "        r6: &[\n"
681    data = ("0x%016x" % node for node in r6)
682    for fragment in generate_table_lines(data, 12):
683        yield fragment
684    yield "\n        ],\n"
685
686    yield "    };\n"
687
688
689def generate_small_bool_trie(name, codepoint_ranges, is_pub=False):
690    # type: (str, List[Tuple[int, int]], bool) -> Iterator[str]
691    """
692    Generate Rust code for `SmallBoolTrie` struct.
693
694    See: `bool_trie.rs`.
695    """
696    last_chunk = max(hi // 64 for (lo, hi) in codepoint_ranges)
697    n_chunks = last_chunk + 1
698    chunks = [0] * n_chunks
699    for (lo, hi) in codepoint_ranges:
700        for cp in range(lo, hi + 1):
701            assert cp // 64 < len(chunks)
702            chunks[cp // 64] |= 1 << (cp & 63)
703
704    pub_string = ""
705    if is_pub:
706        pub_string = "pub "
707
708    yield "\n"
709    yield "    #[rustfmt::skip]\n"
710    yield ("    %sconst %s: &super::SmallBoolTrie = &super::SmallBoolTrie {\n"
711           % (pub_string, name))
712
713    (r1, r2) = compute_trie(chunks, 1)
714
715    yield "        r1: &[\n"
716    data = (str(node) for node in r1)
717    for fragment in generate_table_lines(data, 12):
718        yield fragment
719    yield "\n        ],\n"
720
721    yield "        r2: &[\n"
722    data = ("0x%016x" % node for node in r2)
723    for fragment in generate_table_lines(data, 12):
724        yield fragment
725    yield "\n        ],\n"
726
727    yield "    };\n"
728
729
730def generate_property_module(mod, grouped_categories, category_subset):
731    # type: (str, Dict[str, List[Tuple[int, int]]], Iterable[str]) -> Iterator[str]
732    """
733    Generate Rust code for module defining properties.
734    """
735
736    yield "pub(crate) mod %s {" % mod
737    for cat in sorted(category_subset):
738        if cat in ("Cc", "White_Space"):
739            generator = generate_small_bool_trie("%s_table" % cat, grouped_categories[cat])
740        else:
741            generator = generate_bool_trie("%s_table" % cat, grouped_categories[cat])
742
743        for fragment in generator:
744            yield fragment
745
746        yield "\n"
747        yield "    pub fn %s(c: char) -> bool {\n" % cat
748        yield "        %s_table.lookup(c)\n" % cat
749        yield "    }\n"
750
751    yield "}\n\n"
752
753
754def generate_conversions_module(unicode_data):
755    # type: (UnicodeData) -> Iterator[str]
756    """
757    Generate Rust code for module defining conversions.
758    """
759
760    yield "pub(crate) mod conversions {"
761    yield """
762    pub fn to_lower(c: char) -> [char; 3] {
763        match bsearch_case_table(c, to_lowercase_table) {
764            None => [c, '\\0', '\\0'],
765            Some(index) => to_lowercase_table[index].1,
766        }
767    }
768
769    pub fn to_upper(c: char) -> [char; 3] {
770        match bsearch_case_table(c, to_uppercase_table) {
771            None => [c, '\\0', '\\0'],
772            Some(index) => to_uppercase_table[index].1,
773        }
774    }
775
776    fn bsearch_case_table(c: char, table: &[(char, [char; 3])]) -> Option<usize> {
777        table.binary_search_by(|&(key, _)| key.cmp(&c)).ok()
778    }\n"""
779
780    decl_type = "&[(char, [char; 3])]"
781    format_conversion = lambda x: "({},[{},{},{}])".format(*(
782        escape_char(c) for c in (x[0], x[1][0], x[1][1], x[1][2])
783    ))
784
785    for fragment in generate_table(
786        name="to_lowercase_table",
787        items=sorted(unicode_data.to_lower.items(), key=lambda x: x[0]),
788        decl_type=decl_type,
789        is_pub=False,
790        format_item=format_conversion
791    ):
792        yield fragment
793
794    for fragment in generate_table(
795        name="to_uppercase_table",
796        items=sorted(unicode_data.to_upper.items(), key=lambda x: x[0]),
797        decl_type=decl_type,
798        is_pub=False,
799        format_item=format_conversion
800    ):
801        yield fragment
802
803    yield "}\n"
804
805
806def parse_args():
807    # type: () -> argparse.Namespace
808    """
809    Parse command line arguments.
810    """
811    parser = argparse.ArgumentParser(description=__doc__)
812    parser.add_argument("-v", "--version", default=None, type=str,
813                        help="Unicode version to use (if not specified,"
814                             " defaults to latest release).")
815
816    return parser.parse_args()
817
818
819def main():
820    # type: () -> None
821    """
822    Script entry point.
823    """
824    args = parse_args()
825
826    unicode_version = fetch_files(args.version)
827    print("Using Unicode version: {}".format(unicode_version.as_str))
828
829    # All the writing happens entirely in memory, we only write to file
830    # once we have generated the file content (it's not very large, <1 MB).
831    buf = StringIO()
832    buf.write(PREAMBLE)
833
834    unicode_version_notice = textwrap.dedent("""
835    /// The version of [Unicode](http://www.unicode.org/) that the Unicode parts of
836    /// `char` and `str` methods are based on.
837    #[unstable(feature = "unicode_version", issue = "49726")]
838    pub const UNICODE_VERSION: UnicodeVersion =
839        UnicodeVersion {{ major: {v.major}, minor: {v.minor}, micro: {v.micro}, _priv: () }};
840    """).format(v=unicode_version)
841    buf.write(unicode_version_notice)
842
843    get_path = lambda f: get_unicode_file_path(unicode_version, f)
844
845    unicode_data = load_unicode_data(get_path(UnicodeFiles.UNICODE_DATA))
846    load_special_casing(get_path(UnicodeFiles.SPECIAL_CASING), unicode_data)
847
848    want_derived = {"Alphabetic", "Lowercase", "Uppercase",
849                    "Cased", "Case_Ignorable", "Grapheme_Extend"}
850    derived = load_properties(get_path(UnicodeFiles.DERIVED_CORE_PROPERTIES), want_derived)
851
852    props = load_properties(get_path(UnicodeFiles.PROPS),
853                            {"White_Space", "Join_Control", "Noncharacter_Code_Point"})
854
855    # Category tables
856    for (name, categories, category_subset) in (
857            ("general_category", unicode_data.general_categories, ["N", "Cc"]),
858            ("derived_property", derived, want_derived),
859            ("property", props, ["White_Space"])
860    ):
861        for fragment in generate_property_module(name, categories, category_subset):
862            buf.write(fragment)
863
864    for fragment in generate_conversions_module(unicode_data):
865        buf.write(fragment)
866
867    tables_rs_path = os.path.join(THIS_DIR, "tables.rs")
868
869    # Actually write out the file content.
870    # Will overwrite the file if it exists.
871    with open(tables_rs_path, "w") as fd:
872        fd.write(buf.getvalue())
873
874    print("Regenerated tables.rs.")
875
876
877if __name__ == "__main__":
878    main()