fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #66089 - Mark-Simulacrum:stable-next, r=Mark-Simulacrum4560ea788cb

1.39.0
6.6 KiB234 linesraw
1#!/usr/bin/env python
2
3# This script uses the following Unicode tables:
4# - UnicodeData.txt
5
6
7from collections import namedtuple
8import csv
9import os
10import subprocess
11
12NUM_CODEPOINTS=0x110000
13
14def to_ranges(iter):
15    current = None
16    for i in iter:
17        if current is None or i != current[1] or i in (0x10000, 0x20000):
18            if current is not None:
19                yield tuple(current)
20            current = [i, i + 1]
21        else:
22            current[1] += 1
23    if current is not None:
24        yield tuple(current)
25
26def get_escaped(codepoints):
27    for c in codepoints:
28        if (c.class_ or "Cn") in "Cc Cf Cs Co Cn Zl Zp Zs".split() and c.value != ord(' '):
29            yield c.value
30
31def get_file(f):
32    try:
33        return open(os.path.basename(f))
34    except FileNotFoundError:
35        subprocess.run(["curl", "-O", f], check=True)
36        return open(os.path.basename(f))
37
38Codepoint = namedtuple('Codepoint', 'value class_')
39
40def get_codepoints(f):
41    r = csv.reader(f, delimiter=";")
42    prev_codepoint = 0
43    class_first = None
44    for row in r:
45        codepoint = int(row[0], 16)
46        name = row[1]
47        class_ = row[2]
48
49        if class_first is not None:
50            if not name.endswith("Last>"):
51                raise ValueError("Missing Last after First")
52
53        for c in range(prev_codepoint + 1, codepoint):
54            yield Codepoint(c, class_first)
55
56        class_first = None
57        if name.endswith("First>"):
58            class_first = class_
59
60        yield Codepoint(codepoint, class_)
61        prev_codepoint = codepoint
62
63    if class_first is not None:
64        raise ValueError("Missing Last after First")
65
66    for c in range(prev_codepoint + 1, NUM_CODEPOINTS):
67        yield Codepoint(c, None)
68
69def compress_singletons(singletons):
70    uppers = [] # (upper, # items in lowers)
71    lowers = []
72
73    for i in singletons:
74        upper = i >> 8
75        lower = i & 0xff
76        if len(uppers) == 0 or uppers[-1][0] != upper:
77            uppers.append((upper, 1))
78        else:
79            upper, count = uppers[-1]
80            uppers[-1] = upper, count + 1
81        lowers.append(lower)
82
83    return uppers, lowers
84
85def compress_normal(normal):
86    # lengths 0x00..0x7f are encoded as 00, 01, ..., 7e, 7f
87    # lengths 0x80..0x7fff are encoded as 80 80, 80 81, ..., ff fe, ff ff
88    compressed = [] # [truelen, (truelenaux), falselen, (falselenaux)]
89
90    prev_start = 0
91    for start, count in normal:
92        truelen = start - prev_start
93        falselen = count
94        prev_start = start + count
95
96        assert truelen < 0x8000 and falselen < 0x8000
97        entry = []
98        if truelen > 0x7f:
99            entry.append(0x80 | (truelen >> 8))
100            entry.append(truelen & 0xff)
101        else:
102            entry.append(truelen & 0x7f)
103        if falselen > 0x7f:
104            entry.append(0x80 | (falselen >> 8))
105            entry.append(falselen & 0xff)
106        else:
107            entry.append(falselen & 0x7f)
108
109        compressed.append(entry)
110
111    return compressed
112
113def print_singletons(uppers, lowers, uppersname, lowersname):
114    print("const {}: &[(u8, u8)] = &[".format(uppersname))
115    for u, c in uppers:
116        print("    ({:#04x}, {}),".format(u, c))
117    print("];")
118    print("const {}: &[u8] = &[".format(lowersname))
119    for i in range(0, len(lowers), 8):
120        print("    {}".format(" ".join("{:#04x},".format(l) for l in lowers[i:i+8])))
121    print("];")
122
123def print_normal(normal, normalname):
124    print("const {}: &[u8] = &[".format(normalname))
125    for v in normal:
126        print("    {}".format(" ".join("{:#04x},".format(i) for i in v)))
127    print("];")
128
129def main():
130    file = get_file("http://www.unicode.org/Public/UNIDATA/UnicodeData.txt")
131
132    codepoints = get_codepoints(file)
133
134    CUTOFF=0x10000
135    singletons0 = []
136    singletons1 = []
137    normal0 = []
138    normal1 = []
139    extra = []
140
141    for a, b in to_ranges(get_escaped(codepoints)):
142        if a > 2 * CUTOFF:
143            extra.append((a, b - a))
144        elif a == b - 1:
145            if a & CUTOFF:
146                singletons1.append(a & ~CUTOFF)
147            else:
148                singletons0.append(a)
149        elif a == b - 2:
150            if a & CUTOFF:
151                singletons1.append(a & ~CUTOFF)
152                singletons1.append((a + 1) & ~CUTOFF)
153            else:
154                singletons0.append(a)
155                singletons0.append(a + 1)
156        else:
157            if a >= 2 * CUTOFF:
158                extra.append((a, b - a))
159            elif a & CUTOFF:
160                normal1.append((a & ~CUTOFF, b - a))
161            else:
162                normal0.append((a, b - a))
163
164    singletons0u, singletons0l = compress_singletons(singletons0)
165    singletons1u, singletons1l = compress_singletons(singletons1)
166    normal0 = compress_normal(normal0)
167    normal1 = compress_normal(normal1)
168
169    print("""\
170// NOTE: The following code was generated by "src/libcore/unicode/printable.py",
171//       do not edit directly!
172
173fn check(x: u16, singletonuppers: &[(u8, u8)], singletonlowers: &[u8],
174         normal: &[u8]) -> bool {
175    let xupper = (x >> 8) as u8;
176    let mut lowerstart = 0;
177    for &(upper, lowercount) in singletonuppers {
178        let lowerend = lowerstart + lowercount as usize;
179        if xupper == upper {
180            for &lower in &singletonlowers[lowerstart..lowerend] {
181                if lower == x as u8 {
182                    return false;
183                }
184            }
185        } else if xupper < upper {
186            break;
187        }
188        lowerstart = lowerend;
189    }
190
191    let mut x = x as i32;
192    let mut normal = normal.iter().cloned();
193    let mut current = true;
194    while let Some(v) = normal.next() {
195        let len = if v & 0x80 != 0 {
196            ((v & 0x7f) as i32) << 8 | normal.next().unwrap() as i32
197        } else {
198            v as i32
199        };
200        x -= len;
201        if x < 0 {
202            break;
203        }
204        current = !current;
205    }
206    current
207}
208
209pub(crate) fn is_printable(x: char) -> bool {
210    let x = x as u32;
211    let lower = x as u16;
212    if x < 0x10000 {
213        check(lower, SINGLETONS0U, SINGLETONS0L, NORMAL0)
214    } else if x < 0x20000 {
215        check(lower, SINGLETONS1U, SINGLETONS1L, NORMAL1)
216    } else {\
217""")
218    for a, b in extra:
219        print("        if 0x{:x} <= x && x < 0x{:x} {{".format(a, a + b))
220        print("            return false;")
221        print("        }")
222    print("""\
223        true
224    }
225}\
226""")
227    print()
228    print_singletons(singletons0u, singletons0l, 'SINGLETONS0U', 'SINGLETONS0L')
229    print_singletons(singletons1u, singletons1l, 'SINGLETONS1U', 'SINGLETONS1L')
230    print_normal(normal0, 'NORMAL0')
231    print_normal(normal1, 'NORMAL1')
232
233if __name__ == '__main__':
234    main()