fork/rust
git clone https://git.julia.blue/fork/rust
4fb7144ed15
1.43.0
1#!/usr/bin/env python 2 3# This script uses the following Unicode tables: 4# - UnicodeData.txt 5 6 7from collections import namedtuple 8import csv 9import os 10import subprocess 11 12NUM_CODEPOINTS=0x110000 13 14def to_ranges(iter): 15 current = None 16 for i in iter: 17 if current is None or i != current[1] or i in (0x10000, 0x20000): 18 if current is not None: 19 yield tuple(current) 20 current = [i, i + 1] 21 else: 22 current[1] += 1 23 if current is not None: 24 yield tuple(current) 25 26def get_escaped(codepoints): 27 for c in codepoints: 28 if (c.class_ or "Cn") in "Cc Cf Cs Co Cn Zl Zp Zs".split() and c.value != ord(' '): 29 yield c.value 30 31def get_file(f): 32 try: 33 return open(os.path.basename(f)) 34 except FileNotFoundError: 35 subprocess.run(["curl", "-O", f], check=True) 36 return open(os.path.basename(f)) 37 38Codepoint = namedtuple('Codepoint', 'value class_') 39 40def get_codepoints(f): 41 r = csv.reader(f, delimiter=";") 42 prev_codepoint = 0 43 class_first = None 44 for row in r: 45 codepoint = int(row[0], 16) 46 name = row[1] 47 class_ = row[2] 48 49 if class_first is not None: 50 if not name.endswith("Last>"): 51 raise ValueError("Missing Last after First") 52 53 for c in range(prev_codepoint + 1, codepoint): 54 yield Codepoint(c, class_first) 55 56 class_first = None 57 if name.endswith("First>"): 58 class_first = class_ 59 60 yield Codepoint(codepoint, class_) 61 prev_codepoint = codepoint 62 63 if class_first is not None: 64 raise ValueError("Missing Last after First") 65 66 for c in range(prev_codepoint + 1, NUM_CODEPOINTS): 67 yield Codepoint(c, None) 68 69def compress_singletons(singletons): 70 uppers = [] # (upper, # items in lowers) 71 lowers = [] 72 73 for i in singletons: 74 upper = i >> 8 75 lower = i & 0xff 76 if len(uppers) == 0 or uppers[-1][0] != upper: 77 uppers.append((upper, 1)) 78 else: 79 upper, count = uppers[-1] 80 uppers[-1] = upper, count + 1 81 lowers.append(lower) 82 83 return uppers, lowers 84 85def compress_normal(normal): 86 # lengths 0x00..0x7f are encoded as 00, 01, ..., 7e, 7f 87 # lengths 0x80..0x7fff are encoded as 80 80, 80 81, ..., ff fe, ff ff 88 compressed = [] # [truelen, (truelenaux), falselen, (falselenaux)] 89 90 prev_start = 0 91 for start, count in normal: 92 truelen = start - prev_start 93 falselen = count 94 prev_start = start + count 95 96 assert truelen < 0x8000 and falselen < 0x8000 97 entry = [] 98 if truelen > 0x7f: 99 entry.append(0x80 | (truelen >> 8)) 100 entry.append(truelen & 0xff) 101 else: 102 entry.append(truelen & 0x7f) 103 if falselen > 0x7f: 104 entry.append(0x80 | (falselen >> 8)) 105 entry.append(falselen & 0xff) 106 else: 107 entry.append(falselen & 0x7f) 108 109 compressed.append(entry) 110 111 return compressed 112 113def print_singletons(uppers, lowers, uppersname, lowersname): 114 print("#[rustfmt::skip]") 115 print("const {}: &[(u8, u8)] = &[".format(uppersname)) 116 for u, c in uppers: 117 print(" ({:#04x}, {}),".format(u, c)) 118 print("];") 119 print("#[rustfmt::skip]") 120 print("const {}: &[u8] = &[".format(lowersname)) 121 for i in range(0, len(lowers), 8): 122 print(" {}".format(" ".join("{:#04x},".format(l) for l in lowers[i:i+8]))) 123 print("];") 124 125def print_normal(normal, normalname): 126 print("#[rustfmt::skip]") 127 print("const {}: &[u8] = &[".format(normalname)) 128 for v in normal: 129 print(" {}".format(" ".join("{:#04x},".format(i) for i in v))) 130 print("];") 131 132def main(): 133 file = get_file("http://www.unicode.org/Public/UNIDATA/UnicodeData.txt") 134 135 codepoints = get_codepoints(file) 136 137 CUTOFF=0x10000 138 singletons0 = [] 139 singletons1 = [] 140 normal0 = [] 141 normal1 = [] 142 extra = [] 143 144 for a, b in to_ranges(get_escaped(codepoints)): 145 if a > 2 * CUTOFF: 146 extra.append((a, b - a)) 147 elif a == b - 1: 148 if a & CUTOFF: 149 singletons1.append(a & ~CUTOFF) 150 else: 151 singletons0.append(a) 152 elif a == b - 2: 153 if a & CUTOFF: 154 singletons1.append(a & ~CUTOFF) 155 singletons1.append((a + 1) & ~CUTOFF) 156 else: 157 singletons0.append(a) 158 singletons0.append(a + 1) 159 else: 160 if a >= 2 * CUTOFF: 161 extra.append((a, b - a)) 162 elif a & CUTOFF: 163 normal1.append((a & ~CUTOFF, b - a)) 164 else: 165 normal0.append((a, b - a)) 166 167 singletons0u, singletons0l = compress_singletons(singletons0) 168 singletons1u, singletons1l = compress_singletons(singletons1) 169 normal0 = compress_normal(normal0) 170 normal1 = compress_normal(normal1) 171 172 print("""\ 173// NOTE: The following code was generated by "src/libcore/unicode/printable.py", 174// do not edit directly! 175 176fn check(x: u16, singletonuppers: &[(u8, u8)], singletonlowers: &[u8], normal: &[u8]) -> bool { 177 let xupper = (x >> 8) as u8; 178 let mut lowerstart = 0; 179 for &(upper, lowercount) in singletonuppers { 180 let lowerend = lowerstart + lowercount as usize; 181 if xupper == upper { 182 for &lower in &singletonlowers[lowerstart..lowerend] { 183 if lower == x as u8 { 184 return false; 185 } 186 } 187 } else if xupper < upper { 188 break; 189 } 190 lowerstart = lowerend; 191 } 192 193 let mut x = x as i32; 194 let mut normal = normal.iter().cloned(); 195 let mut current = true; 196 while let Some(v) = normal.next() { 197 let len = if v & 0x80 != 0 { 198 ((v & 0x7f) as i32) << 8 | normal.next().unwrap() as i32 199 } else { 200 v as i32 201 }; 202 x -= len; 203 if x < 0 { 204 break; 205 } 206 current = !current; 207 } 208 current 209} 210 211pub(crate) fn is_printable(x: char) -> bool { 212 let x = x as u32; 213 let lower = x as u16; 214 if x < 0x10000 { 215 check(lower, SINGLETONS0U, SINGLETONS0L, NORMAL0) 216 } else if x < 0x20000 { 217 check(lower, SINGLETONS1U, SINGLETONS1L, NORMAL1) 218 } else {\ 219""") 220 for a, b in extra: 221 print(" if 0x{:x} <= x && x < 0x{:x} {{".format(a, a + b)) 222 print(" return false;") 223 print(" }") 224 print("""\ 225 true 226 } 227}\ 228""") 229 print() 230 print_singletons(singletons0u, singletons0l, 'SINGLETONS0U', 'SINGLETONS0L') 231 print_singletons(singletons1u, singletons1l, 'SINGLETONS1U', 'SINGLETONS1L') 232 print_normal(normal0, 'NORMAL0') 233 print_normal(normal1, 'NORMAL1') 234 235if __name__ == '__main__': 236 main()