fork/rust
git clone https://git.julia.blue/fork/rust
4560ea788cb
1.39.0
1#!/usr/bin/env python 2 3# This script uses the following Unicode tables: 4# - UnicodeData.txt 5 6 7from collections import namedtuple 8import csv 9import os 10import subprocess 11 12NUM_CODEPOINTS=0x110000 13 14def to_ranges(iter): 15 current = None 16 for i in iter: 17 if current is None or i != current[1] or i in (0x10000, 0x20000): 18 if current is not None: 19 yield tuple(current) 20 current = [i, i + 1] 21 else: 22 current[1] += 1 23 if current is not None: 24 yield tuple(current) 25 26def get_escaped(codepoints): 27 for c in codepoints: 28 if (c.class_ or "Cn") in "Cc Cf Cs Co Cn Zl Zp Zs".split() and c.value != ord(' '): 29 yield c.value 30 31def get_file(f): 32 try: 33 return open(os.path.basename(f)) 34 except FileNotFoundError: 35 subprocess.run(["curl", "-O", f], check=True) 36 return open(os.path.basename(f)) 37 38Codepoint = namedtuple('Codepoint', 'value class_') 39 40def get_codepoints(f): 41 r = csv.reader(f, delimiter=";") 42 prev_codepoint = 0 43 class_first = None 44 for row in r: 45 codepoint = int(row[0], 16) 46 name = row[1] 47 class_ = row[2] 48 49 if class_first is not None: 50 if not name.endswith("Last>"): 51 raise ValueError("Missing Last after First") 52 53 for c in range(prev_codepoint + 1, codepoint): 54 yield Codepoint(c, class_first) 55 56 class_first = None 57 if name.endswith("First>"): 58 class_first = class_ 59 60 yield Codepoint(codepoint, class_) 61 prev_codepoint = codepoint 62 63 if class_first is not None: 64 raise ValueError("Missing Last after First") 65 66 for c in range(prev_codepoint + 1, NUM_CODEPOINTS): 67 yield Codepoint(c, None) 68 69def compress_singletons(singletons): 70 uppers = [] # (upper, # items in lowers) 71 lowers = [] 72 73 for i in singletons: 74 upper = i >> 8 75 lower = i & 0xff 76 if len(uppers) == 0 or uppers[-1][0] != upper: 77 uppers.append((upper, 1)) 78 else: 79 upper, count = uppers[-1] 80 uppers[-1] = upper, count + 1 81 lowers.append(lower) 82 83 return uppers, lowers 84 85def compress_normal(normal): 86 # lengths 0x00..0x7f are encoded as 00, 01, ..., 7e, 7f 87 # lengths 0x80..0x7fff are encoded as 80 80, 80 81, ..., ff fe, ff ff 88 compressed = [] # [truelen, (truelenaux), falselen, (falselenaux)] 89 90 prev_start = 0 91 for start, count in normal: 92 truelen = start - prev_start 93 falselen = count 94 prev_start = start + count 95 96 assert truelen < 0x8000 and falselen < 0x8000 97 entry = [] 98 if truelen > 0x7f: 99 entry.append(0x80 | (truelen >> 8)) 100 entry.append(truelen & 0xff) 101 else: 102 entry.append(truelen & 0x7f) 103 if falselen > 0x7f: 104 entry.append(0x80 | (falselen >> 8)) 105 entry.append(falselen & 0xff) 106 else: 107 entry.append(falselen & 0x7f) 108 109 compressed.append(entry) 110 111 return compressed 112 113def print_singletons(uppers, lowers, uppersname, lowersname): 114 print("const {}: &[(u8, u8)] = &[".format(uppersname)) 115 for u, c in uppers: 116 print(" ({:#04x}, {}),".format(u, c)) 117 print("];") 118 print("const {}: &[u8] = &[".format(lowersname)) 119 for i in range(0, len(lowers), 8): 120 print(" {}".format(" ".join("{:#04x},".format(l) for l in lowers[i:i+8]))) 121 print("];") 122 123def print_normal(normal, normalname): 124 print("const {}: &[u8] = &[".format(normalname)) 125 for v in normal: 126 print(" {}".format(" ".join("{:#04x},".format(i) for i in v))) 127 print("];") 128 129def main(): 130 file = get_file("http://www.unicode.org/Public/UNIDATA/UnicodeData.txt") 131 132 codepoints = get_codepoints(file) 133 134 CUTOFF=0x10000 135 singletons0 = [] 136 singletons1 = [] 137 normal0 = [] 138 normal1 = [] 139 extra = [] 140 141 for a, b in to_ranges(get_escaped(codepoints)): 142 if a > 2 * CUTOFF: 143 extra.append((a, b - a)) 144 elif a == b - 1: 145 if a & CUTOFF: 146 singletons1.append(a & ~CUTOFF) 147 else: 148 singletons0.append(a) 149 elif a == b - 2: 150 if a & CUTOFF: 151 singletons1.append(a & ~CUTOFF) 152 singletons1.append((a + 1) & ~CUTOFF) 153 else: 154 singletons0.append(a) 155 singletons0.append(a + 1) 156 else: 157 if a >= 2 * CUTOFF: 158 extra.append((a, b - a)) 159 elif a & CUTOFF: 160 normal1.append((a & ~CUTOFF, b - a)) 161 else: 162 normal0.append((a, b - a)) 163 164 singletons0u, singletons0l = compress_singletons(singletons0) 165 singletons1u, singletons1l = compress_singletons(singletons1) 166 normal0 = compress_normal(normal0) 167 normal1 = compress_normal(normal1) 168 169 print("""\ 170// NOTE: The following code was generated by "src/libcore/unicode/printable.py", 171// do not edit directly! 172 173fn check(x: u16, singletonuppers: &[(u8, u8)], singletonlowers: &[u8], 174 normal: &[u8]) -> bool { 175 let xupper = (x >> 8) as u8; 176 let mut lowerstart = 0; 177 for &(upper, lowercount) in singletonuppers { 178 let lowerend = lowerstart + lowercount as usize; 179 if xupper == upper { 180 for &lower in &singletonlowers[lowerstart..lowerend] { 181 if lower == x as u8 { 182 return false; 183 } 184 } 185 } else if xupper < upper { 186 break; 187 } 188 lowerstart = lowerend; 189 } 190 191 let mut x = x as i32; 192 let mut normal = normal.iter().cloned(); 193 let mut current = true; 194 while let Some(v) = normal.next() { 195 let len = if v & 0x80 != 0 { 196 ((v & 0x7f) as i32) << 8 | normal.next().unwrap() as i32 197 } else { 198 v as i32 199 }; 200 x -= len; 201 if x < 0 { 202 break; 203 } 204 current = !current; 205 } 206 current 207} 208 209pub(crate) fn is_printable(x: char) -> bool { 210 let x = x as u32; 211 let lower = x as u16; 212 if x < 0x10000 { 213 check(lower, SINGLETONS0U, SINGLETONS0L, NORMAL0) 214 } else if x < 0x20000 { 215 check(lower, SINGLETONS1U, SINGLETONS1L, NORMAL1) 216 } else {\ 217""") 218 for a, b in extra: 219 print(" if 0x{:x} <= x && x < 0x{:x} {{".format(a, a + b)) 220 print(" return false;") 221 print(" }") 222 print("""\ 223 true 224 } 225}\ 226""") 227 print() 228 print_singletons(singletons0u, singletons0l, 'SINGLETONS0U', 'SINGLETONS0L') 229 print_singletons(singletons1u, singletons1l, 'SINGLETONS1U', 'SINGLETONS1L') 230 print_normal(normal0, 'NORMAL0') 231 print_normal(normal1, 'NORMAL1') 232 233if __name__ == '__main__': 234 main()