fork/rust
git clone https://git.julia.blue/fork/rust
b8cedc00407
1.42.0
1from __future__ import absolute_import, division, print_function 2import argparse 3import contextlib 4import datetime 5import hashlib 6import os 7import re 8import shutil 9import subprocess 10import sys 11import tarfile 12import tempfile 13 14from time import time 15 16 17def get(url, path, verbose=False): 18 suffix = '.sha256' 19 sha_url = url + suffix 20 with tempfile.NamedTemporaryFile(delete=False) as temp_file: 21 temp_path = temp_file.name 22 with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file: 23 sha_path = sha_file.name 24 25 try: 26 download(sha_path, sha_url, False, verbose) 27 if os.path.exists(path): 28 if verify(path, sha_path, False): 29 if verbose: 30 print("using already-download file", path) 31 return 32 else: 33 if verbose: 34 print("ignoring already-download file", 35 path, "due to failed verification") 36 os.unlink(path) 37 download(temp_path, url, True, verbose) 38 if not verify(temp_path, sha_path, verbose): 39 raise RuntimeError("failed verification") 40 if verbose: 41 print("moving {} to {}".format(temp_path, path)) 42 shutil.move(temp_path, path) 43 finally: 44 delete_if_present(sha_path, verbose) 45 delete_if_present(temp_path, verbose) 46 47 48def delete_if_present(path, verbose): 49 """Remove the given file if present""" 50 if os.path.isfile(path): 51 if verbose: 52 print("removing", path) 53 os.unlink(path) 54 55 56def download(path, url, probably_big, verbose): 57 for _ in range(0, 4): 58 try: 59 _download(path, url, probably_big, verbose, True) 60 return 61 except RuntimeError: 62 print("\nspurious failure, trying again") 63 _download(path, url, probably_big, verbose, False) 64 65 66def _download(path, url, probably_big, verbose, exception): 67 if probably_big or verbose: 68 print("downloading {}".format(url)) 69 # see http://serverfault.com/questions/301128/how-to-download 70 if sys.platform == 'win32': 71 run(["PowerShell.exe", "/nologo", "-Command", 72 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", 73 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)], 74 verbose=verbose, 75 exception=exception) 76 else: 77 if probably_big or verbose: 78 option = "-#" 79 else: 80 option = "-s" 81 run(["curl", option, 82 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds 83 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds 84 "--retry", "3", "-Sf", "-o", path, url], 85 verbose=verbose, 86 exception=exception) 87 88 89def verify(path, sha_path, verbose): 90 """Check if the sha256 sum of the given path is valid""" 91 if verbose: 92 print("verifying", path) 93 with open(path, "rb") as source: 94 found = hashlib.sha256(source.read()).hexdigest() 95 with open(sha_path, "r") as sha256sum: 96 expected = sha256sum.readline().split()[0] 97 verified = found == expected 98 if not verified: 99 print("invalid checksum:\n" 100 " found: {}\n" 101 " expected: {}".format(found, expected)) 102 return verified 103 104 105def unpack(tarball, tarball_suffix, dst, verbose=False, match=None): 106 """Unpack the given tarball file""" 107 print("extracting", tarball) 108 fname = os.path.basename(tarball).replace(tarball_suffix, "") 109 with contextlib.closing(tarfile.open(tarball)) as tar: 110 for member in tar.getnames(): 111 if "/" not in member: 112 continue 113 name = member.replace(fname + "/", "", 1) 114 if match is not None and not name.startswith(match): 115 continue 116 name = name[len(match) + 1:] 117 118 dst_path = os.path.join(dst, name) 119 if verbose: 120 print(" extracting", member) 121 tar.extract(member, dst) 122 src_path = os.path.join(dst, member) 123 if os.path.isdir(src_path) and os.path.exists(dst_path): 124 continue 125 shutil.move(src_path, dst_path) 126 shutil.rmtree(os.path.join(dst, fname)) 127 128 129def run(args, verbose=False, exception=False, **kwargs): 130 """Run a child program in a new process""" 131 if verbose: 132 print("running: " + ' '.join(args)) 133 sys.stdout.flush() 134 # Use Popen here instead of call() as it apparently allows powershell on 135 # Windows to not lock up waiting for input presumably. 136 ret = subprocess.Popen(args, **kwargs) 137 code = ret.wait() 138 if code != 0: 139 err = "failed to run: " + ' '.join(args) 140 if verbose or exception: 141 raise RuntimeError(err) 142 sys.exit(err) 143 144 145def stage0_data(rust_root): 146 """Build a dictionary from stage0.txt""" 147 nightlies = os.path.join(rust_root, "src/stage0.txt") 148 with open(nightlies, 'r') as nightlies: 149 lines = [line.rstrip() for line in nightlies 150 if not line.startswith("#")] 151 return dict([line.split(": ", 1) for line in lines if line]) 152 153 154def format_build_time(duration): 155 """Return a nicer format for build time 156 157 >>> format_build_time('300') 158 '0:05:00' 159 """ 160 return str(datetime.timedelta(seconds=int(duration))) 161 162 163def default_build_triple(): 164 """Build triple as in LLVM""" 165 default_encoding = sys.getdefaultencoding() 166 try: 167 ostype = subprocess.check_output( 168 ['uname', '-s']).strip().decode(default_encoding) 169 cputype = subprocess.check_output( 170 ['uname', '-m']).strip().decode(default_encoding) 171 except (subprocess.CalledProcessError, OSError): 172 if sys.platform == 'win32': 173 return 'x86_64-pc-windows-msvc' 174 err = "uname not found" 175 sys.exit(err) 176 177 # The goal here is to come up with the same triple as LLVM would, 178 # at least for the subset of platforms we're willing to target. 179 ostype_mapper = { 180 'Darwin': 'apple-darwin', 181 'DragonFly': 'unknown-dragonfly', 182 'FreeBSD': 'unknown-freebsd', 183 'Haiku': 'unknown-haiku', 184 'NetBSD': 'unknown-netbsd', 185 'OpenBSD': 'unknown-openbsd' 186 } 187 188 # Consider the direct transformation first and then the special cases 189 if ostype in ostype_mapper: 190 ostype = ostype_mapper[ostype] 191 elif ostype == 'Linux': 192 os_from_sp = subprocess.check_output( 193 ['uname', '-o']).strip().decode(default_encoding) 194 if os_from_sp == 'Android': 195 ostype = 'linux-android' 196 else: 197 ostype = 'unknown-linux-gnu' 198 elif ostype == 'SunOS': 199 ostype = 'sun-solaris' 200 # On Solaris, uname -m will return a machine classification instead 201 # of a cpu type, so uname -p is recommended instead. However, the 202 # output from that option is too generic for our purposes (it will 203 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k 204 # must be used instead. 205 try: 206 cputype = subprocess.check_output( 207 ['isainfo', '-k']).strip().decode(default_encoding) 208 except (subprocess.CalledProcessError, OSError): 209 err = "isainfo not found" 210 sys.exit(err) 211 elif ostype.startswith('MINGW'): 212 # msys' `uname` does not print gcc configuration, but prints msys 213 # configuration. so we cannot believe `uname -m`: 214 # msys1 is always i686 and msys2 is always x86_64. 215 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and 216 # MINGW64 on x86_64. 217 ostype = 'pc-windows-gnu' 218 cputype = 'i686' 219 if os.environ.get('MSYSTEM') == 'MINGW64': 220 cputype = 'x86_64' 221 elif ostype.startswith('MSYS'): 222 ostype = 'pc-windows-gnu' 223 elif ostype.startswith('CYGWIN_NT'): 224 cputype = 'i686' 225 if ostype.endswith('WOW64'): 226 cputype = 'x86_64' 227 ostype = 'pc-windows-gnu' 228 else: 229 err = "unknown OS type: {}".format(ostype) 230 sys.exit(err) 231 232 if cputype == 'powerpc' and ostype == 'unknown-freebsd': 233 cputype = subprocess.check_output( 234 ['uname', '-p']).strip().decode(default_encoding) 235 cputype_mapper = { 236 'BePC': 'i686', 237 'aarch64': 'aarch64', 238 'amd64': 'x86_64', 239 'arm64': 'aarch64', 240 'i386': 'i686', 241 'i486': 'i686', 242 'i686': 'i686', 243 'i786': 'i686', 244 'powerpc': 'powerpc', 245 'powerpc64': 'powerpc64', 246 'powerpc64le': 'powerpc64le', 247 'ppc': 'powerpc', 248 'ppc64': 'powerpc64', 249 'ppc64le': 'powerpc64le', 250 's390x': 's390x', 251 'x64': 'x86_64', 252 'x86': 'i686', 253 'x86-64': 'x86_64', 254 'x86_64': 'x86_64' 255 } 256 257 # Consider the direct transformation first and then the special cases 258 if cputype in cputype_mapper: 259 cputype = cputype_mapper[cputype] 260 elif cputype in {'xscale', 'arm'}: 261 cputype = 'arm' 262 if ostype == 'linux-android': 263 ostype = 'linux-androideabi' 264 elif ostype == 'unknown-freebsd': 265 cputype = subprocess.check_output( 266 ['uname', '-p']).strip().decode(default_encoding) 267 ostype = 'unknown-freebsd' 268 elif cputype == 'armv6l': 269 cputype = 'arm' 270 if ostype == 'linux-android': 271 ostype = 'linux-androideabi' 272 else: 273 ostype += 'eabihf' 274 elif cputype in {'armv7l', 'armv8l'}: 275 cputype = 'armv7' 276 if ostype == 'linux-android': 277 ostype = 'linux-androideabi' 278 else: 279 ostype += 'eabihf' 280 elif cputype == 'mips': 281 if sys.byteorder == 'big': 282 cputype = 'mips' 283 elif sys.byteorder == 'little': 284 cputype = 'mipsel' 285 else: 286 raise ValueError("unknown byteorder: {}".format(sys.byteorder)) 287 elif cputype == 'mips64': 288 if sys.byteorder == 'big': 289 cputype = 'mips64' 290 elif sys.byteorder == 'little': 291 cputype = 'mips64el' 292 else: 293 raise ValueError('unknown byteorder: {}'.format(sys.byteorder)) 294 # only the n64 ABI is supported, indicate it 295 ostype += 'abi64' 296 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64': 297 pass 298 else: 299 err = "unknown cpu type: {}".format(cputype) 300 sys.exit(err) 301 302 return "{}-{}".format(cputype, ostype) 303 304 305@contextlib.contextmanager 306def output(filepath): 307 tmp = filepath + '.tmp' 308 with open(tmp, 'w') as f: 309 yield f 310 try: 311 os.remove(filepath) # PermissionError/OSError on Win32 if in use 312 os.rename(tmp, filepath) 313 except OSError: 314 shutil.copy2(tmp, filepath) 315 os.remove(tmp) 316 317 318class RustBuild(object): 319 """Provide all the methods required to build Rust""" 320 def __init__(self): 321 self.cargo_channel = '' 322 self.date = '' 323 self._download_url = '' 324 self.rustc_channel = '' 325 self.rustfmt_channel = '' 326 self.build = '' 327 self.build_dir = os.path.join(os.getcwd(), "build") 328 self.clean = False 329 self.config_toml = '' 330 self.rust_root = '' 331 self.use_locked_deps = '' 332 self.use_vendored_sources = '' 333 self.verbose = False 334 335 336 def download_stage0(self): 337 """Fetch the build system for Rust, written in Rust 338 339 This method will build a cache directory, then it will fetch the 340 tarball which has the stage0 compiler used to then bootstrap the Rust 341 compiler itself. 342 343 Each downloaded tarball is extracted, after that, the script 344 will move all the content to the right place. 345 """ 346 rustc_channel = self.rustc_channel 347 cargo_channel = self.cargo_channel 348 rustfmt_channel = self.rustfmt_channel 349 350 def support_xz(): 351 try: 352 with tempfile.NamedTemporaryFile(delete=False) as temp_file: 353 temp_path = temp_file.name 354 with tarfile.open(temp_path, "w:xz") as tar: 355 pass 356 return True 357 except tarfile.CompressionError: 358 return False 359 360 if self.rustc().startswith(self.bin_root()) and \ 361 (not os.path.exists(self.rustc()) or 362 self.program_out_of_date(self.rustc_stamp())): 363 if os.path.exists(self.bin_root()): 364 shutil.rmtree(self.bin_root()) 365 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' 366 filename = "rust-std-{}-{}{}".format( 367 rustc_channel, self.build, tarball_suffix) 368 pattern = "rust-std-{}".format(self.build) 369 self._download_stage0_helper(filename, pattern, tarball_suffix) 370 371 filename = "rustc-{}-{}{}".format(rustc_channel, self.build, 372 tarball_suffix) 373 self._download_stage0_helper(filename, "rustc", tarball_suffix) 374 self.fix_executable("{}/bin/rustc".format(self.bin_root())) 375 self.fix_executable("{}/bin/rustdoc".format(self.bin_root())) 376 with output(self.rustc_stamp()) as rust_stamp: 377 rust_stamp.write(self.date) 378 379 # This is required so that we don't mix incompatible MinGW 380 # libraries/binaries that are included in rust-std with 381 # the system MinGW ones. 382 if "pc-windows-gnu" in self.build: 383 filename = "rust-mingw-{}-{}{}".format( 384 rustc_channel, self.build, tarball_suffix) 385 self._download_stage0_helper(filename, "rust-mingw", tarball_suffix) 386 387 if self.cargo().startswith(self.bin_root()) and \ 388 (not os.path.exists(self.cargo()) or 389 self.program_out_of_date(self.cargo_stamp())): 390 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' 391 filename = "cargo-{}-{}{}".format(cargo_channel, self.build, 392 tarball_suffix) 393 self._download_stage0_helper(filename, "cargo", tarball_suffix) 394 self.fix_executable("{}/bin/cargo".format(self.bin_root())) 395 with output(self.cargo_stamp()) as cargo_stamp: 396 cargo_stamp.write(self.date) 397 398 if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and ( 399 not os.path.exists(self.rustfmt()) 400 or self.program_out_of_date(self.rustfmt_stamp()) 401 ): 402 if rustfmt_channel: 403 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' 404 [channel, date] = rustfmt_channel.split('-', 1) 405 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix) 406 self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date) 407 self.fix_executable("{}/bin/rustfmt".format(self.bin_root())) 408 self.fix_executable("{}/bin/cargo-fmt".format(self.bin_root())) 409 with output(self.rustfmt_stamp()) as rustfmt_stamp: 410 rustfmt_stamp.write(self.date) 411 412 def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None): 413 if date is None: 414 date = self.date 415 cache_dst = os.path.join(self.build_dir, "cache") 416 rustc_cache = os.path.join(cache_dst, date) 417 if not os.path.exists(rustc_cache): 418 os.makedirs(rustc_cache) 419 420 url = "{}/dist/{}".format(self._download_url, date) 421 tarball = os.path.join(rustc_cache, filename) 422 if not os.path.exists(tarball): 423 get("{}/{}".format(url, filename), tarball, verbose=self.verbose) 424 unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose) 425 426 @staticmethod 427 def fix_executable(fname): 428 """Modifies the interpreter section of 'fname' to fix the dynamic linker 429 430 This method is only required on NixOS and uses the PatchELF utility to 431 change the dynamic linker of ELF executables. 432 433 Please see https://nixos.org/patchelf.html for more information 434 """ 435 default_encoding = sys.getdefaultencoding() 436 try: 437 ostype = subprocess.check_output( 438 ['uname', '-s']).strip().decode(default_encoding) 439 except subprocess.CalledProcessError: 440 return 441 except OSError as reason: 442 if getattr(reason, 'winerror', None) is not None: 443 return 444 raise reason 445 446 if ostype != "Linux": 447 return 448 449 if not os.path.exists("/etc/NIXOS"): 450 return 451 if os.path.exists("/lib"): 452 return 453 454 # At this point we're pretty sure the user is running NixOS 455 nix_os_msg = "info: you seem to be running NixOS. Attempting to patch" 456 print(nix_os_msg, fname) 457 458 try: 459 interpreter = subprocess.check_output( 460 ["patchelf", "--print-interpreter", fname]) 461 interpreter = interpreter.strip().decode(default_encoding) 462 except subprocess.CalledProcessError as reason: 463 print("warning: failed to call patchelf:", reason) 464 return 465 466 loader = interpreter.split("/")[-1] 467 468 try: 469 ldd_output = subprocess.check_output( 470 ['ldd', '/run/current-system/sw/bin/sh']) 471 ldd_output = ldd_output.strip().decode(default_encoding) 472 except subprocess.CalledProcessError as reason: 473 print("warning: unable to call ldd:", reason) 474 return 475 476 for line in ldd_output.splitlines(): 477 libname = line.split()[0] 478 if libname.endswith(loader): 479 loader_path = libname[:len(libname) - len(loader)] 480 break 481 else: 482 print("warning: unable to find the path to the dynamic linker") 483 return 484 485 correct_interpreter = loader_path + loader 486 487 try: 488 subprocess.check_output( 489 ["patchelf", "--set-interpreter", correct_interpreter, fname]) 490 except subprocess.CalledProcessError as reason: 491 print("warning: failed to call patchelf:", reason) 492 return 493 494 def rustc_stamp(self): 495 """Return the path for .rustc-stamp 496 497 >>> rb = RustBuild() 498 >>> rb.build_dir = "build" 499 >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp") 500 True 501 """ 502 return os.path.join(self.bin_root(), '.rustc-stamp') 503 504 def cargo_stamp(self): 505 """Return the path for .cargo-stamp 506 507 >>> rb = RustBuild() 508 >>> rb.build_dir = "build" 509 >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp") 510 True 511 """ 512 return os.path.join(self.bin_root(), '.cargo-stamp') 513 514 def rustfmt_stamp(self): 515 """Return the path for .rustfmt-stamp 516 517 >>> rb = RustBuild() 518 >>> rb.build_dir = "build" 519 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp") 520 True 521 """ 522 return os.path.join(self.bin_root(), '.rustfmt-stamp') 523 524 def program_out_of_date(self, stamp_path): 525 """Check if the given program stamp is out of date""" 526 if not os.path.exists(stamp_path) or self.clean: 527 return True 528 with open(stamp_path, 'r') as stamp: 529 return self.date != stamp.read() 530 531 def bin_root(self): 532 """Return the binary root directory 533 534 >>> rb = RustBuild() 535 >>> rb.build_dir = "build" 536 >>> rb.bin_root() == os.path.join("build", "stage0") 537 True 538 539 When the 'build' property is given should be a nested directory: 540 541 >>> rb.build = "devel" 542 >>> rb.bin_root() == os.path.join("build", "devel", "stage0") 543 True 544 """ 545 return os.path.join(self.build_dir, self.build, "stage0") 546 547 def get_toml(self, key, section=None): 548 """Returns the value of the given key in config.toml, otherwise returns None 549 550 >>> rb = RustBuild() 551 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"' 552 >>> rb.get_toml("key2") 553 'value2' 554 555 If the key does not exists, the result is None: 556 557 >>> rb.get_toml("key3") is None 558 True 559 560 Optionally also matches the section the key appears in 561 562 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"' 563 >>> rb.get_toml('key', 'a') 564 'value1' 565 >>> rb.get_toml('key', 'b') 566 'value2' 567 >>> rb.get_toml('key', 'c') is None 568 True 569 570 >>> rb.config_toml = 'key1 = true' 571 >>> rb.get_toml("key1") 572 'true' 573 """ 574 575 cur_section = None 576 for line in self.config_toml.splitlines(): 577 section_match = re.match(r'^\s*\[(.*)\]\s*$', line) 578 if section_match is not None: 579 cur_section = section_match.group(1) 580 581 match = re.match(r'^{}\s*=(.*)$'.format(key), line) 582 if match is not None: 583 value = match.group(1) 584 if section is None or section == cur_section: 585 return self.get_string(value) or value.strip() 586 return None 587 588 def cargo(self): 589 """Return config path for cargo""" 590 return self.program_config('cargo') 591 592 def rustc(self): 593 """Return config path for rustc""" 594 return self.program_config('rustc') 595 596 def rustfmt(self): 597 """Return config path for rustfmt""" 598 if not self.rustfmt_channel: 599 return None 600 return self.program_config('rustfmt') 601 602 def program_config(self, program): 603 """Return config path for the given program 604 605 >>> rb = RustBuild() 606 >>> rb.config_toml = 'rustc = "rustc"\\n' 607 >>> rb.program_config('rustc') 608 'rustc' 609 >>> rb.config_toml = '' 610 >>> cargo_path = rb.program_config('cargo') 611 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(), 612 ... "bin", "cargo") 613 True 614 """ 615 config = self.get_toml(program) 616 if config: 617 return os.path.expanduser(config) 618 return os.path.join(self.bin_root(), "bin", "{}{}".format( 619 program, self.exe_suffix())) 620 621 @staticmethod 622 def get_string(line): 623 """Return the value between double quotes 624 625 >>> RustBuild.get_string(' "devel" ') 626 'devel' 627 >>> RustBuild.get_string(" 'devel' ") 628 'devel' 629 >>> RustBuild.get_string('devel') is None 630 True 631 >>> RustBuild.get_string(' "devel ') 632 '' 633 """ 634 start = line.find('"') 635 if start != -1: 636 end = start + 1 + line[start + 1:].find('"') 637 return line[start + 1:end] 638 start = line.find('\'') 639 if start != -1: 640 end = start + 1 + line[start + 1:].find('\'') 641 return line[start + 1:end] 642 return None 643 644 @staticmethod 645 def exe_suffix(): 646 """Return a suffix for executables""" 647 if sys.platform == 'win32': 648 return '.exe' 649 return '' 650 651 def bootstrap_binary(self): 652 """Return the path of the bootstrap binary 653 654 >>> rb = RustBuild() 655 >>> rb.build_dir = "build" 656 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap", 657 ... "debug", "bootstrap") 658 True 659 """ 660 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap") 661 662 def build_bootstrap(self): 663 """Build bootstrap""" 664 build_dir = os.path.join(self.build_dir, "bootstrap") 665 if self.clean and os.path.exists(build_dir): 666 shutil.rmtree(build_dir) 667 env = os.environ.copy() 668 env["RUSTC_BOOTSTRAP"] = '1' 669 env["CARGO_TARGET_DIR"] = build_dir 670 env["RUSTC"] = self.rustc() 671 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ 672 (os.pathsep + env["LD_LIBRARY_PATH"]) \ 673 if "LD_LIBRARY_PATH" in env else "" 674 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ 675 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \ 676 if "DYLD_LIBRARY_PATH" in env else "" 677 env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ 678 (os.pathsep + env["LIBRARY_PATH"]) \ 679 if "LIBRARY_PATH" in env else "" 680 # preserve existing RUSTFLAGS 681 env.setdefault("RUSTFLAGS", "") 682 env["RUSTFLAGS"] += " -Cdebuginfo=2" 683 684 build_section = "target.{}".format(self.build_triple()) 685 target_features = [] 686 if self.get_toml("crt-static", build_section) == "true": 687 target_features += ["+crt-static"] 688 elif self.get_toml("crt-static", build_section) == "false": 689 target_features += ["-crt-static"] 690 if target_features: 691 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features)) 692 target_linker = self.get_toml("linker", build_section) 693 if target_linker is not None: 694 env["RUSTFLAGS"] += " -C linker=" + target_linker 695 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes" 696 if self.get_toml("deny-warnings", "rust") != "false": 697 env["RUSTFLAGS"] += " -Dwarnings" 698 699 env["PATH"] = os.path.join(self.bin_root(), "bin") + \ 700 os.pathsep + env["PATH"] 701 if not os.path.isfile(self.cargo()): 702 raise Exception("no cargo executable found at `{}`".format( 703 self.cargo())) 704 args = [self.cargo(), "build", "--manifest-path", 705 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")] 706 for _ in range(1, self.verbose): 707 args.append("--verbose") 708 if self.use_locked_deps: 709 args.append("--locked") 710 if self.use_vendored_sources: 711 args.append("--frozen") 712 run(args, env=env, verbose=self.verbose) 713 714 def build_triple(self): 715 """Build triple as in LLVM""" 716 config = self.get_toml('build') 717 if config: 718 return config 719 return default_build_triple() 720 721 def check_submodule(self, module, slow_submodules): 722 if not slow_submodules: 723 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"], 724 cwd=os.path.join(self.rust_root, module), 725 stdout=subprocess.PIPE) 726 return checked_out 727 else: 728 return None 729 730 def update_submodule(self, module, checked_out, recorded_submodules): 731 module_path = os.path.join(self.rust_root, module) 732 733 if checked_out is not None: 734 default_encoding = sys.getdefaultencoding() 735 checked_out = checked_out.communicate()[0].decode(default_encoding).strip() 736 if recorded_submodules[module] == checked_out: 737 return 738 739 print("Updating submodule", module) 740 741 run(["git", "submodule", "-q", "sync", module], 742 cwd=self.rust_root, verbose=self.verbose) 743 try: 744 run(["git", "submodule", "update", 745 "--init", "--recursive", "--progress", module], 746 cwd=self.rust_root, verbose=self.verbose, exception=True) 747 except RuntimeError: 748 # Some versions of git don't support --progress. 749 run(["git", "submodule", "update", 750 "--init", "--recursive", module], 751 cwd=self.rust_root, verbose=self.verbose) 752 run(["git", "reset", "-q", "--hard"], 753 cwd=module_path, verbose=self.verbose) 754 run(["git", "clean", "-qdfx"], 755 cwd=module_path, verbose=self.verbose) 756 757 def update_submodules(self): 758 """Update submodules""" 759 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \ 760 self.get_toml('submodules') == "false": 761 return 762 763 # check the existence of 'git' command 764 try: 765 subprocess.check_output(['git', '--version']) 766 except (subprocess.CalledProcessError, OSError): 767 print("error: `git` is not found, please make sure it's installed and in the path.") 768 sys.exit(1) 769 770 slow_submodules = self.get_toml('fast-submodules') == "false" 771 start_time = time() 772 if slow_submodules: 773 print('Unconditionally updating all submodules') 774 else: 775 print('Updating only changed submodules') 776 default_encoding = sys.getdefaultencoding() 777 submodules = [s.split(' ', 1)[1] for s in subprocess.check_output( 778 ["git", "config", "--file", 779 os.path.join(self.rust_root, ".gitmodules"), 780 "--get-regexp", "path"] 781 ).decode(default_encoding).splitlines()] 782 filtered_submodules = [] 783 submodules_names = [] 784 for module in submodules: 785 if module.endswith("llvm-project"): 786 if self.get_toml('llvm-config') and self.get_toml('lld') != 'true': 787 continue 788 check = self.check_submodule(module, slow_submodules) 789 filtered_submodules.append((module, check)) 790 submodules_names.append(module) 791 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names, 792 cwd=self.rust_root, stdout=subprocess.PIPE) 793 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines() 794 recorded_submodules = {} 795 for data in recorded: 796 data = data.split() 797 recorded_submodules[data[3]] = data[2] 798 for module in filtered_submodules: 799 self.update_submodule(module[0], module[1], recorded_submodules) 800 print("Submodules updated in %.2f seconds" % (time() - start_time)) 801 802 def set_normal_environment(self): 803 """Set download URL for normal environment""" 804 if 'RUSTUP_DIST_SERVER' in os.environ: 805 self._download_url = os.environ['RUSTUP_DIST_SERVER'] 806 else: 807 self._download_url = 'https://static.rust-lang.org' 808 809 def set_dev_environment(self): 810 """Set download URL for development environment""" 811 if 'RUSTUP_DEV_DIST_SERVER' in os.environ: 812 self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER'] 813 else: 814 self._download_url = 'https://dev-static.rust-lang.org' 815 816 def check_vendored_status(self): 817 """Check that vendoring is configured properly""" 818 vendor_dir = os.path.join(self.rust_root, 'vendor') 819 if 'SUDO_USER' in os.environ and not self.use_vendored_sources: 820 if os.environ.get('USER') != os.environ['SUDO_USER']: 821 self.use_vendored_sources = True 822 print('info: looks like you are running this command under `sudo`') 823 print(' and so in order to preserve your $HOME this will now') 824 print(' use vendored sources by default.') 825 if not os.path.exists(vendor_dir): 826 print('error: vendoring required, but vendor directory does not exist.') 827 print(' Run `cargo vendor` without sudo to initialize the ' 828 'vendor directory.') 829 raise Exception("{} not found".format(vendor_dir)) 830 831 if self.use_vendored_sources: 832 if not os.path.exists('.cargo'): 833 os.makedirs('.cargo') 834 with output('.cargo/config') as cargo_config: 835 cargo_config.write( 836 "[source.crates-io]\n" 837 "replace-with = 'vendored-sources'\n" 838 "registry = 'https://example.com'\n" 839 "\n" 840 "[source.vendored-sources]\n" 841 "directory = '{}/vendor'\n" 842 .format(self.rust_root)) 843 else: 844 if os.path.exists('.cargo'): 845 shutil.rmtree('.cargo') 846 847 def ensure_vendored(self): 848 """Ensure that the vendored sources are available if needed""" 849 vendor_dir = os.path.join(self.rust_root, 'vendor') 850 # Note that this does not handle updating the vendored dependencies if 851 # the rust git repository is updated. Normal development usually does 852 # not use vendoring, so hopefully this isn't too much of a problem. 853 if self.use_vendored_sources and not os.path.exists(vendor_dir): 854 run([self.cargo(), "vendor"], 855 verbose=self.verbose, cwd=self.rust_root) 856 857 858def bootstrap(help_triggered): 859 """Configure, fetch, build and run the initial bootstrap""" 860 861 # If the user is asking for help, let them know that the whole download-and-build 862 # process has to happen before anything is printed out. 863 if help_triggered: 864 print("info: Downloading and building bootstrap before processing --help") 865 print(" command. See src/bootstrap/README.md for help with common") 866 print(" commands.") 867 868 parser = argparse.ArgumentParser(description='Build rust') 869 parser.add_argument('--config') 870 parser.add_argument('--build') 871 parser.add_argument('--src') 872 parser.add_argument('--clean', action='store_true') 873 parser.add_argument('-v', '--verbose', action='count', default=0) 874 875 args = [a for a in sys.argv if a != '-h' and a != '--help'] 876 args, _ = parser.parse_known_args(args) 877 878 # Configure initial bootstrap 879 build = RustBuild() 880 build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..')) 881 build.verbose = args.verbose 882 build.clean = args.clean 883 884 try: 885 with open(args.config or 'config.toml') as config: 886 build.config_toml = config.read() 887 except (OSError, IOError): 888 pass 889 890 config_verbose = build.get_toml('verbose', 'build') 891 if config_verbose is not None: 892 build.verbose = max(build.verbose, int(config_verbose)) 893 894 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true' 895 896 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true' 897 898 build.check_vendored_status() 899 900 data = stage0_data(build.rust_root) 901 build.date = data['date'] 902 build.rustc_channel = data['rustc'] 903 build.cargo_channel = data['cargo'] 904 905 if "rustfmt" in data: 906 build.rustfmt_channel = data['rustfmt'] 907 908 if 'dev' in data: 909 build.set_dev_environment() 910 else: 911 build.set_normal_environment() 912 913 build.update_submodules() 914 915 # Fetch/build the bootstrap 916 build.build = args.build or build.build_triple() 917 build.download_stage0() 918 sys.stdout.flush() 919 build.ensure_vendored() 920 build.build_bootstrap() 921 sys.stdout.flush() 922 923 # Run the bootstrap 924 args = [build.bootstrap_binary()] 925 args.extend(sys.argv[1:]) 926 env = os.environ.copy() 927 env["BUILD"] = build.build 928 env["SRC"] = build.rust_root 929 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) 930 env["BOOTSTRAP_PYTHON"] = sys.executable 931 env["BUILD_DIR"] = build.build_dir 932 env["RUSTC_BOOTSTRAP"] = '1' 933 env["CARGO"] = build.cargo() 934 env["RUSTC"] = build.rustc() 935 if build.rustfmt(): 936 env["RUSTFMT"] = build.rustfmt() 937 run(args, env=env, verbose=build.verbose) 938 939 940def main(): 941 """Entry point for the bootstrap process""" 942 start_time = time() 943 944 # x.py help <cmd> ... 945 if len(sys.argv) > 1 and sys.argv[1] == 'help': 946 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:] 947 948 help_triggered = ( 949 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1) 950 try: 951 bootstrap(help_triggered) 952 if not help_triggered: 953 print("Build completed successfully in {}".format( 954 format_build_time(time() - start_time))) 955 except (SystemExit, KeyboardInterrupt) as error: 956 if hasattr(error, 'code') and isinstance(error.code, int): 957 exit_code = error.code 958 else: 959 exit_code = 1 960 print(error) 961 if not help_triggered: 962 print("Build completed unsuccessfully in {}".format( 963 format_build_time(time() - start_time))) 964 sys.exit(exit_code) 965 966 967if __name__ == '__main__': 968 main()