fork/rust
git clone https://git.julia.blue/fork/rust
f3e1a954d2e
1.41.1
1from __future__ import absolute_import, division, print_function 2import argparse 3import contextlib 4import datetime 5import hashlib 6import os 7import re 8import shutil 9import subprocess 10import sys 11import tarfile 12import tempfile 13 14from time import time 15 16 17def get(url, path, verbose=False): 18 suffix = '.sha256' 19 sha_url = url + suffix 20 with tempfile.NamedTemporaryFile(delete=False) as temp_file: 21 temp_path = temp_file.name 22 with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file: 23 sha_path = sha_file.name 24 25 try: 26 download(sha_path, sha_url, False, verbose) 27 if os.path.exists(path): 28 if verify(path, sha_path, False): 29 if verbose: 30 print("using already-download file", path) 31 return 32 else: 33 if verbose: 34 print("ignoring already-download file", 35 path, "due to failed verification") 36 os.unlink(path) 37 download(temp_path, url, True, verbose) 38 if not verify(temp_path, sha_path, verbose): 39 raise RuntimeError("failed verification") 40 if verbose: 41 print("moving {} to {}".format(temp_path, path)) 42 shutil.move(temp_path, path) 43 finally: 44 delete_if_present(sha_path, verbose) 45 delete_if_present(temp_path, verbose) 46 47 48def delete_if_present(path, verbose): 49 """Remove the given file if present""" 50 if os.path.isfile(path): 51 if verbose: 52 print("removing", path) 53 os.unlink(path) 54 55 56def download(path, url, probably_big, verbose): 57 for _ in range(0, 4): 58 try: 59 _download(path, url, probably_big, verbose, True) 60 return 61 except RuntimeError: 62 print("\nspurious failure, trying again") 63 _download(path, url, probably_big, verbose, False) 64 65 66def _download(path, url, probably_big, verbose, exception): 67 if probably_big or verbose: 68 print("downloading {}".format(url)) 69 # see http://serverfault.com/questions/301128/how-to-download 70 if sys.platform == 'win32': 71 run(["PowerShell.exe", "/nologo", "-Command", 72 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", 73 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)], 74 verbose=verbose, 75 exception=exception) 76 else: 77 if probably_big or verbose: 78 option = "-#" 79 else: 80 option = "-s" 81 run(["curl", option, 82 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds 83 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds 84 "--retry", "3", "-Sf", "-o", path, url], 85 verbose=verbose, 86 exception=exception) 87 88 89def verify(path, sha_path, verbose): 90 """Check if the sha256 sum of the given path is valid""" 91 if verbose: 92 print("verifying", path) 93 with open(path, "rb") as source: 94 found = hashlib.sha256(source.read()).hexdigest() 95 with open(sha_path, "r") as sha256sum: 96 expected = sha256sum.readline().split()[0] 97 verified = found == expected 98 if not verified: 99 print("invalid checksum:\n" 100 " found: {}\n" 101 " expected: {}".format(found, expected)) 102 return verified 103 104 105def unpack(tarball, tarball_suffix, dst, verbose=False, match=None): 106 """Unpack the given tarball file""" 107 print("extracting", tarball) 108 fname = os.path.basename(tarball).replace(tarball_suffix, "") 109 with contextlib.closing(tarfile.open(tarball)) as tar: 110 for member in tar.getnames(): 111 if "/" not in member: 112 continue 113 name = member.replace(fname + "/", "", 1) 114 if match is not None and not name.startswith(match): 115 continue 116 name = name[len(match) + 1:] 117 118 dst_path = os.path.join(dst, name) 119 if verbose: 120 print(" extracting", member) 121 tar.extract(member, dst) 122 src_path = os.path.join(dst, member) 123 if os.path.isdir(src_path) and os.path.exists(dst_path): 124 continue 125 shutil.move(src_path, dst_path) 126 shutil.rmtree(os.path.join(dst, fname)) 127 128 129def run(args, verbose=False, exception=False, **kwargs): 130 """Run a child program in a new process""" 131 if verbose: 132 print("running: " + ' '.join(args)) 133 sys.stdout.flush() 134 # Use Popen here instead of call() as it apparently allows powershell on 135 # Windows to not lock up waiting for input presumably. 136 ret = subprocess.Popen(args, **kwargs) 137 code = ret.wait() 138 if code != 0: 139 err = "failed to run: " + ' '.join(args) 140 if verbose or exception: 141 raise RuntimeError(err) 142 sys.exit(err) 143 144 145def stage0_data(rust_root): 146 """Build a dictionary from stage0.txt""" 147 nightlies = os.path.join(rust_root, "src/stage0.txt") 148 with open(nightlies, 'r') as nightlies: 149 lines = [line.rstrip() for line in nightlies 150 if not line.startswith("#")] 151 return dict([line.split(": ", 1) for line in lines if line]) 152 153 154def format_build_time(duration): 155 """Return a nicer format for build time 156 157 >>> format_build_time('300') 158 '0:05:00' 159 """ 160 return str(datetime.timedelta(seconds=int(duration))) 161 162 163def default_build_triple(): 164 """Build triple as in LLVM""" 165 default_encoding = sys.getdefaultencoding() 166 try: 167 ostype = subprocess.check_output( 168 ['uname', '-s']).strip().decode(default_encoding) 169 cputype = subprocess.check_output( 170 ['uname', '-m']).strip().decode(default_encoding) 171 except (subprocess.CalledProcessError, OSError): 172 if sys.platform == 'win32': 173 return 'x86_64-pc-windows-msvc' 174 err = "uname not found" 175 sys.exit(err) 176 177 # The goal here is to come up with the same triple as LLVM would, 178 # at least for the subset of platforms we're willing to target. 179 ostype_mapper = { 180 'Darwin': 'apple-darwin', 181 'DragonFly': 'unknown-dragonfly', 182 'FreeBSD': 'unknown-freebsd', 183 'Haiku': 'unknown-haiku', 184 'NetBSD': 'unknown-netbsd', 185 'OpenBSD': 'unknown-openbsd' 186 } 187 188 # Consider the direct transformation first and then the special cases 189 if ostype in ostype_mapper: 190 ostype = ostype_mapper[ostype] 191 elif ostype == 'Linux': 192 os_from_sp = subprocess.check_output( 193 ['uname', '-o']).strip().decode(default_encoding) 194 if os_from_sp == 'Android': 195 ostype = 'linux-android' 196 else: 197 ostype = 'unknown-linux-gnu' 198 elif ostype == 'SunOS': 199 ostype = 'sun-solaris' 200 # On Solaris, uname -m will return a machine classification instead 201 # of a cpu type, so uname -p is recommended instead. However, the 202 # output from that option is too generic for our purposes (it will 203 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k 204 # must be used instead. 205 try: 206 cputype = subprocess.check_output( 207 ['isainfo', '-k']).strip().decode(default_encoding) 208 except (subprocess.CalledProcessError, OSError): 209 err = "isainfo not found" 210 sys.exit(err) 211 elif ostype.startswith('MINGW'): 212 # msys' `uname` does not print gcc configuration, but prints msys 213 # configuration. so we cannot believe `uname -m`: 214 # msys1 is always i686 and msys2 is always x86_64. 215 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and 216 # MINGW64 on x86_64. 217 ostype = 'pc-windows-gnu' 218 cputype = 'i686' 219 if os.environ.get('MSYSTEM') == 'MINGW64': 220 cputype = 'x86_64' 221 elif ostype.startswith('MSYS'): 222 ostype = 'pc-windows-gnu' 223 elif ostype.startswith('CYGWIN_NT'): 224 cputype = 'i686' 225 if ostype.endswith('WOW64'): 226 cputype = 'x86_64' 227 ostype = 'pc-windows-gnu' 228 else: 229 err = "unknown OS type: {}".format(ostype) 230 sys.exit(err) 231 232 if cputype == 'powerpc' and ostype == 'unknown-freebsd': 233 cputype = subprocess.check_output( 234 ['uname', '-p']).strip().decode(default_encoding) 235 cputype_mapper = { 236 'BePC': 'i686', 237 'aarch64': 'aarch64', 238 'amd64': 'x86_64', 239 'arm64': 'aarch64', 240 'i386': 'i686', 241 'i486': 'i686', 242 'i686': 'i686', 243 'i786': 'i686', 244 'powerpc': 'powerpc', 245 'powerpc64': 'powerpc64', 246 'powerpc64le': 'powerpc64le', 247 'ppc': 'powerpc', 248 'ppc64': 'powerpc64', 249 'ppc64le': 'powerpc64le', 250 's390x': 's390x', 251 'x64': 'x86_64', 252 'x86': 'i686', 253 'x86-64': 'x86_64', 254 'x86_64': 'x86_64' 255 } 256 257 # Consider the direct transformation first and then the special cases 258 if cputype in cputype_mapper: 259 cputype = cputype_mapper[cputype] 260 elif cputype in {'xscale', 'arm'}: 261 cputype = 'arm' 262 if ostype == 'linux-android': 263 ostype = 'linux-androideabi' 264 elif ostype == 'unknown-freebsd': 265 cputype = subprocess.check_output( 266 ['uname', '-p']).strip().decode(default_encoding) 267 ostype = 'unknown-freebsd' 268 elif cputype == 'armv6l': 269 cputype = 'arm' 270 if ostype == 'linux-android': 271 ostype = 'linux-androideabi' 272 else: 273 ostype += 'eabihf' 274 elif cputype in {'armv7l', 'armv8l'}: 275 cputype = 'armv7' 276 if ostype == 'linux-android': 277 ostype = 'linux-androideabi' 278 else: 279 ostype += 'eabihf' 280 elif cputype == 'mips': 281 if sys.byteorder == 'big': 282 cputype = 'mips' 283 elif sys.byteorder == 'little': 284 cputype = 'mipsel' 285 else: 286 raise ValueError("unknown byteorder: {}".format(sys.byteorder)) 287 elif cputype == 'mips64': 288 if sys.byteorder == 'big': 289 cputype = 'mips64' 290 elif sys.byteorder == 'little': 291 cputype = 'mips64el' 292 else: 293 raise ValueError('unknown byteorder: {}'.format(sys.byteorder)) 294 # only the n64 ABI is supported, indicate it 295 ostype += 'abi64' 296 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64': 297 pass 298 else: 299 err = "unknown cpu type: {}".format(cputype) 300 sys.exit(err) 301 302 return "{}-{}".format(cputype, ostype) 303 304 305@contextlib.contextmanager 306def output(filepath): 307 tmp = filepath + '.tmp' 308 with open(tmp, 'w') as f: 309 yield f 310 try: 311 os.remove(filepath) # PermissionError/OSError on Win32 if in use 312 os.rename(tmp, filepath) 313 except OSError: 314 shutil.copy2(tmp, filepath) 315 os.remove(tmp) 316 317 318class RustBuild(object): 319 """Provide all the methods required to build Rust""" 320 def __init__(self): 321 self.cargo_channel = '' 322 self.date = '' 323 self._download_url = '' 324 self.rustc_channel = '' 325 self.build = '' 326 self.build_dir = os.path.join(os.getcwd(), "build") 327 self.clean = False 328 self.config_toml = '' 329 self.rust_root = '' 330 self.use_locked_deps = '' 331 self.use_vendored_sources = '' 332 self.verbose = False 333 334 335 def download_stage0(self): 336 """Fetch the build system for Rust, written in Rust 337 338 This method will build a cache directory, then it will fetch the 339 tarball which has the stage0 compiler used to then bootstrap the Rust 340 compiler itself. 341 342 Each downloaded tarball is extracted, after that, the script 343 will move all the content to the right place. 344 """ 345 rustc_channel = self.rustc_channel 346 cargo_channel = self.cargo_channel 347 348 def support_xz(): 349 try: 350 with tempfile.NamedTemporaryFile(delete=False) as temp_file: 351 temp_path = temp_file.name 352 with tarfile.open(temp_path, "w:xz") as tar: 353 pass 354 return True 355 except tarfile.CompressionError: 356 return False 357 358 if self.rustc().startswith(self.bin_root()) and \ 359 (not os.path.exists(self.rustc()) or 360 self.program_out_of_date(self.rustc_stamp())): 361 if os.path.exists(self.bin_root()): 362 shutil.rmtree(self.bin_root()) 363 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' 364 filename = "rust-std-{}-{}{}".format( 365 rustc_channel, self.build, tarball_suffix) 366 pattern = "rust-std-{}".format(self.build) 367 self._download_stage0_helper(filename, pattern, tarball_suffix) 368 369 filename = "rustc-{}-{}{}".format(rustc_channel, self.build, 370 tarball_suffix) 371 self._download_stage0_helper(filename, "rustc", tarball_suffix) 372 self.fix_executable("{}/bin/rustc".format(self.bin_root())) 373 self.fix_executable("{}/bin/rustdoc".format(self.bin_root())) 374 with output(self.rustc_stamp()) as rust_stamp: 375 rust_stamp.write(self.date) 376 377 # This is required so that we don't mix incompatible MinGW 378 # libraries/binaries that are included in rust-std with 379 # the system MinGW ones. 380 if "pc-windows-gnu" in self.build: 381 filename = "rust-mingw-{}-{}{}".format( 382 rustc_channel, self.build, tarball_suffix) 383 self._download_stage0_helper(filename, "rust-mingw", tarball_suffix) 384 385 if self.cargo().startswith(self.bin_root()) and \ 386 (not os.path.exists(self.cargo()) or 387 self.program_out_of_date(self.cargo_stamp())): 388 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' 389 filename = "cargo-{}-{}{}".format(cargo_channel, self.build, 390 tarball_suffix) 391 self._download_stage0_helper(filename, "cargo", tarball_suffix) 392 self.fix_executable("{}/bin/cargo".format(self.bin_root())) 393 with output(self.cargo_stamp()) as cargo_stamp: 394 cargo_stamp.write(self.date) 395 396 def _download_stage0_helper(self, filename, pattern, tarball_suffix): 397 cache_dst = os.path.join(self.build_dir, "cache") 398 rustc_cache = os.path.join(cache_dst, self.date) 399 if not os.path.exists(rustc_cache): 400 os.makedirs(rustc_cache) 401 402 url = "{}/dist/{}".format(self._download_url, self.date) 403 tarball = os.path.join(rustc_cache, filename) 404 if not os.path.exists(tarball): 405 get("{}/{}".format(url, filename), tarball, verbose=self.verbose) 406 unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose) 407 408 @staticmethod 409 def fix_executable(fname): 410 """Modifies the interpreter section of 'fname' to fix the dynamic linker 411 412 This method is only required on NixOS and uses the PatchELF utility to 413 change the dynamic linker of ELF executables. 414 415 Please see https://nixos.org/patchelf.html for more information 416 """ 417 default_encoding = sys.getdefaultencoding() 418 try: 419 ostype = subprocess.check_output( 420 ['uname', '-s']).strip().decode(default_encoding) 421 except subprocess.CalledProcessError: 422 return 423 except OSError as reason: 424 if getattr(reason, 'winerror', None) is not None: 425 return 426 raise reason 427 428 if ostype != "Linux": 429 return 430 431 if not os.path.exists("/etc/NIXOS"): 432 return 433 if os.path.exists("/lib"): 434 return 435 436 # At this point we're pretty sure the user is running NixOS 437 nix_os_msg = "info: you seem to be running NixOS. Attempting to patch" 438 print(nix_os_msg, fname) 439 440 try: 441 interpreter = subprocess.check_output( 442 ["patchelf", "--print-interpreter", fname]) 443 interpreter = interpreter.strip().decode(default_encoding) 444 except subprocess.CalledProcessError as reason: 445 print("warning: failed to call patchelf:", reason) 446 return 447 448 loader = interpreter.split("/")[-1] 449 450 try: 451 ldd_output = subprocess.check_output( 452 ['ldd', '/run/current-system/sw/bin/sh']) 453 ldd_output = ldd_output.strip().decode(default_encoding) 454 except subprocess.CalledProcessError as reason: 455 print("warning: unable to call ldd:", reason) 456 return 457 458 for line in ldd_output.splitlines(): 459 libname = line.split()[0] 460 if libname.endswith(loader): 461 loader_path = libname[:len(libname) - len(loader)] 462 break 463 else: 464 print("warning: unable to find the path to the dynamic linker") 465 return 466 467 correct_interpreter = loader_path + loader 468 469 try: 470 subprocess.check_output( 471 ["patchelf", "--set-interpreter", correct_interpreter, fname]) 472 except subprocess.CalledProcessError as reason: 473 print("warning: failed to call patchelf:", reason) 474 return 475 476 def rustc_stamp(self): 477 """Return the path for .rustc-stamp 478 479 >>> rb = RustBuild() 480 >>> rb.build_dir = "build" 481 >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp") 482 True 483 """ 484 return os.path.join(self.bin_root(), '.rustc-stamp') 485 486 def cargo_stamp(self): 487 """Return the path for .cargo-stamp 488 489 >>> rb = RustBuild() 490 >>> rb.build_dir = "build" 491 >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp") 492 True 493 """ 494 return os.path.join(self.bin_root(), '.cargo-stamp') 495 496 def program_out_of_date(self, stamp_path): 497 """Check if the given program stamp is out of date""" 498 if not os.path.exists(stamp_path) or self.clean: 499 return True 500 with open(stamp_path, 'r') as stamp: 501 return self.date != stamp.read() 502 503 def bin_root(self): 504 """Return the binary root directory 505 506 >>> rb = RustBuild() 507 >>> rb.build_dir = "build" 508 >>> rb.bin_root() == os.path.join("build", "stage0") 509 True 510 511 When the 'build' property is given should be a nested directory: 512 513 >>> rb.build = "devel" 514 >>> rb.bin_root() == os.path.join("build", "devel", "stage0") 515 True 516 """ 517 return os.path.join(self.build_dir, self.build, "stage0") 518 519 def get_toml(self, key, section=None): 520 """Returns the value of the given key in config.toml, otherwise returns None 521 522 >>> rb = RustBuild() 523 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"' 524 >>> rb.get_toml("key2") 525 'value2' 526 527 If the key does not exists, the result is None: 528 529 >>> rb.get_toml("key3") is None 530 True 531 532 Optionally also matches the section the key appears in 533 534 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"' 535 >>> rb.get_toml('key', 'a') 536 'value1' 537 >>> rb.get_toml('key', 'b') 538 'value2' 539 >>> rb.get_toml('key', 'c') is None 540 True 541 542 >>> rb.config_toml = 'key1 = true' 543 >>> rb.get_toml("key1") 544 'true' 545 """ 546 547 cur_section = None 548 for line in self.config_toml.splitlines(): 549 section_match = re.match(r'^\s*\[(.*)\]\s*$', line) 550 if section_match is not None: 551 cur_section = section_match.group(1) 552 553 match = re.match(r'^{}\s*=(.*)$'.format(key), line) 554 if match is not None: 555 value = match.group(1) 556 if section is None or section == cur_section: 557 return self.get_string(value) or value.strip() 558 return None 559 560 def cargo(self): 561 """Return config path for cargo""" 562 return self.program_config('cargo') 563 564 def rustc(self): 565 """Return config path for rustc""" 566 return self.program_config('rustc') 567 568 def program_config(self, program): 569 """Return config path for the given program 570 571 >>> rb = RustBuild() 572 >>> rb.config_toml = 'rustc = "rustc"\\n' 573 >>> rb.program_config('rustc') 574 'rustc' 575 >>> rb.config_toml = '' 576 >>> cargo_path = rb.program_config('cargo') 577 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(), 578 ... "bin", "cargo") 579 True 580 """ 581 config = self.get_toml(program) 582 if config: 583 return os.path.expanduser(config) 584 return os.path.join(self.bin_root(), "bin", "{}{}".format( 585 program, self.exe_suffix())) 586 587 @staticmethod 588 def get_string(line): 589 """Return the value between double quotes 590 591 >>> RustBuild.get_string(' "devel" ') 592 'devel' 593 >>> RustBuild.get_string(" 'devel' ") 594 'devel' 595 >>> RustBuild.get_string('devel') is None 596 True 597 >>> RustBuild.get_string(' "devel ') 598 '' 599 """ 600 start = line.find('"') 601 if start != -1: 602 end = start + 1 + line[start + 1:].find('"') 603 return line[start + 1:end] 604 start = line.find('\'') 605 if start != -1: 606 end = start + 1 + line[start + 1:].find('\'') 607 return line[start + 1:end] 608 return None 609 610 @staticmethod 611 def exe_suffix(): 612 """Return a suffix for executables""" 613 if sys.platform == 'win32': 614 return '.exe' 615 return '' 616 617 def bootstrap_binary(self): 618 """Return the path of the bootstrap binary 619 620 >>> rb = RustBuild() 621 >>> rb.build_dir = "build" 622 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap", 623 ... "debug", "bootstrap") 624 True 625 """ 626 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap") 627 628 def build_bootstrap(self): 629 """Build bootstrap""" 630 build_dir = os.path.join(self.build_dir, "bootstrap") 631 if self.clean and os.path.exists(build_dir): 632 shutil.rmtree(build_dir) 633 env = os.environ.copy() 634 env["RUSTC_BOOTSTRAP"] = '1' 635 env["CARGO_TARGET_DIR"] = build_dir 636 env["RUSTC"] = self.rustc() 637 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ 638 (os.pathsep + env["LD_LIBRARY_PATH"]) \ 639 if "LD_LIBRARY_PATH" in env else "" 640 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ 641 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \ 642 if "DYLD_LIBRARY_PATH" in env else "" 643 env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ 644 (os.pathsep + env["LIBRARY_PATH"]) \ 645 if "LIBRARY_PATH" in env else "" 646 # preserve existing RUSTFLAGS 647 env.setdefault("RUSTFLAGS", "") 648 env["RUSTFLAGS"] += " -Cdebuginfo=2" 649 650 build_section = "target.{}".format(self.build_triple()) 651 target_features = [] 652 if self.get_toml("crt-static", build_section) == "true": 653 target_features += ["+crt-static"] 654 elif self.get_toml("crt-static", build_section) == "false": 655 target_features += ["-crt-static"] 656 if target_features: 657 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features)) 658 target_linker = self.get_toml("linker", build_section) 659 if target_linker is not None: 660 env["RUSTFLAGS"] += " -C linker=" + target_linker 661 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes" 662 if self.get_toml("deny-warnings", "rust") != "false": 663 env["RUSTFLAGS"] += " -Dwarnings" 664 665 env["PATH"] = os.path.join(self.bin_root(), "bin") + \ 666 os.pathsep + env["PATH"] 667 if not os.path.isfile(self.cargo()): 668 raise Exception("no cargo executable found at `{}`".format( 669 self.cargo())) 670 args = [self.cargo(), "build", "--manifest-path", 671 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")] 672 for _ in range(1, self.verbose): 673 args.append("--verbose") 674 if self.use_locked_deps: 675 args.append("--locked") 676 if self.use_vendored_sources: 677 args.append("--frozen") 678 run(args, env=env, verbose=self.verbose) 679 680 def build_triple(self): 681 """Build triple as in LLVM""" 682 config = self.get_toml('build') 683 if config: 684 return config 685 return default_build_triple() 686 687 def check_submodule(self, module, slow_submodules): 688 if not slow_submodules: 689 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"], 690 cwd=os.path.join(self.rust_root, module), 691 stdout=subprocess.PIPE) 692 return checked_out 693 else: 694 return None 695 696 def update_submodule(self, module, checked_out, recorded_submodules): 697 module_path = os.path.join(self.rust_root, module) 698 699 if checked_out is not None: 700 default_encoding = sys.getdefaultencoding() 701 checked_out = checked_out.communicate()[0].decode(default_encoding).strip() 702 if recorded_submodules[module] == checked_out: 703 return 704 705 print("Updating submodule", module) 706 707 run(["git", "submodule", "-q", "sync", module], 708 cwd=self.rust_root, verbose=self.verbose) 709 try: 710 run(["git", "submodule", "update", 711 "--init", "--recursive", "--progress", module], 712 cwd=self.rust_root, verbose=self.verbose, exception=True) 713 except RuntimeError: 714 # Some versions of git don't support --progress. 715 run(["git", "submodule", "update", 716 "--init", "--recursive", module], 717 cwd=self.rust_root, verbose=self.verbose) 718 run(["git", "reset", "-q", "--hard"], 719 cwd=module_path, verbose=self.verbose) 720 run(["git", "clean", "-qdfx"], 721 cwd=module_path, verbose=self.verbose) 722 723 def update_submodules(self): 724 """Update submodules""" 725 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \ 726 self.get_toml('submodules') == "false": 727 return 728 729 # check the existence of 'git' command 730 try: 731 subprocess.check_output(['git', '--version']) 732 except (subprocess.CalledProcessError, OSError): 733 print("error: `git` is not found, please make sure it's installed and in the path.") 734 sys.exit(1) 735 736 slow_submodules = self.get_toml('fast-submodules') == "false" 737 start_time = time() 738 if slow_submodules: 739 print('Unconditionally updating all submodules') 740 else: 741 print('Updating only changed submodules') 742 default_encoding = sys.getdefaultencoding() 743 submodules = [s.split(' ', 1)[1] for s in subprocess.check_output( 744 ["git", "config", "--file", 745 os.path.join(self.rust_root, ".gitmodules"), 746 "--get-regexp", "path"] 747 ).decode(default_encoding).splitlines()] 748 filtered_submodules = [] 749 submodules_names = [] 750 for module in submodules: 751 if module.endswith("llvm-project"): 752 if self.get_toml('llvm-config') and self.get_toml('lld') != 'true': 753 continue 754 check = self.check_submodule(module, slow_submodules) 755 filtered_submodules.append((module, check)) 756 submodules_names.append(module) 757 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names, 758 cwd=self.rust_root, stdout=subprocess.PIPE) 759 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines() 760 recorded_submodules = {} 761 for data in recorded: 762 data = data.split() 763 recorded_submodules[data[3]] = data[2] 764 for module in filtered_submodules: 765 self.update_submodule(module[0], module[1], recorded_submodules) 766 print("Submodules updated in %.2f seconds" % (time() - start_time)) 767 768 def set_normal_environment(self): 769 """Set download URL for normal environment""" 770 if 'RUSTUP_DIST_SERVER' in os.environ: 771 self._download_url = os.environ['RUSTUP_DIST_SERVER'] 772 else: 773 self._download_url = 'https://static.rust-lang.org' 774 775 def set_dev_environment(self): 776 """Set download URL for development environment""" 777 if 'RUSTUP_DEV_DIST_SERVER' in os.environ: 778 self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER'] 779 else: 780 self._download_url = 'https://dev-static.rust-lang.org' 781 782 def check_vendored_status(self): 783 """Check that vendoring is configured properly""" 784 vendor_dir = os.path.join(self.rust_root, 'vendor') 785 if 'SUDO_USER' in os.environ and not self.use_vendored_sources: 786 if os.environ.get('USER') != os.environ['SUDO_USER']: 787 self.use_vendored_sources = True 788 print('info: looks like you are running this command under `sudo`') 789 print(' and so in order to preserve your $HOME this will now') 790 print(' use vendored sources by default.') 791 if not os.path.exists(vendor_dir): 792 print('error: vendoring required, but vendor directory does not exist.') 793 print(' Run `cargo vendor` without sudo to initialize the ' 794 'vendor directory.') 795 raise Exception("{} not found".format(vendor_dir)) 796 797 if self.use_vendored_sources: 798 if not os.path.exists('.cargo'): 799 os.makedirs('.cargo') 800 with output('.cargo/config') as cargo_config: 801 cargo_config.write( 802 "[source.crates-io]\n" 803 "replace-with = 'vendored-sources'\n" 804 "registry = 'https://example.com'\n" 805 "\n" 806 "[source.vendored-sources]\n" 807 "directory = '{}/vendor'\n" 808 .format(self.rust_root)) 809 else: 810 if os.path.exists('.cargo'): 811 shutil.rmtree('.cargo') 812 813 def ensure_vendored(self): 814 """Ensure that the vendored sources are available if needed""" 815 vendor_dir = os.path.join(self.rust_root, 'vendor') 816 # Note that this does not handle updating the vendored dependencies if 817 # the rust git repository is updated. Normal development usually does 818 # not use vendoring, so hopefully this isn't too much of a problem. 819 if self.use_vendored_sources and not os.path.exists(vendor_dir): 820 run([self.cargo(), "vendor"], 821 verbose=self.verbose, cwd=self.rust_root) 822 823 824def bootstrap(help_triggered): 825 """Configure, fetch, build and run the initial bootstrap""" 826 827 # If the user is asking for help, let them know that the whole download-and-build 828 # process has to happen before anything is printed out. 829 if help_triggered: 830 print("info: Downloading and building bootstrap before processing --help") 831 print(" command. See src/bootstrap/README.md for help with common") 832 print(" commands.") 833 834 parser = argparse.ArgumentParser(description='Build rust') 835 parser.add_argument('--config') 836 parser.add_argument('--build') 837 parser.add_argument('--src') 838 parser.add_argument('--clean', action='store_true') 839 parser.add_argument('-v', '--verbose', action='count', default=0) 840 841 args = [a for a in sys.argv if a != '-h' and a != '--help'] 842 args, _ = parser.parse_known_args(args) 843 844 # Configure initial bootstrap 845 build = RustBuild() 846 build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..')) 847 build.verbose = args.verbose 848 build.clean = args.clean 849 850 try: 851 with open(args.config or 'config.toml') as config: 852 build.config_toml = config.read() 853 except (OSError, IOError): 854 pass 855 856 config_verbose = build.get_toml('verbose', 'build') 857 if config_verbose is not None: 858 build.verbose = max(build.verbose, int(config_verbose)) 859 860 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true' 861 862 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true' 863 864 build.check_vendored_status() 865 866 data = stage0_data(build.rust_root) 867 build.date = data['date'] 868 build.rustc_channel = data['rustc'] 869 build.cargo_channel = data['cargo'] 870 871 if 'dev' in data: 872 build.set_dev_environment() 873 else: 874 build.set_normal_environment() 875 876 build.update_submodules() 877 878 # Fetch/build the bootstrap 879 build.build = args.build or build.build_triple() 880 build.download_stage0() 881 sys.stdout.flush() 882 build.ensure_vendored() 883 build.build_bootstrap() 884 sys.stdout.flush() 885 886 # Run the bootstrap 887 args = [build.bootstrap_binary()] 888 args.extend(sys.argv[1:]) 889 env = os.environ.copy() 890 env["BUILD"] = build.build 891 env["SRC"] = build.rust_root 892 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) 893 env["BOOTSTRAP_PYTHON"] = sys.executable 894 env["BUILD_DIR"] = build.build_dir 895 env["RUSTC_BOOTSTRAP"] = '1' 896 env["CARGO"] = build.cargo() 897 env["RUSTC"] = build.rustc() 898 run(args, env=env, verbose=build.verbose) 899 900 901def main(): 902 """Entry point for the bootstrap process""" 903 start_time = time() 904 905 # x.py help <cmd> ... 906 if len(sys.argv) > 1 and sys.argv[1] == 'help': 907 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:] 908 909 help_triggered = ( 910 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1) 911 try: 912 bootstrap(help_triggered) 913 if not help_triggered: 914 print("Build completed successfully in {}".format( 915 format_build_time(time() - start_time))) 916 except (SystemExit, KeyboardInterrupt) as error: 917 if hasattr(error, 'code') and isinstance(error.code, int): 918 exit_code = error.code 919 else: 920 exit_code = 1 921 print(error) 922 if not help_triggered: 923 print("Build completed unsuccessfully in {}".format( 924 format_build_time(time() - start_time))) 925 sys.exit(exit_code) 926 927 928if __name__ == '__main__': 929 main()