fork/rust
git clone https://git.julia.blue/fork/rust
4560ea788cb
1.39.0
1from __future__ import absolute_import, division, print_function 2import argparse 3import contextlib 4import datetime 5import hashlib 6import os 7import re 8import shutil 9import subprocess 10import sys 11import tarfile 12import tempfile 13 14from time import time 15 16 17def get(url, path, verbose=False): 18 suffix = '.sha256' 19 sha_url = url + suffix 20 with tempfile.NamedTemporaryFile(delete=False) as temp_file: 21 temp_path = temp_file.name 22 with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file: 23 sha_path = sha_file.name 24 25 try: 26 download(sha_path, sha_url, False, verbose) 27 if os.path.exists(path): 28 if verify(path, sha_path, False): 29 if verbose: 30 print("using already-download file", path) 31 return 32 else: 33 if verbose: 34 print("ignoring already-download file", 35 path, "due to failed verification") 36 os.unlink(path) 37 download(temp_path, url, True, verbose) 38 if not verify(temp_path, sha_path, verbose): 39 raise RuntimeError("failed verification") 40 if verbose: 41 print("moving {} to {}".format(temp_path, path)) 42 shutil.move(temp_path, path) 43 finally: 44 delete_if_present(sha_path, verbose) 45 delete_if_present(temp_path, verbose) 46 47 48def delete_if_present(path, verbose): 49 """Remove the given file if present""" 50 if os.path.isfile(path): 51 if verbose: 52 print("removing", path) 53 os.unlink(path) 54 55 56def download(path, url, probably_big, verbose): 57 for _ in range(0, 4): 58 try: 59 _download(path, url, probably_big, verbose, True) 60 return 61 except RuntimeError: 62 print("\nspurious failure, trying again") 63 _download(path, url, probably_big, verbose, False) 64 65 66def _download(path, url, probably_big, verbose, exception): 67 if probably_big or verbose: 68 print("downloading {}".format(url)) 69 # see http://serverfault.com/questions/301128/how-to-download 70 if sys.platform == 'win32': 71 run(["PowerShell.exe", "/nologo", "-Command", 72 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", 73 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)], 74 verbose=verbose, 75 exception=exception) 76 else: 77 if probably_big or verbose: 78 option = "-#" 79 else: 80 option = "-s" 81 run(["curl", option, 82 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds 83 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds 84 "--retry", "3", "-Sf", "-o", path, url], 85 verbose=verbose, 86 exception=exception) 87 88 89def verify(path, sha_path, verbose): 90 """Check if the sha256 sum of the given path is valid""" 91 if verbose: 92 print("verifying", path) 93 with open(path, "rb") as source: 94 found = hashlib.sha256(source.read()).hexdigest() 95 with open(sha_path, "r") as sha256sum: 96 expected = sha256sum.readline().split()[0] 97 verified = found == expected 98 if not verified: 99 print("invalid checksum:\n" 100 " found: {}\n" 101 " expected: {}".format(found, expected)) 102 return verified 103 104 105def unpack(tarball, dst, verbose=False, match=None): 106 """Unpack the given tarball file""" 107 print("extracting", tarball) 108 fname = os.path.basename(tarball).replace(".tar.gz", "") 109 with contextlib.closing(tarfile.open(tarball)) as tar: 110 for member in tar.getnames(): 111 if "/" not in member: 112 continue 113 name = member.replace(fname + "/", "", 1) 114 if match is not None and not name.startswith(match): 115 continue 116 name = name[len(match) + 1:] 117 118 dst_path = os.path.join(dst, name) 119 if verbose: 120 print(" extracting", member) 121 tar.extract(member, dst) 122 src_path = os.path.join(dst, member) 123 if os.path.isdir(src_path) and os.path.exists(dst_path): 124 continue 125 shutil.move(src_path, dst_path) 126 shutil.rmtree(os.path.join(dst, fname)) 127 128 129def run(args, verbose=False, exception=False, **kwargs): 130 """Run a child program in a new process""" 131 if verbose: 132 print("running: " + ' '.join(args)) 133 sys.stdout.flush() 134 # Use Popen here instead of call() as it apparently allows powershell on 135 # Windows to not lock up waiting for input presumably. 136 ret = subprocess.Popen(args, **kwargs) 137 code = ret.wait() 138 if code != 0: 139 err = "failed to run: " + ' '.join(args) 140 if verbose or exception: 141 raise RuntimeError(err) 142 sys.exit(err) 143 144 145def stage0_data(rust_root): 146 """Build a dictionary from stage0.txt""" 147 nightlies = os.path.join(rust_root, "src/stage0.txt") 148 with open(nightlies, 'r') as nightlies: 149 lines = [line.rstrip() for line in nightlies 150 if not line.startswith("#")] 151 return dict([line.split(": ", 1) for line in lines if line]) 152 153 154def format_build_time(duration): 155 """Return a nicer format for build time 156 157 >>> format_build_time('300') 158 '0:05:00' 159 """ 160 return str(datetime.timedelta(seconds=int(duration))) 161 162 163def default_build_triple(): 164 """Build triple as in LLVM""" 165 default_encoding = sys.getdefaultencoding() 166 try: 167 ostype = subprocess.check_output( 168 ['uname', '-s']).strip().decode(default_encoding) 169 cputype = subprocess.check_output( 170 ['uname', '-m']).strip().decode(default_encoding) 171 except (subprocess.CalledProcessError, OSError): 172 if sys.platform == 'win32': 173 return 'x86_64-pc-windows-msvc' 174 err = "uname not found" 175 sys.exit(err) 176 177 # The goal here is to come up with the same triple as LLVM would, 178 # at least for the subset of platforms we're willing to target. 179 ostype_mapper = { 180 'Darwin': 'apple-darwin', 181 'DragonFly': 'unknown-dragonfly', 182 'FreeBSD': 'unknown-freebsd', 183 'Haiku': 'unknown-haiku', 184 'NetBSD': 'unknown-netbsd', 185 'OpenBSD': 'unknown-openbsd' 186 } 187 188 # Consider the direct transformation first and then the special cases 189 if ostype in ostype_mapper: 190 ostype = ostype_mapper[ostype] 191 elif ostype == 'Linux': 192 os_from_sp = subprocess.check_output( 193 ['uname', '-o']).strip().decode(default_encoding) 194 if os_from_sp == 'Android': 195 ostype = 'linux-android' 196 else: 197 ostype = 'unknown-linux-gnu' 198 elif ostype == 'SunOS': 199 ostype = 'sun-solaris' 200 # On Solaris, uname -m will return a machine classification instead 201 # of a cpu type, so uname -p is recommended instead. However, the 202 # output from that option is too generic for our purposes (it will 203 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k 204 # must be used instead. 205 try: 206 cputype = subprocess.check_output( 207 ['isainfo', '-k']).strip().decode(default_encoding) 208 except (subprocess.CalledProcessError, OSError): 209 err = "isainfo not found" 210 sys.exit(err) 211 elif ostype.startswith('MINGW'): 212 # msys' `uname` does not print gcc configuration, but prints msys 213 # configuration. so we cannot believe `uname -m`: 214 # msys1 is always i686 and msys2 is always x86_64. 215 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and 216 # MINGW64 on x86_64. 217 ostype = 'pc-windows-gnu' 218 cputype = 'i686' 219 if os.environ.get('MSYSTEM') == 'MINGW64': 220 cputype = 'x86_64' 221 elif ostype.startswith('MSYS'): 222 ostype = 'pc-windows-gnu' 223 elif ostype.startswith('CYGWIN_NT'): 224 cputype = 'i686' 225 if ostype.endswith('WOW64'): 226 cputype = 'x86_64' 227 ostype = 'pc-windows-gnu' 228 else: 229 err = "unknown OS type: {}".format(ostype) 230 sys.exit(err) 231 232 if cputype == 'powerpc' and ostype == 'unknown-freebsd': 233 cputype = subprocess.check_output( 234 ['uname', '-p']).strip().decode(default_encoding) 235 cputype_mapper = { 236 'BePC': 'i686', 237 'aarch64': 'aarch64', 238 'amd64': 'x86_64', 239 'arm64': 'aarch64', 240 'i386': 'i686', 241 'i486': 'i686', 242 'i686': 'i686', 243 'i786': 'i686', 244 'powerpc': 'powerpc', 245 'powerpc64': 'powerpc64', 246 'powerpc64le': 'powerpc64le', 247 'ppc': 'powerpc', 248 'ppc64': 'powerpc64', 249 'ppc64le': 'powerpc64le', 250 's390x': 's390x', 251 'x64': 'x86_64', 252 'x86': 'i686', 253 'x86-64': 'x86_64', 254 'x86_64': 'x86_64' 255 } 256 257 # Consider the direct transformation first and then the special cases 258 if cputype in cputype_mapper: 259 cputype = cputype_mapper[cputype] 260 elif cputype in {'xscale', 'arm'}: 261 cputype = 'arm' 262 if ostype == 'linux-android': 263 ostype = 'linux-androideabi' 264 elif ostype == 'unknown-freebsd': 265 cputype = subprocess.check_output( 266 ['uname', '-p']).strip().decode(default_encoding) 267 ostype = 'unknown-freebsd' 268 elif cputype == 'armv6l': 269 cputype = 'arm' 270 if ostype == 'linux-android': 271 ostype = 'linux-androideabi' 272 else: 273 ostype += 'eabihf' 274 elif cputype in {'armv7l', 'armv8l'}: 275 cputype = 'armv7' 276 if ostype == 'linux-android': 277 ostype = 'linux-androideabi' 278 else: 279 ostype += 'eabihf' 280 elif cputype == 'mips': 281 if sys.byteorder == 'big': 282 cputype = 'mips' 283 elif sys.byteorder == 'little': 284 cputype = 'mipsel' 285 else: 286 raise ValueError("unknown byteorder: {}".format(sys.byteorder)) 287 elif cputype == 'mips64': 288 if sys.byteorder == 'big': 289 cputype = 'mips64' 290 elif sys.byteorder == 'little': 291 cputype = 'mips64el' 292 else: 293 raise ValueError('unknown byteorder: {}'.format(sys.byteorder)) 294 # only the n64 ABI is supported, indicate it 295 ostype += 'abi64' 296 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64': 297 pass 298 else: 299 err = "unknown cpu type: {}".format(cputype) 300 sys.exit(err) 301 302 return "{}-{}".format(cputype, ostype) 303 304 305@contextlib.contextmanager 306def output(filepath): 307 tmp = filepath + '.tmp' 308 with open(tmp, 'w') as f: 309 yield f 310 try: 311 os.remove(filepath) # PermissionError/OSError on Win32 if in use 312 os.rename(tmp, filepath) 313 except OSError: 314 shutil.copy2(tmp, filepath) 315 os.remove(tmp) 316 317 318class RustBuild(object): 319 """Provide all the methods required to build Rust""" 320 def __init__(self): 321 self.cargo_channel = '' 322 self.date = '' 323 self._download_url = '' 324 self.rustc_channel = '' 325 self.build = '' 326 self.build_dir = os.path.join(os.getcwd(), "build") 327 self.clean = False 328 self.config_toml = '' 329 self.rust_root = '' 330 self.use_locked_deps = '' 331 self.use_vendored_sources = '' 332 self.verbose = False 333 334 def download_stage0(self): 335 """Fetch the build system for Rust, written in Rust 336 337 This method will build a cache directory, then it will fetch the 338 tarball which has the stage0 compiler used to then bootstrap the Rust 339 compiler itself. 340 341 Each downloaded tarball is extracted, after that, the script 342 will move all the content to the right place. 343 """ 344 rustc_channel = self.rustc_channel 345 cargo_channel = self.cargo_channel 346 347 if self.rustc().startswith(self.bin_root()) and \ 348 (not os.path.exists(self.rustc()) or 349 self.program_out_of_date(self.rustc_stamp())): 350 if os.path.exists(self.bin_root()): 351 shutil.rmtree(self.bin_root()) 352 filename = "rust-std-{}-{}.tar.gz".format( 353 rustc_channel, self.build) 354 pattern = "rust-std-{}".format(self.build) 355 self._download_stage0_helper(filename, pattern) 356 357 filename = "rustc-{}-{}.tar.gz".format(rustc_channel, self.build) 358 self._download_stage0_helper(filename, "rustc") 359 self.fix_executable("{}/bin/rustc".format(self.bin_root())) 360 self.fix_executable("{}/bin/rustdoc".format(self.bin_root())) 361 with output(self.rustc_stamp()) as rust_stamp: 362 rust_stamp.write(self.date) 363 364 # This is required so that we don't mix incompatible MinGW 365 # libraries/binaries that are included in rust-std with 366 # the system MinGW ones. 367 if "pc-windows-gnu" in self.build: 368 filename = "rust-mingw-{}-{}.tar.gz".format( 369 rustc_channel, self.build) 370 self._download_stage0_helper(filename, "rust-mingw") 371 372 if self.cargo().startswith(self.bin_root()) and \ 373 (not os.path.exists(self.cargo()) or 374 self.program_out_of_date(self.cargo_stamp())): 375 filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build) 376 self._download_stage0_helper(filename, "cargo") 377 self.fix_executable("{}/bin/cargo".format(self.bin_root())) 378 with output(self.cargo_stamp()) as cargo_stamp: 379 cargo_stamp.write(self.date) 380 381 def _download_stage0_helper(self, filename, pattern): 382 cache_dst = os.path.join(self.build_dir, "cache") 383 rustc_cache = os.path.join(cache_dst, self.date) 384 if not os.path.exists(rustc_cache): 385 os.makedirs(rustc_cache) 386 387 url = "{}/dist/{}".format(self._download_url, self.date) 388 tarball = os.path.join(rustc_cache, filename) 389 if not os.path.exists(tarball): 390 get("{}/{}".format(url, filename), tarball, verbose=self.verbose) 391 unpack(tarball, self.bin_root(), match=pattern, verbose=self.verbose) 392 393 @staticmethod 394 def fix_executable(fname): 395 """Modifies the interpreter section of 'fname' to fix the dynamic linker 396 397 This method is only required on NixOS and uses the PatchELF utility to 398 change the dynamic linker of ELF executables. 399 400 Please see https://nixos.org/patchelf.html for more information 401 """ 402 default_encoding = sys.getdefaultencoding() 403 try: 404 ostype = subprocess.check_output( 405 ['uname', '-s']).strip().decode(default_encoding) 406 except subprocess.CalledProcessError: 407 return 408 except OSError as reason: 409 if getattr(reason, 'winerror', None) is not None: 410 return 411 raise reason 412 413 if ostype != "Linux": 414 return 415 416 if not os.path.exists("/etc/NIXOS"): 417 return 418 if os.path.exists("/lib"): 419 return 420 421 # At this point we're pretty sure the user is running NixOS 422 nix_os_msg = "info: you seem to be running NixOS. Attempting to patch" 423 print(nix_os_msg, fname) 424 425 try: 426 interpreter = subprocess.check_output( 427 ["patchelf", "--print-interpreter", fname]) 428 interpreter = interpreter.strip().decode(default_encoding) 429 except subprocess.CalledProcessError as reason: 430 print("warning: failed to call patchelf:", reason) 431 return 432 433 loader = interpreter.split("/")[-1] 434 435 try: 436 ldd_output = subprocess.check_output( 437 ['ldd', '/run/current-system/sw/bin/sh']) 438 ldd_output = ldd_output.strip().decode(default_encoding) 439 except subprocess.CalledProcessError as reason: 440 print("warning: unable to call ldd:", reason) 441 return 442 443 for line in ldd_output.splitlines(): 444 libname = line.split()[0] 445 if libname.endswith(loader): 446 loader_path = libname[:len(libname) - len(loader)] 447 break 448 else: 449 print("warning: unable to find the path to the dynamic linker") 450 return 451 452 correct_interpreter = loader_path + loader 453 454 try: 455 subprocess.check_output( 456 ["patchelf", "--set-interpreter", correct_interpreter, fname]) 457 except subprocess.CalledProcessError as reason: 458 print("warning: failed to call patchelf:", reason) 459 return 460 461 def rustc_stamp(self): 462 """Return the path for .rustc-stamp 463 464 >>> rb = RustBuild() 465 >>> rb.build_dir = "build" 466 >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp") 467 True 468 """ 469 return os.path.join(self.bin_root(), '.rustc-stamp') 470 471 def cargo_stamp(self): 472 """Return the path for .cargo-stamp 473 474 >>> rb = RustBuild() 475 >>> rb.build_dir = "build" 476 >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp") 477 True 478 """ 479 return os.path.join(self.bin_root(), '.cargo-stamp') 480 481 def program_out_of_date(self, stamp_path): 482 """Check if the given program stamp is out of date""" 483 if not os.path.exists(stamp_path) or self.clean: 484 return True 485 with open(stamp_path, 'r') as stamp: 486 return self.date != stamp.read() 487 488 def bin_root(self): 489 """Return the binary root directory 490 491 >>> rb = RustBuild() 492 >>> rb.build_dir = "build" 493 >>> rb.bin_root() == os.path.join("build", "stage0") 494 True 495 496 When the 'build' property is given should be a nested directory: 497 498 >>> rb.build = "devel" 499 >>> rb.bin_root() == os.path.join("build", "devel", "stage0") 500 True 501 """ 502 return os.path.join(self.build_dir, self.build, "stage0") 503 504 def get_toml(self, key, section=None): 505 """Returns the value of the given key in config.toml, otherwise returns None 506 507 >>> rb = RustBuild() 508 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"' 509 >>> rb.get_toml("key2") 510 'value2' 511 512 If the key does not exists, the result is None: 513 514 >>> rb.get_toml("key3") is None 515 True 516 517 Optionally also matches the section the key appears in 518 519 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"' 520 >>> rb.get_toml('key', 'a') 521 'value1' 522 >>> rb.get_toml('key', 'b') 523 'value2' 524 >>> rb.get_toml('key', 'c') is None 525 True 526 527 >>> rb.config_toml = 'key1 = true' 528 >>> rb.get_toml("key1") 529 'true' 530 """ 531 532 cur_section = None 533 for line in self.config_toml.splitlines(): 534 section_match = re.match(r'^\s*\[(.*)\]\s*$', line) 535 if section_match is not None: 536 cur_section = section_match.group(1) 537 538 match = re.match(r'^{}\s*=(.*)$'.format(key), line) 539 if match is not None: 540 value = match.group(1) 541 if section is None or section == cur_section: 542 return self.get_string(value) or value.strip() 543 return None 544 545 def cargo(self): 546 """Return config path for cargo""" 547 return self.program_config('cargo') 548 549 def rustc(self): 550 """Return config path for rustc""" 551 return self.program_config('rustc') 552 553 def program_config(self, program): 554 """Return config path for the given program 555 556 >>> rb = RustBuild() 557 >>> rb.config_toml = 'rustc = "rustc"\\n' 558 >>> rb.program_config('rustc') 559 'rustc' 560 >>> rb.config_toml = '' 561 >>> cargo_path = rb.program_config('cargo') 562 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(), 563 ... "bin", "cargo") 564 True 565 """ 566 config = self.get_toml(program) 567 if config: 568 return os.path.expanduser(config) 569 return os.path.join(self.bin_root(), "bin", "{}{}".format( 570 program, self.exe_suffix())) 571 572 @staticmethod 573 def get_string(line): 574 """Return the value between double quotes 575 576 >>> RustBuild.get_string(' "devel" ') 577 'devel' 578 >>> RustBuild.get_string(" 'devel' ") 579 'devel' 580 >>> RustBuild.get_string('devel') is None 581 True 582 >>> RustBuild.get_string(' "devel ') 583 '' 584 """ 585 start = line.find('"') 586 if start != -1: 587 end = start + 1 + line[start + 1:].find('"') 588 return line[start + 1:end] 589 start = line.find('\'') 590 if start != -1: 591 end = start + 1 + line[start + 1:].find('\'') 592 return line[start + 1:end] 593 return None 594 595 @staticmethod 596 def exe_suffix(): 597 """Return a suffix for executables""" 598 if sys.platform == 'win32': 599 return '.exe' 600 return '' 601 602 def bootstrap_binary(self): 603 """Return the path of the bootstrap binary 604 605 >>> rb = RustBuild() 606 >>> rb.build_dir = "build" 607 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap", 608 ... "debug", "bootstrap") 609 True 610 """ 611 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap") 612 613 def build_bootstrap(self): 614 """Build bootstrap""" 615 build_dir = os.path.join(self.build_dir, "bootstrap") 616 if self.clean and os.path.exists(build_dir): 617 shutil.rmtree(build_dir) 618 env = os.environ.copy() 619 env["RUSTC_BOOTSTRAP"] = '1' 620 env["CARGO_TARGET_DIR"] = build_dir 621 env["RUSTC"] = self.rustc() 622 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ 623 (os.pathsep + env["LD_LIBRARY_PATH"]) \ 624 if "LD_LIBRARY_PATH" in env else "" 625 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ 626 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \ 627 if "DYLD_LIBRARY_PATH" in env else "" 628 env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ 629 (os.pathsep + env["LIBRARY_PATH"]) \ 630 if "LIBRARY_PATH" in env else "" 631 env["RUSTFLAGS"] = "-Cdebuginfo=2 " 632 633 build_section = "target.{}".format(self.build_triple()) 634 target_features = [] 635 if self.get_toml("crt-static", build_section) == "true": 636 target_features += ["+crt-static"] 637 elif self.get_toml("crt-static", build_section) == "false": 638 target_features += ["-crt-static"] 639 if target_features: 640 env["RUSTFLAGS"] += "-C target-feature=" + (",".join(target_features)) + " " 641 target_linker = self.get_toml("linker", build_section) 642 if target_linker is not None: 643 env["RUSTFLAGS"] += "-C linker=" + target_linker + " " 644 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes " 645 if self.get_toml("deny-warnings", "rust") != "false": 646 env["RUSTFLAGS"] += "-Dwarnings " 647 648 env["PATH"] = os.path.join(self.bin_root(), "bin") + \ 649 os.pathsep + env["PATH"] 650 if not os.path.isfile(self.cargo()): 651 raise Exception("no cargo executable found at `{}`".format( 652 self.cargo())) 653 args = [self.cargo(), "build", "--manifest-path", 654 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")] 655 for _ in range(1, self.verbose): 656 args.append("--verbose") 657 if self.use_locked_deps: 658 args.append("--locked") 659 if self.use_vendored_sources: 660 args.append("--frozen") 661 run(args, env=env, verbose=self.verbose) 662 663 def build_triple(self): 664 """Build triple as in LLVM""" 665 config = self.get_toml('build') 666 if config: 667 return config 668 return default_build_triple() 669 670 def check_submodule(self, module, slow_submodules): 671 if not slow_submodules: 672 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"], 673 cwd=os.path.join(self.rust_root, module), 674 stdout=subprocess.PIPE) 675 return checked_out 676 else: 677 return None 678 679 def update_submodule(self, module, checked_out, recorded_submodules): 680 module_path = os.path.join(self.rust_root, module) 681 682 if checked_out is not None: 683 default_encoding = sys.getdefaultencoding() 684 checked_out = checked_out.communicate()[0].decode(default_encoding).strip() 685 if recorded_submodules[module] == checked_out: 686 return 687 688 print("Updating submodule", module) 689 690 run(["git", "submodule", "-q", "sync", module], 691 cwd=self.rust_root, verbose=self.verbose) 692 try: 693 run(["git", "submodule", "update", 694 "--init", "--recursive", "--progress", module], 695 cwd=self.rust_root, verbose=self.verbose, exception=True) 696 except RuntimeError: 697 # Some versions of git don't support --progress. 698 run(["git", "submodule", "update", 699 "--init", "--recursive", module], 700 cwd=self.rust_root, verbose=self.verbose) 701 run(["git", "reset", "-q", "--hard"], 702 cwd=module_path, verbose=self.verbose) 703 run(["git", "clean", "-qdfx"], 704 cwd=module_path, verbose=self.verbose) 705 706 def update_submodules(self): 707 """Update submodules""" 708 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \ 709 self.get_toml('submodules') == "false": 710 return 711 712 # check the existence of 'git' command 713 try: 714 subprocess.check_output(['git', '--version']) 715 except (subprocess.CalledProcessError, OSError): 716 print("error: `git` is not found, please make sure it's installed and in the path.") 717 sys.exit(1) 718 719 slow_submodules = self.get_toml('fast-submodules') == "false" 720 start_time = time() 721 if slow_submodules: 722 print('Unconditionally updating all submodules') 723 else: 724 print('Updating only changed submodules') 725 default_encoding = sys.getdefaultencoding() 726 submodules = [s.split(' ', 1)[1] for s in subprocess.check_output( 727 ["git", "config", "--file", 728 os.path.join(self.rust_root, ".gitmodules"), 729 "--get-regexp", "path"] 730 ).decode(default_encoding).splitlines()] 731 filtered_submodules = [] 732 submodules_names = [] 733 for module in submodules: 734 if module.endswith("llvm-project"): 735 if self.get_toml('llvm-config') and self.get_toml('lld') != 'true': 736 continue 737 if module.endswith("llvm-emscripten"): 738 backends = self.get_toml('codegen-backends') 739 if backends is None or not 'emscripten' in backends: 740 continue 741 check = self.check_submodule(module, slow_submodules) 742 filtered_submodules.append((module, check)) 743 submodules_names.append(module) 744 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names, 745 cwd=self.rust_root, stdout=subprocess.PIPE) 746 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines() 747 recorded_submodules = {} 748 for data in recorded: 749 data = data.split() 750 recorded_submodules[data[3]] = data[2] 751 for module in filtered_submodules: 752 self.update_submodule(module[0], module[1], recorded_submodules) 753 print("Submodules updated in %.2f seconds" % (time() - start_time)) 754 755 def set_normal_environment(self): 756 """Set download URL for normal environment""" 757 if 'RUSTUP_DIST_SERVER' in os.environ: 758 self._download_url = os.environ['RUSTUP_DIST_SERVER'] 759 else: 760 self._download_url = 'https://static.rust-lang.org' 761 762 def set_dev_environment(self): 763 """Set download URL for development environment""" 764 if 'RUSTUP_DEV_DIST_SERVER' in os.environ: 765 self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER'] 766 else: 767 self._download_url = 'https://dev-static.rust-lang.org' 768 769 def check_vendored_status(self): 770 """Check that vendoring is configured properly""" 771 vendor_dir = os.path.join(self.rust_root, 'vendor') 772 if 'SUDO_USER' in os.environ and not self.use_vendored_sources: 773 if os.environ.get('USER') != os.environ['SUDO_USER']: 774 self.use_vendored_sources = True 775 print('info: looks like you are running this command under `sudo`') 776 print(' and so in order to preserve your $HOME this will now') 777 print(' use vendored sources by default.') 778 if not os.path.exists(vendor_dir): 779 print('error: vendoring required, but vendor directory does not exist.') 780 print(' Run `cargo vendor` without sudo to initialize the ' 781 'vendor directory.') 782 raise Exception("{} not found".format(vendor_dir)) 783 784 if self.use_vendored_sources: 785 if not os.path.exists('.cargo'): 786 os.makedirs('.cargo') 787 with output('.cargo/config') as cargo_config: 788 cargo_config.write( 789 "[source.crates-io]\n" 790 "replace-with = 'vendored-sources'\n" 791 "registry = 'https://example.com'\n" 792 "\n" 793 "[source.vendored-sources]\n" 794 "directory = '{}/vendor'\n" 795 .format(self.rust_root)) 796 else: 797 if os.path.exists('.cargo'): 798 shutil.rmtree('.cargo') 799 800 def ensure_vendored(self): 801 """Ensure that the vendored sources are available if needed""" 802 vendor_dir = os.path.join(self.rust_root, 'vendor') 803 # Note that this does not handle updating the vendored dependencies if 804 # the rust git repository is updated. Normal development usually does 805 # not use vendoring, so hopefully this isn't too much of a problem. 806 if self.use_vendored_sources and not os.path.exists(vendor_dir): 807 run([self.cargo(), "vendor"], 808 verbose=self.verbose, cwd=self.rust_root) 809 810 811def bootstrap(help_triggered): 812 """Configure, fetch, build and run the initial bootstrap""" 813 814 # If the user is asking for help, let them know that the whole download-and-build 815 # process has to happen before anything is printed out. 816 if help_triggered: 817 print("info: Downloading and building bootstrap before processing --help") 818 print(" command. See src/bootstrap/README.md for help with common") 819 print(" commands.") 820 821 parser = argparse.ArgumentParser(description='Build rust') 822 parser.add_argument('--config') 823 parser.add_argument('--build') 824 parser.add_argument('--src') 825 parser.add_argument('--clean', action='store_true') 826 parser.add_argument('-v', '--verbose', action='count', default=0) 827 828 args = [a for a in sys.argv if a != '-h' and a != '--help'] 829 args, _ = parser.parse_known_args(args) 830 831 # Configure initial bootstrap 832 build = RustBuild() 833 build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..')) 834 build.verbose = args.verbose 835 build.clean = args.clean 836 837 try: 838 with open(args.config or 'config.toml') as config: 839 build.config_toml = config.read() 840 except (OSError, IOError): 841 pass 842 843 config_verbose = build.get_toml('verbose', 'build') 844 if config_verbose is not None: 845 build.verbose = max(build.verbose, int(config_verbose)) 846 847 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true' 848 849 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true' 850 851 build.check_vendored_status() 852 853 data = stage0_data(build.rust_root) 854 build.date = data['date'] 855 build.rustc_channel = data['rustc'] 856 build.cargo_channel = data['cargo'] 857 858 if 'dev' in data: 859 build.set_dev_environment() 860 else: 861 build.set_normal_environment() 862 863 build.update_submodules() 864 865 # Fetch/build the bootstrap 866 build.build = args.build or build.build_triple() 867 build.download_stage0() 868 sys.stdout.flush() 869 build.ensure_vendored() 870 build.build_bootstrap() 871 sys.stdout.flush() 872 873 # Run the bootstrap 874 args = [build.bootstrap_binary()] 875 args.extend(sys.argv[1:]) 876 env = os.environ.copy() 877 env["BUILD"] = build.build 878 env["SRC"] = build.rust_root 879 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) 880 env["BOOTSTRAP_PYTHON"] = sys.executable 881 env["BUILD_DIR"] = build.build_dir 882 env["RUSTC_BOOTSTRAP"] = '1' 883 env["CARGO"] = build.cargo() 884 env["RUSTC"] = build.rustc() 885 run(args, env=env, verbose=build.verbose) 886 887 888def main(): 889 """Entry point for the bootstrap process""" 890 start_time = time() 891 892 # x.py help <cmd> ... 893 if len(sys.argv) > 1 and sys.argv[1] == 'help': 894 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:] 895 896 help_triggered = ( 897 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1) 898 try: 899 bootstrap(help_triggered) 900 if not help_triggered: 901 print("Build completed successfully in {}".format( 902 format_build_time(time() - start_time))) 903 except (SystemExit, KeyboardInterrupt) as error: 904 if hasattr(error, 'code') and isinstance(error.code, int): 905 exit_code = error.code 906 else: 907 exit_code = 1 908 print(error) 909 if not help_triggered: 910 print("Build completed unsuccessfully in {}".format( 911 format_build_time(time() - start_time))) 912 sys.exit(exit_code) 913 914 915if __name__ == '__main__': 916 main()