fork/rust
git clone https://git.julia.blue/fork/rust
49cae55760d
1.44.0
1from __future__ import absolute_import, division, print_function 2import argparse 3import contextlib 4import datetime 5import hashlib 6import os 7import re 8import shutil 9import subprocess 10import sys 11import tarfile 12import tempfile 13 14from time import time 15 16 17def get(url, path, verbose=False): 18 suffix = '.sha256' 19 sha_url = url + suffix 20 with tempfile.NamedTemporaryFile(delete=False) as temp_file: 21 temp_path = temp_file.name 22 with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file: 23 sha_path = sha_file.name 24 25 try: 26 download(sha_path, sha_url, False, verbose) 27 if os.path.exists(path): 28 if verify(path, sha_path, False): 29 if verbose: 30 print("using already-download file", path) 31 return 32 else: 33 if verbose: 34 print("ignoring already-download file", 35 path, "due to failed verification") 36 os.unlink(path) 37 download(temp_path, url, True, verbose) 38 if not verify(temp_path, sha_path, verbose): 39 raise RuntimeError("failed verification") 40 if verbose: 41 print("moving {} to {}".format(temp_path, path)) 42 shutil.move(temp_path, path) 43 finally: 44 delete_if_present(sha_path, verbose) 45 delete_if_present(temp_path, verbose) 46 47 48def delete_if_present(path, verbose): 49 """Remove the given file if present""" 50 if os.path.isfile(path): 51 if verbose: 52 print("removing", path) 53 os.unlink(path) 54 55 56def download(path, url, probably_big, verbose): 57 for _ in range(0, 4): 58 try: 59 _download(path, url, probably_big, verbose, True) 60 return 61 except RuntimeError: 62 print("\nspurious failure, trying again") 63 _download(path, url, probably_big, verbose, False) 64 65 66def _download(path, url, probably_big, verbose, exception): 67 if probably_big or verbose: 68 print("downloading {}".format(url)) 69 # see http://serverfault.com/questions/301128/how-to-download 70 if sys.platform == 'win32': 71 run(["PowerShell.exe", "/nologo", "-Command", 72 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", 73 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)], 74 verbose=verbose, 75 exception=exception) 76 else: 77 if probably_big or verbose: 78 option = "-#" 79 else: 80 option = "-s" 81 run(["curl", option, 82 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds 83 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds 84 "--retry", "3", "-Sf", "-o", path, url], 85 verbose=verbose, 86 exception=exception) 87 88 89def verify(path, sha_path, verbose): 90 """Check if the sha256 sum of the given path is valid""" 91 if verbose: 92 print("verifying", path) 93 with open(path, "rb") as source: 94 found = hashlib.sha256(source.read()).hexdigest() 95 with open(sha_path, "r") as sha256sum: 96 expected = sha256sum.readline().split()[0] 97 verified = found == expected 98 if not verified: 99 print("invalid checksum:\n" 100 " found: {}\n" 101 " expected: {}".format(found, expected)) 102 return verified 103 104 105def unpack(tarball, tarball_suffix, dst, verbose=False, match=None): 106 """Unpack the given tarball file""" 107 print("extracting", tarball) 108 fname = os.path.basename(tarball).replace(tarball_suffix, "") 109 with contextlib.closing(tarfile.open(tarball)) as tar: 110 for member in tar.getnames(): 111 if "/" not in member: 112 continue 113 name = member.replace(fname + "/", "", 1) 114 if match is not None and not name.startswith(match): 115 continue 116 name = name[len(match) + 1:] 117 118 dst_path = os.path.join(dst, name) 119 if verbose: 120 print(" extracting", member) 121 tar.extract(member, dst) 122 src_path = os.path.join(dst, member) 123 if os.path.isdir(src_path) and os.path.exists(dst_path): 124 continue 125 shutil.move(src_path, dst_path) 126 shutil.rmtree(os.path.join(dst, fname)) 127 128 129def run(args, verbose=False, exception=False, **kwargs): 130 """Run a child program in a new process""" 131 if verbose: 132 print("running: " + ' '.join(args)) 133 sys.stdout.flush() 134 # Use Popen here instead of call() as it apparently allows powershell on 135 # Windows to not lock up waiting for input presumably. 136 ret = subprocess.Popen(args, **kwargs) 137 code = ret.wait() 138 if code != 0: 139 err = "failed to run: " + ' '.join(args) 140 if verbose or exception: 141 raise RuntimeError(err) 142 sys.exit(err) 143 144 145def stage0_data(rust_root): 146 """Build a dictionary from stage0.txt""" 147 nightlies = os.path.join(rust_root, "src/stage0.txt") 148 with open(nightlies, 'r') as nightlies: 149 lines = [line.rstrip() for line in nightlies 150 if not line.startswith("#")] 151 return dict([line.split(": ", 1) for line in lines if line]) 152 153 154def format_build_time(duration): 155 """Return a nicer format for build time 156 157 >>> format_build_time('300') 158 '0:05:00' 159 """ 160 return str(datetime.timedelta(seconds=int(duration))) 161 162 163def default_build_triple(): 164 """Build triple as in LLVM""" 165 default_encoding = sys.getdefaultencoding() 166 try: 167 ostype = subprocess.check_output( 168 ['uname', '-s']).strip().decode(default_encoding) 169 cputype = subprocess.check_output( 170 ['uname', '-m']).strip().decode(default_encoding) 171 except (subprocess.CalledProcessError, OSError): 172 if sys.platform == 'win32': 173 return 'x86_64-pc-windows-msvc' 174 err = "uname not found" 175 sys.exit(err) 176 177 # The goal here is to come up with the same triple as LLVM would, 178 # at least for the subset of platforms we're willing to target. 179 ostype_mapper = { 180 'Darwin': 'apple-darwin', 181 'DragonFly': 'unknown-dragonfly', 182 'FreeBSD': 'unknown-freebsd', 183 'Haiku': 'unknown-haiku', 184 'NetBSD': 'unknown-netbsd', 185 'OpenBSD': 'unknown-openbsd' 186 } 187 188 # Consider the direct transformation first and then the special cases 189 if ostype in ostype_mapper: 190 ostype = ostype_mapper[ostype] 191 elif ostype == 'Linux': 192 os_from_sp = subprocess.check_output( 193 ['uname', '-o']).strip().decode(default_encoding) 194 if os_from_sp == 'Android': 195 ostype = 'linux-android' 196 else: 197 ostype = 'unknown-linux-gnu' 198 elif ostype == 'SunOS': 199 ostype = 'sun-solaris' 200 # On Solaris, uname -m will return a machine classification instead 201 # of a cpu type, so uname -p is recommended instead. However, the 202 # output from that option is too generic for our purposes (it will 203 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k 204 # must be used instead. 205 try: 206 cputype = subprocess.check_output( 207 ['isainfo', '-k']).strip().decode(default_encoding) 208 except (subprocess.CalledProcessError, OSError): 209 err = "isainfo not found" 210 sys.exit(err) 211 elif ostype.startswith('MINGW'): 212 # msys' `uname` does not print gcc configuration, but prints msys 213 # configuration. so we cannot believe `uname -m`: 214 # msys1 is always i686 and msys2 is always x86_64. 215 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and 216 # MINGW64 on x86_64. 217 ostype = 'pc-windows-gnu' 218 cputype = 'i686' 219 if os.environ.get('MSYSTEM') == 'MINGW64': 220 cputype = 'x86_64' 221 elif ostype.startswith('MSYS'): 222 ostype = 'pc-windows-gnu' 223 elif ostype.startswith('CYGWIN_NT'): 224 cputype = 'i686' 225 if ostype.endswith('WOW64'): 226 cputype = 'x86_64' 227 ostype = 'pc-windows-gnu' 228 else: 229 err = "unknown OS type: {}".format(ostype) 230 sys.exit(err) 231 232 if cputype == 'powerpc' and ostype == 'unknown-freebsd': 233 cputype = subprocess.check_output( 234 ['uname', '-p']).strip().decode(default_encoding) 235 cputype_mapper = { 236 'BePC': 'i686', 237 'aarch64': 'aarch64', 238 'amd64': 'x86_64', 239 'arm64': 'aarch64', 240 'i386': 'i686', 241 'i486': 'i686', 242 'i686': 'i686', 243 'i786': 'i686', 244 'powerpc': 'powerpc', 245 'powerpc64': 'powerpc64', 246 'powerpc64le': 'powerpc64le', 247 'ppc': 'powerpc', 248 'ppc64': 'powerpc64', 249 'ppc64le': 'powerpc64le', 250 's390x': 's390x', 251 'x64': 'x86_64', 252 'x86': 'i686', 253 'x86-64': 'x86_64', 254 'x86_64': 'x86_64' 255 } 256 257 # Consider the direct transformation first and then the special cases 258 if cputype in cputype_mapper: 259 cputype = cputype_mapper[cputype] 260 elif cputype in {'xscale', 'arm'}: 261 cputype = 'arm' 262 if ostype == 'linux-android': 263 ostype = 'linux-androideabi' 264 elif ostype == 'unknown-freebsd': 265 cputype = subprocess.check_output( 266 ['uname', '-p']).strip().decode(default_encoding) 267 ostype = 'unknown-freebsd' 268 elif cputype == 'armv6l': 269 cputype = 'arm' 270 if ostype == 'linux-android': 271 ostype = 'linux-androideabi' 272 else: 273 ostype += 'eabihf' 274 elif cputype in {'armv7l', 'armv8l'}: 275 cputype = 'armv7' 276 if ostype == 'linux-android': 277 ostype = 'linux-androideabi' 278 else: 279 ostype += 'eabihf' 280 elif cputype == 'mips': 281 if sys.byteorder == 'big': 282 cputype = 'mips' 283 elif sys.byteorder == 'little': 284 cputype = 'mipsel' 285 else: 286 raise ValueError("unknown byteorder: {}".format(sys.byteorder)) 287 elif cputype == 'mips64': 288 if sys.byteorder == 'big': 289 cputype = 'mips64' 290 elif sys.byteorder == 'little': 291 cputype = 'mips64el' 292 else: 293 raise ValueError('unknown byteorder: {}'.format(sys.byteorder)) 294 # only the n64 ABI is supported, indicate it 295 ostype += 'abi64' 296 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64': 297 pass 298 else: 299 err = "unknown cpu type: {}".format(cputype) 300 sys.exit(err) 301 302 return "{}-{}".format(cputype, ostype) 303 304 305@contextlib.contextmanager 306def output(filepath): 307 tmp = filepath + '.tmp' 308 with open(tmp, 'w') as f: 309 yield f 310 try: 311 os.remove(filepath) # PermissionError/OSError on Win32 if in use 312 os.rename(tmp, filepath) 313 except OSError: 314 shutil.copy2(tmp, filepath) 315 os.remove(tmp) 316 317 318class RustBuild(object): 319 """Provide all the methods required to build Rust""" 320 def __init__(self): 321 self.cargo_channel = '' 322 self.date = '' 323 self._download_url = '' 324 self.rustc_channel = '' 325 self.rustfmt_channel = '' 326 self.build = '' 327 self.build_dir = os.path.join(os.getcwd(), "build") 328 self.clean = False 329 self.config_toml = '' 330 self.rust_root = '' 331 self.use_locked_deps = '' 332 self.use_vendored_sources = '' 333 self.verbose = False 334 335 def download_stage0(self): 336 """Fetch the build system for Rust, written in Rust 337 338 This method will build a cache directory, then it will fetch the 339 tarball which has the stage0 compiler used to then bootstrap the Rust 340 compiler itself. 341 342 Each downloaded tarball is extracted, after that, the script 343 will move all the content to the right place. 344 """ 345 rustc_channel = self.rustc_channel 346 cargo_channel = self.cargo_channel 347 rustfmt_channel = self.rustfmt_channel 348 349 def support_xz(): 350 try: 351 with tempfile.NamedTemporaryFile(delete=False) as temp_file: 352 temp_path = temp_file.name 353 with tarfile.open(temp_path, "w:xz"): 354 pass 355 return True 356 except tarfile.CompressionError: 357 return False 358 359 if self.rustc().startswith(self.bin_root()) and \ 360 (not os.path.exists(self.rustc()) or 361 self.program_out_of_date(self.rustc_stamp())): 362 if os.path.exists(self.bin_root()): 363 shutil.rmtree(self.bin_root()) 364 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' 365 filename = "rust-std-{}-{}{}".format( 366 rustc_channel, self.build, tarball_suffix) 367 pattern = "rust-std-{}".format(self.build) 368 self._download_stage0_helper(filename, pattern, tarball_suffix) 369 370 filename = "rustc-{}-{}{}".format(rustc_channel, self.build, 371 tarball_suffix) 372 self._download_stage0_helper(filename, "rustc", tarball_suffix) 373 self.fix_executable("{}/bin/rustc".format(self.bin_root())) 374 self.fix_executable("{}/bin/rustdoc".format(self.bin_root())) 375 with output(self.rustc_stamp()) as rust_stamp: 376 rust_stamp.write(self.date) 377 378 # This is required so that we don't mix incompatible MinGW 379 # libraries/binaries that are included in rust-std with 380 # the system MinGW ones. 381 if "pc-windows-gnu" in self.build: 382 filename = "rust-mingw-{}-{}{}".format( 383 rustc_channel, self.build, tarball_suffix) 384 self._download_stage0_helper(filename, "rust-mingw", tarball_suffix) 385 386 if self.cargo().startswith(self.bin_root()) and \ 387 (not os.path.exists(self.cargo()) or 388 self.program_out_of_date(self.cargo_stamp())): 389 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' 390 filename = "cargo-{}-{}{}".format(cargo_channel, self.build, 391 tarball_suffix) 392 self._download_stage0_helper(filename, "cargo", tarball_suffix) 393 self.fix_executable("{}/bin/cargo".format(self.bin_root())) 394 with output(self.cargo_stamp()) as cargo_stamp: 395 cargo_stamp.write(self.date) 396 397 if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and ( 398 not os.path.exists(self.rustfmt()) 399 or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel) 400 ): 401 if rustfmt_channel: 402 tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz' 403 [channel, date] = rustfmt_channel.split('-', 1) 404 filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix) 405 self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date) 406 self.fix_executable("{}/bin/rustfmt".format(self.bin_root())) 407 self.fix_executable("{}/bin/cargo-fmt".format(self.bin_root())) 408 with output(self.rustfmt_stamp()) as rustfmt_stamp: 409 rustfmt_stamp.write(self.date + self.rustfmt_channel) 410 411 def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None): 412 if date is None: 413 date = self.date 414 cache_dst = os.path.join(self.build_dir, "cache") 415 rustc_cache = os.path.join(cache_dst, date) 416 if not os.path.exists(rustc_cache): 417 os.makedirs(rustc_cache) 418 419 url = "{}/dist/{}".format(self._download_url, date) 420 tarball = os.path.join(rustc_cache, filename) 421 if not os.path.exists(tarball): 422 get("{}/{}".format(url, filename), tarball, verbose=self.verbose) 423 unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose) 424 425 @staticmethod 426 def fix_executable(fname): 427 """Modifies the interpreter section of 'fname' to fix the dynamic linker 428 429 This method is only required on NixOS and uses the PatchELF utility to 430 change the dynamic linker of ELF executables. 431 432 Please see https://nixos.org/patchelf.html for more information 433 """ 434 default_encoding = sys.getdefaultencoding() 435 try: 436 ostype = subprocess.check_output( 437 ['uname', '-s']).strip().decode(default_encoding) 438 except subprocess.CalledProcessError: 439 return 440 except OSError as reason: 441 if getattr(reason, 'winerror', None) is not None: 442 return 443 raise reason 444 445 if ostype != "Linux": 446 return 447 448 if not os.path.exists("/etc/NIXOS"): 449 return 450 if os.path.exists("/lib"): 451 return 452 453 # At this point we're pretty sure the user is running NixOS 454 nix_os_msg = "info: you seem to be running NixOS. Attempting to patch" 455 print(nix_os_msg, fname) 456 457 try: 458 interpreter = subprocess.check_output( 459 ["patchelf", "--print-interpreter", fname]) 460 interpreter = interpreter.strip().decode(default_encoding) 461 except subprocess.CalledProcessError as reason: 462 print("warning: failed to call patchelf:", reason) 463 return 464 465 loader = interpreter.split("/")[-1] 466 467 try: 468 ldd_output = subprocess.check_output( 469 ['ldd', '/run/current-system/sw/bin/sh']) 470 ldd_output = ldd_output.strip().decode(default_encoding) 471 except subprocess.CalledProcessError as reason: 472 print("warning: unable to call ldd:", reason) 473 return 474 475 for line in ldd_output.splitlines(): 476 libname = line.split()[0] 477 if libname.endswith(loader): 478 loader_path = libname[:len(libname) - len(loader)] 479 break 480 else: 481 print("warning: unable to find the path to the dynamic linker") 482 return 483 484 correct_interpreter = loader_path + loader 485 486 try: 487 subprocess.check_output( 488 ["patchelf", "--set-interpreter", correct_interpreter, fname]) 489 except subprocess.CalledProcessError as reason: 490 print("warning: failed to call patchelf:", reason) 491 return 492 493 def rustc_stamp(self): 494 """Return the path for .rustc-stamp 495 496 >>> rb = RustBuild() 497 >>> rb.build_dir = "build" 498 >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp") 499 True 500 """ 501 return os.path.join(self.bin_root(), '.rustc-stamp') 502 503 def cargo_stamp(self): 504 """Return the path for .cargo-stamp 505 506 >>> rb = RustBuild() 507 >>> rb.build_dir = "build" 508 >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp") 509 True 510 """ 511 return os.path.join(self.bin_root(), '.cargo-stamp') 512 513 def rustfmt_stamp(self): 514 """Return the path for .rustfmt-stamp 515 516 >>> rb = RustBuild() 517 >>> rb.build_dir = "build" 518 >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp") 519 True 520 """ 521 return os.path.join(self.bin_root(), '.rustfmt-stamp') 522 523 def program_out_of_date(self, stamp_path, extra=""): 524 """Check if the given program stamp is out of date""" 525 if not os.path.exists(stamp_path) or self.clean: 526 return True 527 with open(stamp_path, 'r') as stamp: 528 return (self.date + extra) != stamp.read() 529 530 def bin_root(self): 531 """Return the binary root directory 532 533 >>> rb = RustBuild() 534 >>> rb.build_dir = "build" 535 >>> rb.bin_root() == os.path.join("build", "stage0") 536 True 537 538 When the 'build' property is given should be a nested directory: 539 540 >>> rb.build = "devel" 541 >>> rb.bin_root() == os.path.join("build", "devel", "stage0") 542 True 543 """ 544 return os.path.join(self.build_dir, self.build, "stage0") 545 546 def get_toml(self, key, section=None): 547 """Returns the value of the given key in config.toml, otherwise returns None 548 549 >>> rb = RustBuild() 550 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"' 551 >>> rb.get_toml("key2") 552 'value2' 553 554 If the key does not exists, the result is None: 555 556 >>> rb.get_toml("key3") is None 557 True 558 559 Optionally also matches the section the key appears in 560 561 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"' 562 >>> rb.get_toml('key', 'a') 563 'value1' 564 >>> rb.get_toml('key', 'b') 565 'value2' 566 >>> rb.get_toml('key', 'c') is None 567 True 568 569 >>> rb.config_toml = 'key1 = true' 570 >>> rb.get_toml("key1") 571 'true' 572 """ 573 574 cur_section = None 575 for line in self.config_toml.splitlines(): 576 section_match = re.match(r'^\s*\[(.*)\]\s*$', line) 577 if section_match is not None: 578 cur_section = section_match.group(1) 579 580 match = re.match(r'^{}\s*=(.*)$'.format(key), line) 581 if match is not None: 582 value = match.group(1) 583 if section is None or section == cur_section: 584 return self.get_string(value) or value.strip() 585 return None 586 587 def cargo(self): 588 """Return config path for cargo""" 589 return self.program_config('cargo') 590 591 def rustc(self): 592 """Return config path for rustc""" 593 return self.program_config('rustc') 594 595 def rustfmt(self): 596 """Return config path for rustfmt""" 597 if not self.rustfmt_channel: 598 return None 599 return self.program_config('rustfmt') 600 601 def program_config(self, program): 602 """Return config path for the given program 603 604 >>> rb = RustBuild() 605 >>> rb.config_toml = 'rustc = "rustc"\\n' 606 >>> rb.program_config('rustc') 607 'rustc' 608 >>> rb.config_toml = '' 609 >>> cargo_path = rb.program_config('cargo') 610 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(), 611 ... "bin", "cargo") 612 True 613 """ 614 config = self.get_toml(program) 615 if config: 616 return os.path.expanduser(config) 617 return os.path.join(self.bin_root(), "bin", "{}{}".format( 618 program, self.exe_suffix())) 619 620 @staticmethod 621 def get_string(line): 622 """Return the value between double quotes 623 624 >>> RustBuild.get_string(' "devel" ') 625 'devel' 626 >>> RustBuild.get_string(" 'devel' ") 627 'devel' 628 >>> RustBuild.get_string('devel') is None 629 True 630 >>> RustBuild.get_string(' "devel ') 631 '' 632 """ 633 start = line.find('"') 634 if start != -1: 635 end = start + 1 + line[start + 1:].find('"') 636 return line[start + 1:end] 637 start = line.find('\'') 638 if start != -1: 639 end = start + 1 + line[start + 1:].find('\'') 640 return line[start + 1:end] 641 return None 642 643 @staticmethod 644 def exe_suffix(): 645 """Return a suffix for executables""" 646 if sys.platform == 'win32': 647 return '.exe' 648 return '' 649 650 def bootstrap_binary(self): 651 """Return the path of the bootstrap binary 652 653 >>> rb = RustBuild() 654 >>> rb.build_dir = "build" 655 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap", 656 ... "debug", "bootstrap") 657 True 658 """ 659 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap") 660 661 def build_bootstrap(self): 662 """Build bootstrap""" 663 build_dir = os.path.join(self.build_dir, "bootstrap") 664 if self.clean and os.path.exists(build_dir): 665 shutil.rmtree(build_dir) 666 env = os.environ.copy() 667 # `CARGO_BUILD_TARGET` breaks bootstrap build. 668 # See also: <https://github.com/rust-lang/rust/issues/70208>. 669 if "CARGO_BUILD_TARGET" in env: 670 del env["CARGO_BUILD_TARGET"] 671 env["RUSTC_BOOTSTRAP"] = '1' 672 env["CARGO_TARGET_DIR"] = build_dir 673 env["RUSTC"] = self.rustc() 674 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ 675 (os.pathsep + env["LD_LIBRARY_PATH"]) \ 676 if "LD_LIBRARY_PATH" in env else "" 677 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ 678 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \ 679 if "DYLD_LIBRARY_PATH" in env else "" 680 env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ 681 (os.pathsep + env["LIBRARY_PATH"]) \ 682 if "LIBRARY_PATH" in env else "" 683 # preserve existing RUSTFLAGS 684 env.setdefault("RUSTFLAGS", "") 685 env["RUSTFLAGS"] += " -Cdebuginfo=2" 686 687 build_section = "target.{}".format(self.build_triple()) 688 target_features = [] 689 if self.get_toml("crt-static", build_section) == "true": 690 target_features += ["+crt-static"] 691 elif self.get_toml("crt-static", build_section) == "false": 692 target_features += ["-crt-static"] 693 if target_features: 694 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features)) 695 target_linker = self.get_toml("linker", build_section) 696 if target_linker is not None: 697 env["RUSTFLAGS"] += " -C linker=" + target_linker 698 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes" 699 if self.get_toml("deny-warnings", "rust") != "false": 700 env["RUSTFLAGS"] += " -Dwarnings" 701 702 env["PATH"] = os.path.join(self.bin_root(), "bin") + \ 703 os.pathsep + env["PATH"] 704 if not os.path.isfile(self.cargo()): 705 raise Exception("no cargo executable found at `{}`".format( 706 self.cargo())) 707 args = [self.cargo(), "build", "--manifest-path", 708 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")] 709 for _ in range(1, self.verbose): 710 args.append("--verbose") 711 if self.use_locked_deps: 712 args.append("--locked") 713 if self.use_vendored_sources: 714 args.append("--frozen") 715 run(args, env=env, verbose=self.verbose) 716 717 def build_triple(self): 718 """Build triple as in LLVM""" 719 config = self.get_toml('build') 720 if config: 721 return config 722 return default_build_triple() 723 724 def check_submodule(self, module, slow_submodules): 725 if not slow_submodules: 726 checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"], 727 cwd=os.path.join(self.rust_root, module), 728 stdout=subprocess.PIPE) 729 return checked_out 730 else: 731 return None 732 733 def update_submodule(self, module, checked_out, recorded_submodules): 734 module_path = os.path.join(self.rust_root, module) 735 736 if checked_out is not None: 737 default_encoding = sys.getdefaultencoding() 738 checked_out = checked_out.communicate()[0].decode(default_encoding).strip() 739 if recorded_submodules[module] == checked_out: 740 return 741 742 print("Updating submodule", module) 743 744 run(["git", "submodule", "-q", "sync", module], 745 cwd=self.rust_root, verbose=self.verbose) 746 try: 747 run(["git", "submodule", "update", 748 "--init", "--recursive", "--progress", module], 749 cwd=self.rust_root, verbose=self.verbose, exception=True) 750 except RuntimeError: 751 # Some versions of git don't support --progress. 752 run(["git", "submodule", "update", 753 "--init", "--recursive", module], 754 cwd=self.rust_root, verbose=self.verbose) 755 run(["git", "reset", "-q", "--hard"], 756 cwd=module_path, verbose=self.verbose) 757 run(["git", "clean", "-qdfx"], 758 cwd=module_path, verbose=self.verbose) 759 760 def update_submodules(self): 761 """Update submodules""" 762 if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \ 763 self.get_toml('submodules') == "false": 764 return 765 766 # check the existence of 'git' command 767 try: 768 subprocess.check_output(['git', '--version']) 769 except (subprocess.CalledProcessError, OSError): 770 print("error: `git` is not found, please make sure it's installed and in the path.") 771 sys.exit(1) 772 773 slow_submodules = self.get_toml('fast-submodules') == "false" 774 start_time = time() 775 if slow_submodules: 776 print('Unconditionally updating all submodules') 777 else: 778 print('Updating only changed submodules') 779 default_encoding = sys.getdefaultencoding() 780 submodules = [s.split(' ', 1)[1] for s in subprocess.check_output( 781 ["git", "config", "--file", 782 os.path.join(self.rust_root, ".gitmodules"), 783 "--get-regexp", "path"] 784 ).decode(default_encoding).splitlines()] 785 filtered_submodules = [] 786 submodules_names = [] 787 for module in submodules: 788 if module.endswith("llvm-project"): 789 if self.get_toml('llvm-config') and self.get_toml('lld') != 'true': 790 continue 791 check = self.check_submodule(module, slow_submodules) 792 filtered_submodules.append((module, check)) 793 submodules_names.append(module) 794 recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names, 795 cwd=self.rust_root, stdout=subprocess.PIPE) 796 recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines() 797 recorded_submodules = {} 798 for data in recorded: 799 data = data.split() 800 recorded_submodules[data[3]] = data[2] 801 for module in filtered_submodules: 802 self.update_submodule(module[0], module[1], recorded_submodules) 803 print("Submodules updated in %.2f seconds" % (time() - start_time)) 804 805 def set_normal_environment(self): 806 """Set download URL for normal environment""" 807 if 'RUSTUP_DIST_SERVER' in os.environ: 808 self._download_url = os.environ['RUSTUP_DIST_SERVER'] 809 else: 810 self._download_url = 'https://static.rust-lang.org' 811 812 def set_dev_environment(self): 813 """Set download URL for development environment""" 814 if 'RUSTUP_DEV_DIST_SERVER' in os.environ: 815 self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER'] 816 else: 817 self._download_url = 'https://dev-static.rust-lang.org' 818 819 def check_vendored_status(self): 820 """Check that vendoring is configured properly""" 821 vendor_dir = os.path.join(self.rust_root, 'vendor') 822 if 'SUDO_USER' in os.environ and not self.use_vendored_sources: 823 if os.environ.get('USER') != os.environ['SUDO_USER']: 824 self.use_vendored_sources = True 825 print('info: looks like you are running this command under `sudo`') 826 print(' and so in order to preserve your $HOME this will now') 827 print(' use vendored sources by default.') 828 if not os.path.exists(vendor_dir): 829 print('error: vendoring required, but vendor directory does not exist.') 830 print(' Run `cargo vendor` without sudo to initialize the ' 831 'vendor directory.') 832 raise Exception("{} not found".format(vendor_dir)) 833 834 if self.use_vendored_sources: 835 if not os.path.exists('.cargo'): 836 os.makedirs('.cargo') 837 with output('.cargo/config') as cargo_config: 838 cargo_config.write( 839 "[source.crates-io]\n" 840 "replace-with = 'vendored-sources'\n" 841 "registry = 'https://example.com'\n" 842 "\n" 843 "[source.vendored-sources]\n" 844 "directory = '{}/vendor'\n" 845 .format(self.rust_root)) 846 else: 847 if os.path.exists('.cargo'): 848 shutil.rmtree('.cargo') 849 850 def ensure_vendored(self): 851 """Ensure that the vendored sources are available if needed""" 852 vendor_dir = os.path.join(self.rust_root, 'vendor') 853 # Note that this does not handle updating the vendored dependencies if 854 # the rust git repository is updated. Normal development usually does 855 # not use vendoring, so hopefully this isn't too much of a problem. 856 if self.use_vendored_sources and not os.path.exists(vendor_dir): 857 run([self.cargo(), "vendor"], 858 verbose=self.verbose, cwd=self.rust_root) 859 860 861def bootstrap(help_triggered): 862 """Configure, fetch, build and run the initial bootstrap""" 863 864 # If the user is asking for help, let them know that the whole download-and-build 865 # process has to happen before anything is printed out. 866 if help_triggered: 867 print("info: Downloading and building bootstrap before processing --help") 868 print(" command. See src/bootstrap/README.md for help with common") 869 print(" commands.") 870 871 parser = argparse.ArgumentParser(description='Build rust') 872 parser.add_argument('--config') 873 parser.add_argument('--build') 874 parser.add_argument('--src') 875 parser.add_argument('--clean', action='store_true') 876 parser.add_argument('-v', '--verbose', action='count', default=0) 877 878 args = [a for a in sys.argv if a != '-h' and a != '--help'] 879 args, _ = parser.parse_known_args(args) 880 881 # Configure initial bootstrap 882 build = RustBuild() 883 build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..')) 884 build.verbose = args.verbose 885 build.clean = args.clean 886 887 try: 888 with open(args.config or 'config.toml') as config: 889 build.config_toml = config.read() 890 except (OSError, IOError): 891 pass 892 893 config_verbose = build.get_toml('verbose', 'build') 894 if config_verbose is not None: 895 build.verbose = max(build.verbose, int(config_verbose)) 896 897 build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true' 898 899 build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true' 900 901 build.check_vendored_status() 902 903 data = stage0_data(build.rust_root) 904 build.date = data['date'] 905 build.rustc_channel = data['rustc'] 906 build.cargo_channel = data['cargo'] 907 908 if "rustfmt" in data: 909 build.rustfmt_channel = data['rustfmt'] 910 911 if 'dev' in data: 912 build.set_dev_environment() 913 else: 914 build.set_normal_environment() 915 916 build.update_submodules() 917 918 # Fetch/build the bootstrap 919 build.build = args.build or build.build_triple() 920 build.download_stage0() 921 sys.stdout.flush() 922 build.ensure_vendored() 923 build.build_bootstrap() 924 sys.stdout.flush() 925 926 # Run the bootstrap 927 args = [build.bootstrap_binary()] 928 args.extend(sys.argv[1:]) 929 env = os.environ.copy() 930 env["BUILD"] = build.build 931 env["SRC"] = build.rust_root 932 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) 933 env["BOOTSTRAP_PYTHON"] = sys.executable 934 env["BUILD_DIR"] = build.build_dir 935 env["RUSTC_BOOTSTRAP"] = '1' 936 env["CARGO"] = build.cargo() 937 env["RUSTC"] = build.rustc() 938 if build.rustfmt(): 939 env["RUSTFMT"] = build.rustfmt() 940 run(args, env=env, verbose=build.verbose) 941 942 943def main(): 944 """Entry point for the bootstrap process""" 945 start_time = time() 946 947 # x.py help <cmd> ... 948 if len(sys.argv) > 1 and sys.argv[1] == 'help': 949 sys.argv = [sys.argv[0], '-h'] + sys.argv[2:] 950 951 help_triggered = ( 952 '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1) 953 try: 954 bootstrap(help_triggered) 955 if not help_triggered: 956 print("Build completed successfully in {}".format( 957 format_build_time(time() - start_time))) 958 except (SystemExit, KeyboardInterrupt) as error: 959 if hasattr(error, 'code') and isinstance(error.code, int): 960 exit_code = error.code 961 else: 962 exit_code = 1 963 print(error) 964 if not help_triggered: 965 print("Build completed unsuccessfully in {}".format( 966 format_build_time(time() - start_time))) 967 sys.exit(exit_code) 968 969 970if __name__ == '__main__': 971 main()