fork/rust
git clone https://git.julia.blue/fork/rust
3bcad65fbf6
master
1from __future__ import absolute_import, division, print_function 2import argparse 3import contextlib 4import datetime 5import hashlib 6import json 7import os 8import re 9import shutil 10import subprocess 11import sys 12import tarfile 13import tempfile 14 15from time import time 16from multiprocessing import Pool, cpu_count 17 18try: 19 import lzma 20except ImportError: 21 lzma = None 22 23def platform_is_win32(): 24 return sys.platform == 'win32' 25 26if platform_is_win32(): 27 EXE_SUFFIX = ".exe" 28else: 29 EXE_SUFFIX = "" 30 31def get_cpus(): 32 if hasattr(os, "sched_getaffinity"): 33 return len(os.sched_getaffinity(0)) 34 if hasattr(os, "cpu_count"): 35 cpus = os.cpu_count() 36 if cpus is not None: 37 return cpus 38 try: 39 return cpu_count() 40 except NotImplementedError: 41 return 1 42 43 44def eprint(*args, **kwargs): 45 kwargs["file"] = sys.stderr 46 print(*args, **kwargs) 47 48 49def get(base, url, path, checksums, verbose=False): 50 with tempfile.NamedTemporaryFile(delete=False) as temp_file: 51 temp_path = temp_file.name 52 53 try: 54 if url not in checksums: 55 raise RuntimeError(("src/stage0.json doesn't contain a checksum for {}. " 56 "Pre-built artifacts might not be available for this " 57 "target at this time, see https://doc.rust-lang.org/nightly" 58 "/rustc/platform-support.html for more information.") 59 .format(url)) 60 sha256 = checksums[url] 61 if os.path.exists(path): 62 if verify(path, sha256, False): 63 if verbose: 64 eprint("using already-download file", path) 65 return 66 else: 67 if verbose: 68 eprint("ignoring already-download file", 69 path, "due to failed verification") 70 os.unlink(path) 71 download(temp_path, "{}/{}".format(base, url), True, verbose) 72 if not verify(temp_path, sha256, verbose): 73 raise RuntimeError("failed verification") 74 if verbose: 75 eprint("moving {} to {}".format(temp_path, path)) 76 shutil.move(temp_path, path) 77 finally: 78 if os.path.isfile(temp_path): 79 if verbose: 80 eprint("removing", temp_path) 81 os.unlink(temp_path) 82 83 84def download(path, url, probably_big, verbose): 85 for _ in range(4): 86 try: 87 _download(path, url, probably_big, verbose, True) 88 return 89 except RuntimeError: 90 eprint("\nspurious failure, trying again") 91 _download(path, url, probably_big, verbose, False) 92 93 94def _download(path, url, probably_big, verbose, exception): 95 # Try to use curl (potentially available on win32 96 # https://devblogs.microsoft.com/commandline/tar-and-curl-come-to-windows/) 97 # If an error occurs: 98 # - If we are on win32 fallback to powershell 99 # - Otherwise raise the error if appropriate 100 if probably_big or verbose: 101 eprint("downloading {}".format(url)) 102 103 try: 104 if (probably_big or verbose) and "GITHUB_ACTIONS" not in os.environ: 105 option = "-#" 106 else: 107 option = "-s" 108 # If curl is not present on Win32, we should not sys.exit 109 # but raise `CalledProcessError` or `OSError` instead 110 require(["curl", "--version"], exception=platform_is_win32()) 111 run(["curl", option, 112 "-L", # Follow redirect. 113 "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds 114 "--connect-timeout", "30", # timeout if cannot connect within 30 seconds 115 "-o", path, 116 "--retry", "3", "-SRf", url], 117 verbose=verbose, 118 exception=True, # Will raise RuntimeError on failure 119 ) 120 except (subprocess.CalledProcessError, OSError, RuntimeError): 121 # see http://serverfault.com/questions/301128/how-to-download 122 if platform_is_win32(): 123 run_powershell([ 124 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", 125 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)], 126 verbose=verbose, 127 exception=exception) 128 # Check if the RuntimeError raised by run(curl) should be silenced 129 elif verbose or exception: 130 raise 131 132 133def verify(path, expected, verbose): 134 """Check if the sha256 sum of the given path is valid""" 135 if verbose: 136 eprint("verifying", path) 137 with open(path, "rb") as source: 138 found = hashlib.sha256(source.read()).hexdigest() 139 verified = found == expected 140 if not verified: 141 eprint("invalid checksum:\n" 142 " found: {}\n" 143 " expected: {}".format(found, expected)) 144 return verified 145 146 147def unpack(tarball, tarball_suffix, dst, verbose=False, match=None): 148 """Unpack the given tarball file""" 149 eprint("extracting", tarball) 150 fname = os.path.basename(tarball).replace(tarball_suffix, "") 151 with contextlib.closing(tarfile.open(tarball)) as tar: 152 for member in tar.getnames(): 153 if "/" not in member: 154 continue 155 name = member.replace(fname + "/", "", 1) 156 if match is not None and not name.startswith(match): 157 continue 158 name = name[len(match) + 1:] 159 160 dst_path = os.path.join(dst, name) 161 if verbose: 162 eprint(" extracting", member) 163 tar.extract(member, dst) 164 src_path = os.path.join(dst, member) 165 if os.path.isdir(src_path) and os.path.exists(dst_path): 166 continue 167 shutil.move(src_path, dst_path) 168 shutil.rmtree(os.path.join(dst, fname)) 169 170 171def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs): 172 """Run a child program in a new process""" 173 if verbose: 174 eprint("running: " + ' '.join(args)) 175 sys.stdout.flush() 176 # Ensure that the .exe is used on Windows just in case a Linux ELF has been 177 # compiled in the same directory. 178 if os.name == 'nt' and not args[0].endswith('.exe'): 179 args[0] += '.exe' 180 # Use Popen here instead of call() as it apparently allows powershell on 181 # Windows to not lock up waiting for input presumably. 182 ret = subprocess.Popen(args, **kwargs) 183 code = ret.wait() 184 if code != 0: 185 err = "failed to run: " + ' '.join(args) 186 if verbose or exception: 187 raise RuntimeError(err) 188 # For most failures, we definitely do want to print this error, or the user will have no 189 # idea what went wrong. But when we've successfully built bootstrap and it failed, it will 190 # have already printed an error above, so there's no need to print the exact command we're 191 # running. 192 if is_bootstrap: 193 sys.exit(1) 194 else: 195 sys.exit(err) 196 197def run_powershell(script, *args, **kwargs): 198 """Run a powershell script""" 199 run(["PowerShell.exe", "/nologo", "-Command"] + script, *args, **kwargs) 200 201 202def require(cmd, exit=True, exception=False): 203 '''Run a command, returning its output. 204 On error, 205 If `exception` is `True`, raise the error 206 Otherwise If `exit` is `True`, exit the process 207 Else return None.''' 208 try: 209 return subprocess.check_output(cmd).strip() 210 except (subprocess.CalledProcessError, OSError) as exc: 211 if exception: 212 raise 213 elif exit: 214 eprint("error: unable to run `{}`: {}".format(' '.join(cmd), exc)) 215 eprint("Please make sure it's installed and in the path.") 216 sys.exit(1) 217 return None 218 219 220 221def format_build_time(duration): 222 """Return a nicer format for build time 223 224 >>> format_build_time('300') 225 '0:05:00' 226 """ 227 return str(datetime.timedelta(seconds=int(duration))) 228 229 230def default_build_triple(verbose): 231 """Build triple as in LLVM""" 232 # If we're on Windows and have an existing `rustc` toolchain, use `rustc --version --verbose` 233 # to find our host target triple. This fixes an issue with Windows builds being detected 234 # as GNU instead of MSVC. 235 # Otherwise, detect it via `uname` 236 default_encoding = sys.getdefaultencoding() 237 238 if platform_is_win32(): 239 try: 240 version = subprocess.check_output(["rustc", "--version", "--verbose"], 241 stderr=subprocess.DEVNULL) 242 version = version.decode(default_encoding) 243 host = next(x for x in version.split('\n') if x.startswith("host: ")) 244 triple = host.split("host: ")[1] 245 if verbose: 246 eprint("detected default triple {} from pre-installed rustc".format(triple)) 247 return triple 248 except Exception as e: 249 if verbose: 250 eprint("pre-installed rustc not detected: {}".format(e)) 251 eprint("falling back to auto-detect") 252 253 required = not platform_is_win32() 254 uname = require(["uname", "-smp"], exit=required) 255 256 # If we do not have `uname`, assume Windows. 257 if uname is None: 258 return 'x86_64-pc-windows-msvc' 259 260 kernel, cputype, processor = uname.decode(default_encoding).split(maxsplit=2) 261 262 # The goal here is to come up with the same triple as LLVM would, 263 # at least for the subset of platforms we're willing to target. 264 kerneltype_mapper = { 265 'Darwin': 'apple-darwin', 266 'DragonFly': 'unknown-dragonfly', 267 'FreeBSD': 'unknown-freebsd', 268 'Haiku': 'unknown-haiku', 269 'NetBSD': 'unknown-netbsd', 270 'OpenBSD': 'unknown-openbsd', 271 'GNU': 'unknown-hurd', 272 } 273 274 # Consider the direct transformation first and then the special cases 275 if kernel in kerneltype_mapper: 276 kernel = kerneltype_mapper[kernel] 277 elif kernel == 'Linux': 278 # Apple doesn't support `-o` so this can't be used in the combined 279 # uname invocation above 280 ostype = require(["uname", "-o"], exit=required).decode(default_encoding) 281 if ostype == 'Android': 282 kernel = 'linux-android' 283 else: 284 kernel = 'unknown-linux-gnu' 285 elif kernel == 'SunOS': 286 kernel = 'pc-solaris' 287 # On Solaris, uname -m will return a machine classification instead 288 # of a cpu type, so uname -p is recommended instead. However, the 289 # output from that option is too generic for our purposes (it will 290 # always emit 'i386' on x86/amd64 systems). As such, isainfo -k 291 # must be used instead. 292 cputype = require(['isainfo', '-k']).decode(default_encoding) 293 # sparc cpus have sun as a target vendor 294 if 'sparc' in cputype: 295 kernel = 'sun-solaris' 296 elif kernel.startswith('MINGW'): 297 # msys' `uname` does not print gcc configuration, but prints msys 298 # configuration. so we cannot believe `uname -m`: 299 # msys1 is always i686 and msys2 is always x86_64. 300 # instead, msys defines $MSYSTEM which is MINGW32 on i686 and 301 # MINGW64 on x86_64. 302 kernel = 'pc-windows-gnu' 303 cputype = 'i686' 304 if os.environ.get('MSYSTEM') == 'MINGW64': 305 cputype = 'x86_64' 306 elif kernel.startswith('MSYS'): 307 kernel = 'pc-windows-gnu' 308 elif kernel.startswith('CYGWIN_NT'): 309 cputype = 'i686' 310 if kernel.endswith('WOW64'): 311 cputype = 'x86_64' 312 kernel = 'pc-windows-gnu' 313 elif platform_is_win32(): 314 # Some Windows platforms might have a `uname` command that returns a 315 # non-standard string (e.g. gnuwin32 tools returns `windows32`). In 316 # these cases, fall back to using sys.platform. 317 return 'x86_64-pc-windows-msvc' 318 elif kernel == 'AIX': 319 # `uname -m` returns the machine ID rather than machine hardware on AIX, 320 # so we are unable to use cputype to form triple. AIX 7.2 and 321 # above supports 32-bit and 64-bit mode simultaneously and `uname -p` 322 # returns `powerpc`, however we only supports `powerpc64-ibm-aix` in 323 # rust on AIX. For above reasons, kerneltype_mapper and cputype_mapper 324 # are not used to infer AIX's triple. 325 return 'powerpc64-ibm-aix' 326 else: 327 err = "unknown OS type: {}".format(kernel) 328 sys.exit(err) 329 330 if cputype in ['powerpc', 'riscv'] and kernel == 'unknown-freebsd': 331 cputype = subprocess.check_output( 332 ['uname', '-p']).strip().decode(default_encoding) 333 cputype_mapper = { 334 'BePC': 'i686', 335 'aarch64': 'aarch64', 336 'aarch64eb': 'aarch64', 337 'amd64': 'x86_64', 338 'arm64': 'aarch64', 339 'i386': 'i686', 340 'i486': 'i686', 341 'i686': 'i686', 342 'i686-AT386': 'i686', 343 'i786': 'i686', 344 'loongarch64': 'loongarch64', 345 'm68k': 'm68k', 346 'csky': 'csky', 347 'powerpc': 'powerpc', 348 'powerpc64': 'powerpc64', 349 'powerpc64le': 'powerpc64le', 350 'ppc': 'powerpc', 351 'ppc64': 'powerpc64', 352 'ppc64le': 'powerpc64le', 353 'riscv64': 'riscv64gc', 354 's390x': 's390x', 355 'x64': 'x86_64', 356 'x86': 'i686', 357 'x86-64': 'x86_64', 358 'x86_64': 'x86_64' 359 } 360 361 # Consider the direct transformation first and then the special cases 362 if cputype in cputype_mapper: 363 cputype = cputype_mapper[cputype] 364 elif cputype in {'xscale', 'arm'}: 365 cputype = 'arm' 366 if kernel == 'linux-android': 367 kernel = 'linux-androideabi' 368 elif kernel == 'unknown-freebsd': 369 cputype = processor 370 kernel = 'unknown-freebsd' 371 elif cputype == 'armv6l': 372 cputype = 'arm' 373 if kernel == 'linux-android': 374 kernel = 'linux-androideabi' 375 else: 376 kernel += 'eabihf' 377 elif cputype in {'armv7l', 'armv8l'}: 378 cputype = 'armv7' 379 if kernel == 'linux-android': 380 kernel = 'linux-androideabi' 381 else: 382 kernel += 'eabihf' 383 elif cputype == 'mips': 384 if sys.byteorder == 'big': 385 cputype = 'mips' 386 elif sys.byteorder == 'little': 387 cputype = 'mipsel' 388 else: 389 raise ValueError("unknown byteorder: {}".format(sys.byteorder)) 390 elif cputype == 'mips64': 391 if sys.byteorder == 'big': 392 cputype = 'mips64' 393 elif sys.byteorder == 'little': 394 cputype = 'mips64el' 395 else: 396 raise ValueError('unknown byteorder: {}'.format(sys.byteorder)) 397 # only the n64 ABI is supported, indicate it 398 kernel += 'abi64' 399 elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64': 400 pass 401 else: 402 err = "unknown cpu type: {}".format(cputype) 403 sys.exit(err) 404 405 return "{}-{}".format(cputype, kernel) 406 407 408@contextlib.contextmanager 409def output(filepath): 410 tmp = filepath + '.tmp' 411 with open(tmp, 'w') as f: 412 yield f 413 try: 414 if os.path.exists(filepath): 415 os.remove(filepath) # PermissionError/OSError on Win32 if in use 416 except OSError: 417 shutil.copy2(tmp, filepath) 418 os.remove(tmp) 419 return 420 os.rename(tmp, filepath) 421 422 423class Stage0Toolchain: 424 def __init__(self, stage0_payload): 425 self.date = stage0_payload["date"] 426 self.version = stage0_payload["version"] 427 428 def channel(self): 429 return self.version + "-" + self.date 430 431 432class DownloadInfo: 433 """A helper class that can be pickled into a parallel subprocess""" 434 435 def __init__( 436 self, 437 base_download_url, 438 download_path, 439 bin_root, 440 tarball_path, 441 tarball_suffix, 442 checksums_sha256, 443 pattern, 444 verbose, 445 ): 446 self.base_download_url = base_download_url 447 self.download_path = download_path 448 self.bin_root = bin_root 449 self.tarball_path = tarball_path 450 self.tarball_suffix = tarball_suffix 451 self.checksums_sha256 = checksums_sha256 452 self.pattern = pattern 453 self.verbose = verbose 454 455def download_component(download_info): 456 if not os.path.exists(download_info.tarball_path): 457 get( 458 download_info.base_download_url, 459 download_info.download_path, 460 download_info.tarball_path, 461 download_info.checksums_sha256, 462 verbose=download_info.verbose, 463 ) 464 465def unpack_component(download_info): 466 unpack( 467 download_info.tarball_path, 468 download_info.tarball_suffix, 469 download_info.bin_root, 470 match=download_info.pattern, 471 verbose=download_info.verbose, 472 ) 473 474class FakeArgs: 475 """Used for unit tests to avoid updating all call sites""" 476 def __init__(self): 477 self.build = '' 478 self.build_dir = '' 479 self.clean = False 480 self.verbose = False 481 self.json_output = False 482 self.color = 'auto' 483 self.warnings = 'default' 484 485class RustBuild(object): 486 """Provide all the methods required to build Rust""" 487 def __init__(self, config_toml="", args=None): 488 if args is None: 489 args = FakeArgs() 490 self.git_version = None 491 self.nix_deps_dir = None 492 self._should_fix_bins_and_dylibs = None 493 self.rust_root = os.path.abspath(os.path.join(__file__, '../../..')) 494 495 self.config_toml = config_toml 496 497 self.clean = args.clean 498 self.json_output = args.json_output 499 self.verbose = args.verbose 500 self.color = args.color 501 self.warnings = args.warnings 502 503 config_verbose_count = self.get_toml('verbose', 'build') 504 if config_verbose_count is not None: 505 self.verbose = max(self.verbose, int(config_verbose_count)) 506 507 self.use_vendored_sources = self.get_toml('vendor', 'build') == 'true' 508 self.use_locked_deps = self.get_toml('locked-deps', 'build') == 'true' 509 510 build_dir = args.build_dir or self.get_toml('build-dir', 'build') or 'build' 511 self.build_dir = os.path.abspath(build_dir) 512 513 with open(os.path.join(self.rust_root, "src", "stage0.json")) as f: 514 data = json.load(f) 515 self.checksums_sha256 = data["checksums_sha256"] 516 self.stage0_compiler = Stage0Toolchain(data["compiler"]) 517 self.download_url = os.getenv("RUSTUP_DIST_SERVER") or data["config"]["dist_server"] 518 519 self.build = args.build or self.build_triple() 520 521 522 def download_toolchain(self): 523 """Fetch the build system for Rust, written in Rust 524 525 This method will build a cache directory, then it will fetch the 526 tarball which has the stage0 compiler used to then bootstrap the Rust 527 compiler itself. 528 529 Each downloaded tarball is extracted, after that, the script 530 will move all the content to the right place. 531 """ 532 rustc_channel = self.stage0_compiler.version 533 bin_root = self.bin_root() 534 535 key = self.stage0_compiler.date 536 if self.rustc().startswith(bin_root) and \ 537 (not os.path.exists(self.rustc()) or 538 self.program_out_of_date(self.rustc_stamp(), key)): 539 if os.path.exists(bin_root): 540 # HACK: On Windows, we can't delete rust-analyzer-proc-macro-server while it's 541 # running. Kill it. 542 if platform_is_win32(): 543 print("Killing rust-analyzer-proc-macro-srv before deleting stage0 toolchain") 544 regex = '{}\\\\(host|{})\\\\stage0\\\\libexec'.format( 545 os.path.basename(self.build_dir), 546 self.build 547 ) 548 script = ( 549 # NOTE: can't use `taskkill` or `Get-Process -Name` because they error if 550 # the server isn't running. 551 'Get-Process | ' + 552 'Where-Object {$_.Name -eq "rust-analyzer-proc-macro-srv"} |' + 553 'Where-Object {{$_.Path -match "{}"}} |'.format(regex) + 554 'Stop-Process' 555 ) 556 run_powershell([script]) 557 shutil.rmtree(bin_root) 558 559 key = self.stage0_compiler.date 560 cache_dst = os.path.join(self.build_dir, "cache") 561 rustc_cache = os.path.join(cache_dst, key) 562 if not os.path.exists(rustc_cache): 563 os.makedirs(rustc_cache) 564 565 tarball_suffix = '.tar.gz' if lzma is None else '.tar.xz' 566 567 toolchain_suffix = "{}-{}{}".format(rustc_channel, self.build, tarball_suffix) 568 569 tarballs_to_download = [ 570 ("rust-std-{}".format(toolchain_suffix), "rust-std-{}".format(self.build)), 571 ("rustc-{}".format(toolchain_suffix), "rustc"), 572 ("cargo-{}".format(toolchain_suffix), "cargo"), 573 ] 574 575 tarballs_download_info = [ 576 DownloadInfo( 577 base_download_url=self.download_url, 578 download_path="dist/{}/{}".format(self.stage0_compiler.date, filename), 579 bin_root=self.bin_root(), 580 tarball_path=os.path.join(rustc_cache, filename), 581 tarball_suffix=tarball_suffix, 582 checksums_sha256=self.checksums_sha256, 583 pattern=pattern, 584 verbose=self.verbose, 585 ) 586 for filename, pattern in tarballs_to_download 587 ] 588 589 # Download the components serially to show the progress bars properly. 590 for download_info in tarballs_download_info: 591 download_component(download_info) 592 593 # Unpack the tarballs in parallle. 594 # In Python 2.7, Pool cannot be used as a context manager. 595 pool_size = min(len(tarballs_download_info), get_cpus()) 596 if self.verbose: 597 print('Choosing a pool size of', pool_size, 'for the unpacking of the tarballs') 598 p = Pool(pool_size) 599 try: 600 p.map(unpack_component, tarballs_download_info) 601 finally: 602 p.close() 603 p.join() 604 605 if self.should_fix_bins_and_dylibs(): 606 self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root)) 607 608 self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root)) 609 self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root)) 610 self.fix_bin_or_dylib("{}/libexec/rust-analyzer-proc-macro-srv".format(bin_root)) 611 lib_dir = "{}/lib".format(bin_root) 612 for lib in os.listdir(lib_dir): 613 if lib.endswith(".so"): 614 self.fix_bin_or_dylib(os.path.join(lib_dir, lib)) 615 616 with output(self.rustc_stamp()) as rust_stamp: 617 rust_stamp.write(key) 618 619 def _download_component_helper( 620 self, filename, pattern, tarball_suffix, rustc_cache, 621 ): 622 key = self.stage0_compiler.date 623 624 tarball = os.path.join(rustc_cache, filename) 625 if not os.path.exists(tarball): 626 get( 627 self.download_url, 628 "dist/{}/{}".format(key, filename), 629 tarball, 630 self.checksums_sha256, 631 verbose=self.verbose, 632 ) 633 unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose) 634 635 def should_fix_bins_and_dylibs(self): 636 """Whether or not `fix_bin_or_dylib` needs to be run; can only be True 637 on NixOS or if config.toml has `build.patch-binaries-for-nix` set. 638 """ 639 if self._should_fix_bins_and_dylibs is not None: 640 return self._should_fix_bins_and_dylibs 641 642 def get_answer(): 643 default_encoding = sys.getdefaultencoding() 644 try: 645 ostype = subprocess.check_output( 646 ['uname', '-s']).strip().decode(default_encoding) 647 except subprocess.CalledProcessError: 648 return False 649 except OSError as reason: 650 if getattr(reason, 'winerror', None) is not None: 651 return False 652 raise reason 653 654 if ostype != "Linux": 655 return False 656 657 # If the user has explicitly indicated whether binaries should be 658 # patched for Nix, then don't check for NixOS. 659 if self.get_toml("patch-binaries-for-nix", "build") == "true": 660 return True 661 if self.get_toml("patch-binaries-for-nix", "build") == "false": 662 return False 663 664 # Use `/etc/os-release` instead of `/etc/NIXOS`. 665 # The latter one does not exist on NixOS when using tmpfs as root. 666 try: 667 with open("/etc/os-release", "r") as f: 668 is_nixos = any(ln.strip() in ("ID=nixos", "ID='nixos'", 'ID="nixos"') 669 for ln in f) 670 except FileNotFoundError: 671 is_nixos = False 672 673 # If not on NixOS, then warn if user seems to be atop Nix shell 674 if not is_nixos: 675 in_nix_shell = os.getenv('IN_NIX_SHELL') 676 if in_nix_shell: 677 eprint("The IN_NIX_SHELL environment variable is `{}`;".format(in_nix_shell), 678 "you may need to set `patch-binaries-for-nix=true` in config.toml") 679 680 return is_nixos 681 682 answer = self._should_fix_bins_and_dylibs = get_answer() 683 if answer: 684 eprint("info: You seem to be using Nix.") 685 return answer 686 687 def fix_bin_or_dylib(self, fname): 688 """Modifies the interpreter section of 'fname' to fix the dynamic linker, 689 or the RPATH section, to fix the dynamic library search path 690 691 This method is only required on NixOS and uses the PatchELF utility to 692 change the interpreter/RPATH of ELF executables. 693 694 Please see https://nixos.org/patchelf.html for more information 695 """ 696 assert self._should_fix_bins_and_dylibs is True 697 eprint("attempting to patch", fname) 698 699 # Only build `.nix-deps` once. 700 nix_deps_dir = self.nix_deps_dir 701 if not nix_deps_dir: 702 # Run `nix-build` to "build" each dependency (which will likely reuse 703 # the existing `/nix/store` copy, or at most download a pre-built copy). 704 # 705 # Importantly, we create a gc-root called `.nix-deps` in the `build/` 706 # directory, but still reference the actual `/nix/store` path in the rpath 707 # as it makes it significantly more robust against changes to the location of 708 # the `.nix-deps` location. 709 # 710 # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`). 711 # zlib: Needed as a system dependency of `libLLVM-*.so`. 712 # patchelf: Needed for patching ELF binaries (see doc comment above). 713 nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps") 714 nix_expr = ''' 715 with (import <nixpkgs> {}); 716 symlinkJoin { 717 name = "rust-stage0-dependencies"; 718 paths = [ 719 zlib 720 patchelf 721 stdenv.cc.bintools 722 ]; 723 } 724 ''' 725 try: 726 subprocess.check_output([ 727 "nix-build", "-E", nix_expr, "-o", nix_deps_dir, 728 ]) 729 except subprocess.CalledProcessError as reason: 730 eprint("warning: failed to call nix-build:", reason) 731 return 732 self.nix_deps_dir = nix_deps_dir 733 734 patchelf = "{}/bin/patchelf".format(nix_deps_dir) 735 rpath_entries = [ 736 # Relative default, all binary and dynamic libraries we ship 737 # appear to have this (even when `../lib` is redundant). 738 "$ORIGIN/../lib", 739 os.path.join(os.path.realpath(nix_deps_dir), "lib") 740 ] 741 patchelf_args = ["--set-rpath", ":".join(rpath_entries)] 742 if not fname.endswith(".so"): 743 # Finally, set the correct .interp for binaries 744 with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker: 745 patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()] 746 747 try: 748 subprocess.check_output([patchelf] + patchelf_args + [fname]) 749 except subprocess.CalledProcessError as reason: 750 eprint("warning: failed to call patchelf:", reason) 751 return 752 753 def rustc_stamp(self): 754 """Return the path for .rustc-stamp at the given stage 755 756 >>> rb = RustBuild() 757 >>> rb.build = "host" 758 >>> rb.build_dir = "build" 759 >>> expected = os.path.join("build", "host", "stage0", ".rustc-stamp") 760 >>> assert rb.rustc_stamp() == expected, rb.rustc_stamp() 761 """ 762 return os.path.join(self.bin_root(), '.rustc-stamp') 763 764 def program_out_of_date(self, stamp_path, key): 765 """Check if the given program stamp is out of date""" 766 if not os.path.exists(stamp_path) or self.clean: 767 return True 768 with open(stamp_path, 'r') as stamp: 769 return key != stamp.read() 770 771 def bin_root(self): 772 """Return the binary root directory for the given stage 773 774 >>> rb = RustBuild() 775 >>> rb.build = "devel" 776 >>> expected = os.path.abspath(os.path.join("build", "devel", "stage0")) 777 >>> assert rb.bin_root() == expected, rb.bin_root() 778 """ 779 subdir = "stage0" 780 return os.path.join(self.build_dir, self.build, subdir) 781 782 def get_toml(self, key, section=None): 783 """Returns the value of the given key in config.toml, otherwise returns None 784 785 >>> rb = RustBuild() 786 >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"' 787 >>> rb.get_toml("key2") 788 'value2' 789 790 If the key does not exist, the result is None: 791 792 >>> rb.get_toml("key3") is None 793 True 794 795 Optionally also matches the section the key appears in 796 797 >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"' 798 >>> rb.get_toml('key', 'a') 799 'value1' 800 >>> rb.get_toml('key', 'b') 801 'value2' 802 >>> rb.get_toml('key', 'c') is None 803 True 804 805 >>> rb.config_toml = 'key1 = true' 806 >>> rb.get_toml("key1") 807 'true' 808 """ 809 return RustBuild.get_toml_static(self.config_toml, key, section) 810 811 @staticmethod 812 def get_toml_static(config_toml, key, section=None): 813 cur_section = None 814 for line in config_toml.splitlines(): 815 section_match = re.match(r'^\s*\[(.*)\]\s*$', line) 816 if section_match is not None: 817 cur_section = section_match.group(1) 818 819 match = re.match(r'^{}\s*=(.*)$'.format(key), line) 820 if match is not None: 821 value = match.group(1) 822 if section is None or section == cur_section: 823 return RustBuild.get_string(value) or value.strip() 824 return None 825 826 def cargo(self): 827 """Return config path for cargo""" 828 return self.program_config('cargo') 829 830 def rustc(self): 831 """Return config path for rustc""" 832 return self.program_config('rustc') 833 834 def program_config(self, program): 835 """Return config path for the given program at the given stage 836 837 >>> rb = RustBuild() 838 >>> rb.config_toml = 'rustc = "rustc"\\n' 839 >>> rb.program_config('rustc') 840 'rustc' 841 >>> rb.config_toml = '' 842 >>> cargo_path = rb.program_config('cargo') 843 >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(), 844 ... "bin", "cargo") 845 True 846 """ 847 config = self.get_toml(program) 848 if config: 849 return os.path.expanduser(config) 850 return os.path.join(self.bin_root(), "bin", "{}{}".format(program, EXE_SUFFIX)) 851 852 @staticmethod 853 def get_string(line): 854 """Return the value between double quotes 855 856 >>> RustBuild.get_string(' "devel" ') 857 'devel' 858 >>> RustBuild.get_string(" 'devel' ") 859 'devel' 860 >>> RustBuild.get_string('devel') is None 861 True 862 >>> RustBuild.get_string(' "devel ') 863 '' 864 """ 865 start = line.find('"') 866 if start != -1: 867 end = start + 1 + line[start + 1:].find('"') 868 return line[start + 1:end] 869 start = line.find('\'') 870 if start != -1: 871 end = start + 1 + line[start + 1:].find('\'') 872 return line[start + 1:end] 873 return None 874 875 def bootstrap_binary(self): 876 """Return the path of the bootstrap binary 877 878 >>> rb = RustBuild() 879 >>> rb.build_dir = "build" 880 >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap", 881 ... "debug", "bootstrap") 882 True 883 """ 884 return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap") 885 886 def build_bootstrap(self): 887 """Build bootstrap""" 888 env = os.environ.copy() 889 if "GITHUB_ACTIONS" in env: 890 print("::group::Building bootstrap") 891 else: 892 eprint("Building bootstrap") 893 894 args = self.build_bootstrap_cmd(env) 895 # Run this from the source directory so cargo finds .cargo/config 896 run(args, env=env, verbose=self.verbose, cwd=self.rust_root) 897 898 if "GITHUB_ACTIONS" in env: 899 print("::endgroup::") 900 901 def build_bootstrap_cmd(self, env): 902 """For tests.""" 903 build_dir = os.path.join(self.build_dir, "bootstrap") 904 if self.clean and os.path.exists(build_dir): 905 shutil.rmtree(build_dir) 906 # `CARGO_BUILD_TARGET` breaks bootstrap build. 907 # See also: <https://github.com/rust-lang/rust/issues/70208>. 908 if "CARGO_BUILD_TARGET" in env: 909 del env["CARGO_BUILD_TARGET"] 910 env["CARGO_TARGET_DIR"] = build_dir 911 env["RUSTC"] = self.rustc() 912 env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ 913 (os.pathsep + env["LD_LIBRARY_PATH"]) \ 914 if "LD_LIBRARY_PATH" in env else "" 915 env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ 916 (os.pathsep + env["DYLD_LIBRARY_PATH"]) \ 917 if "DYLD_LIBRARY_PATH" in env else "" 918 env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ 919 (os.pathsep + env["LIBRARY_PATH"]) \ 920 if "LIBRARY_PATH" in env else "" 921 env["LIBPATH"] = os.path.join(self.bin_root(), "lib") + \ 922 (os.pathsep + env["LIBPATH"]) \ 923 if "LIBPATH" in env else "" 924 925 # Export Stage0 snapshot compiler related env variables 926 build_section = "target.{}".format(self.build) 927 host_triple_sanitized = self.build.replace("-", "_") 928 var_data = { 929 "CC": "cc", "CXX": "cxx", "LD": "linker", "AR": "ar", "RANLIB": "ranlib" 930 } 931 for var_name, toml_key in var_data.items(): 932 toml_val = self.get_toml(toml_key, build_section) 933 if toml_val is not None: 934 env["{}_{}".format(var_name, host_triple_sanitized)] = toml_val 935 936 # preserve existing RUSTFLAGS 937 env.setdefault("RUSTFLAGS", "") 938 939 target_features = [] 940 if self.get_toml("crt-static", build_section) == "true": 941 target_features += ["+crt-static"] 942 elif self.get_toml("crt-static", build_section) == "false": 943 target_features += ["-crt-static"] 944 if target_features: 945 env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features)) 946 target_linker = self.get_toml("linker", build_section) 947 if target_linker is not None: 948 env["RUSTFLAGS"] += " -C linker=" + target_linker 949 env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes" 950 if self.warnings == "default": 951 deny_warnings = self.get_toml("deny-warnings", "rust") != "false" 952 else: 953 deny_warnings = self.warnings == "deny" 954 if deny_warnings: 955 env["RUSTFLAGS"] += " -Dwarnings" 956 957 env["PATH"] = os.path.join(self.bin_root(), "bin") + \ 958 os.pathsep + env["PATH"] 959 if not os.path.isfile(self.cargo()): 960 raise Exception("no cargo executable found at `{}`".format( 961 self.cargo())) 962 args = [self.cargo(), "build", "--manifest-path", 963 os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")] 964 args.extend("--verbose" for _ in range(self.verbose)) 965 if self.use_locked_deps: 966 args.append("--locked") 967 if self.use_vendored_sources: 968 args.append("--frozen") 969 if self.get_toml("metrics", "build"): 970 args.append("--features") 971 args.append("build-metrics") 972 if self.json_output: 973 args.append("--message-format=json") 974 if self.color == "always": 975 args.append("--color=always") 976 elif self.color == "never": 977 args.append("--color=never") 978 try: 979 args += env["CARGOFLAGS"].split() 980 except KeyError: 981 pass 982 983 return args 984 985 def build_triple(self): 986 """Build triple as in LLVM 987 988 Note that `default_build_triple` is moderately expensive, 989 so use `self.build` where possible. 990 """ 991 config = self.get_toml('build') 992 return config or default_build_triple(self.verbose) 993 994 def check_vendored_status(self): 995 """Check that vendoring is configured properly""" 996 # keep this consistent with the equivalent check in rustbuild: 997 # https://github.com/rust-lang/rust/blob/a8a33cf27166d3eabaffc58ed3799e054af3b0c6/src/bootstrap/lib.rs#L399-L405 998 if 'SUDO_USER' in os.environ and not self.use_vendored_sources: 999 if os.getuid() == 0: 1000 self.use_vendored_sources = True 1001 eprint('info: looks like you\'re trying to run this command as root') 1002 eprint(' and so in order to preserve your $HOME this will now') 1003 eprint(' use vendored sources by default.') 1004 1005 cargo_dir = os.path.join(self.rust_root, '.cargo') 1006 if self.use_vendored_sources: 1007 vendor_dir = os.path.join(self.rust_root, 'vendor') 1008 if not os.path.exists(vendor_dir): 1009 sync_dirs = "--sync ./src/tools/cargo/Cargo.toml " \ 1010 "--sync ./src/tools/rust-analyzer/Cargo.toml " \ 1011 "--sync ./compiler/rustc_codegen_cranelift/Cargo.toml " \ 1012 "--sync ./src/bootstrap/Cargo.toml " 1013 eprint('error: vendoring required, but vendor directory does not exist.') 1014 eprint(' Run `cargo vendor {}` to initialize the ' 1015 'vendor directory.'.format(sync_dirs)) 1016 eprint('Alternatively, use the pre-vendored `rustc-src` dist component.') 1017 raise Exception("{} not found".format(vendor_dir)) 1018 1019 if not os.path.exists(cargo_dir): 1020 eprint('error: vendoring required, but .cargo/config does not exist.') 1021 raise Exception("{} not found".format(cargo_dir)) 1022 else: 1023 if os.path.exists(cargo_dir): 1024 shutil.rmtree(cargo_dir) 1025 1026def parse_args(args): 1027 """Parse the command line arguments that the python script needs.""" 1028 parser = argparse.ArgumentParser(add_help=False) 1029 parser.add_argument('-h', '--help', action='store_true') 1030 parser.add_argument('--config') 1031 parser.add_argument('--build-dir') 1032 parser.add_argument('--build') 1033 parser.add_argument('--color', choices=['always', 'never', 'auto']) 1034 parser.add_argument('--clean', action='store_true') 1035 parser.add_argument('--json-output', action='store_true') 1036 parser.add_argument('--warnings', choices=['deny', 'warn', 'default'], default='default') 1037 parser.add_argument('-v', '--verbose', action='count', default=0) 1038 1039 return parser.parse_known_args(args)[0] 1040 1041def bootstrap(args): 1042 """Configure, fetch, build and run the initial bootstrap""" 1043 rust_root = os.path.abspath(os.path.join(__file__, '../../..')) 1044 1045 if not os.path.exists(os.path.join(rust_root, '.git')) and \ 1046 os.path.exists(os.path.join(rust_root, '.github')): 1047 eprint("warn: Looks like you are trying to bootstrap Rust from a source that is neither a " 1048 "git clone nor distributed tarball.\nThis build may fail due to missing submodules " 1049 "unless you put them in place manually.") 1050 1051 # Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, 1052 # then `config.toml` in the root directory. 1053 toml_path = args.config or os.getenv('RUST_BOOTSTRAP_CONFIG') 1054 using_default_path = toml_path is None 1055 if using_default_path: 1056 toml_path = 'config.toml' 1057 if not os.path.exists(toml_path): 1058 toml_path = os.path.join(rust_root, toml_path) 1059 1060 # Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, 1061 # but not if `config.toml` hasn't been created. 1062 if not using_default_path or os.path.exists(toml_path): 1063 with open(toml_path) as config: 1064 config_toml = config.read() 1065 else: 1066 config_toml = '' 1067 1068 profile = RustBuild.get_toml_static(config_toml, 'profile') 1069 if profile is not None: 1070 # Allows creating alias for profile names, allowing 1071 # profiles to be renamed while maintaining back compatibility 1072 # Keep in sync with `profile_aliases` in config.rs 1073 profile_aliases = { 1074 "user": "dist" 1075 } 1076 include_file = 'config.{}.toml'.format(profile_aliases.get(profile) or profile) 1077 include_dir = os.path.join(rust_root, 'src', 'bootstrap', 'defaults') 1078 include_path = os.path.join(include_dir, include_file) 1079 # HACK: This works because `self.get_toml()` returns the first match it finds for a 1080 # specific key, so appending our defaults at the end allows the user to override them 1081 with open(include_path) as included_toml: 1082 config_toml += os.linesep + included_toml.read() 1083 1084 # Configure initial bootstrap 1085 build = RustBuild(config_toml, args) 1086 build.check_vendored_status() 1087 1088 if not os.path.exists(build.build_dir): 1089 os.makedirs(build.build_dir) 1090 1091 # Fetch/build the bootstrap 1092 build.download_toolchain() 1093 sys.stdout.flush() 1094 build.build_bootstrap() 1095 sys.stdout.flush() 1096 1097 # Run the bootstrap 1098 args = [build.bootstrap_binary()] 1099 args.extend(sys.argv[1:]) 1100 env = os.environ.copy() 1101 env["BOOTSTRAP_PARENT_ID"] = str(os.getpid()) 1102 env["BOOTSTRAP_PYTHON"] = sys.executable 1103 run(args, env=env, verbose=build.verbose, is_bootstrap=True) 1104 1105 1106def main(): 1107 """Entry point for the bootstrap process""" 1108 start_time = time() 1109 1110 # x.py help <cmd> ... 1111 if len(sys.argv) > 1 and sys.argv[1] == 'help': 1112 sys.argv[1] = '-h' 1113 1114 args = parse_args(sys.argv) 1115 help_triggered = args.help or len(sys.argv) == 1 1116 1117 # If the user is asking for help, let them know that the whole download-and-build 1118 # process has to happen before anything is printed out. 1119 if help_triggered: 1120 eprint( 1121 "info: Downloading and building bootstrap before processing --help command.\n" 1122 " See src/bootstrap/README.md for help with common commands.") 1123 1124 exit_code = 0 1125 success_word = "successfully" 1126 try: 1127 bootstrap(args) 1128 except (SystemExit, KeyboardInterrupt) as error: 1129 if hasattr(error, 'code') and isinstance(error.code, int): 1130 exit_code = error.code 1131 else: 1132 exit_code = 1 1133 eprint(error) 1134 success_word = "unsuccessfully" 1135 1136 if not help_triggered: 1137 eprint("Build completed", success_word, "in", format_build_time(time() - start_time)) 1138 sys.exit(exit_code) 1139 1140 1141if __name__ == '__main__': 1142 main()