fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #103046 - JanBeh:PR_clarify_cmp_terminology, r=workingjubilee3bcad65fbf6

master
43.0 KiB1142 linesraw
1from __future__ import absolute_import, division, print_function
2import argparse
3import contextlib
4import datetime
5import hashlib
6import json
7import os
8import re
9import shutil
10import subprocess
11import sys
12import tarfile
13import tempfile
14
15from time import time
16from multiprocessing import Pool, cpu_count
17
18try:
19    import lzma
20except ImportError:
21    lzma = None
22
23def platform_is_win32():
24    return sys.platform == 'win32'
25
26if platform_is_win32():
27    EXE_SUFFIX = ".exe"
28else:
29    EXE_SUFFIX = ""
30
31def get_cpus():
32    if hasattr(os, "sched_getaffinity"):
33        return len(os.sched_getaffinity(0))
34    if hasattr(os, "cpu_count"):
35        cpus = os.cpu_count()
36        if cpus is not None:
37            return cpus
38    try:
39        return cpu_count()
40    except NotImplementedError:
41        return 1
42
43
44def eprint(*args, **kwargs):
45    kwargs["file"] = sys.stderr
46    print(*args, **kwargs)
47
48
49def get(base, url, path, checksums, verbose=False):
50    with tempfile.NamedTemporaryFile(delete=False) as temp_file:
51        temp_path = temp_file.name
52
53    try:
54        if url not in checksums:
55            raise RuntimeError(("src/stage0.json doesn't contain a checksum for {}. "
56                                "Pre-built artifacts might not be available for this "
57                                "target at this time, see https://doc.rust-lang.org/nightly"
58                                "/rustc/platform-support.html for more information.")
59                                .format(url))
60        sha256 = checksums[url]
61        if os.path.exists(path):
62            if verify(path, sha256, False):
63                if verbose:
64                    eprint("using already-download file", path)
65                return
66            else:
67                if verbose:
68                    eprint("ignoring already-download file",
69                        path, "due to failed verification")
70                os.unlink(path)
71        download(temp_path, "{}/{}".format(base, url), True, verbose)
72        if not verify(temp_path, sha256, verbose):
73            raise RuntimeError("failed verification")
74        if verbose:
75            eprint("moving {} to {}".format(temp_path, path))
76        shutil.move(temp_path, path)
77    finally:
78        if os.path.isfile(temp_path):
79            if verbose:
80                eprint("removing", temp_path)
81            os.unlink(temp_path)
82
83
84def download(path, url, probably_big, verbose):
85    for _ in range(4):
86        try:
87            _download(path, url, probably_big, verbose, True)
88            return
89        except RuntimeError:
90            eprint("\nspurious failure, trying again")
91    _download(path, url, probably_big, verbose, False)
92
93
94def _download(path, url, probably_big, verbose, exception):
95    # Try to use curl (potentially available on win32
96    #    https://devblogs.microsoft.com/commandline/tar-and-curl-come-to-windows/)
97    # If an error occurs:
98    #  - If we are on win32 fallback to powershell
99    #  - Otherwise raise the error if appropriate
100    if probably_big or verbose:
101        eprint("downloading {}".format(url))
102
103    try:
104        if (probably_big or verbose) and "GITHUB_ACTIONS" not in os.environ:
105            option = "-#"
106        else:
107            option = "-s"
108        # If curl is not present on Win32, we should not sys.exit
109        #   but raise `CalledProcessError` or `OSError` instead
110        require(["curl", "--version"], exception=platform_is_win32())
111        run(["curl", option,
112            "-L", # Follow redirect.
113            "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
114            "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
115            "-o", path,
116            "--retry", "3", "-SRf", url],
117            verbose=verbose,
118            exception=True, # Will raise RuntimeError on failure
119        )
120    except (subprocess.CalledProcessError, OSError, RuntimeError):
121        # see http://serverfault.com/questions/301128/how-to-download
122        if platform_is_win32():
123            run_powershell([
124                 "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
125                 "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
126                verbose=verbose,
127                exception=exception)
128        # Check if the RuntimeError raised by run(curl) should be silenced
129        elif verbose or exception:
130            raise
131
132
133def verify(path, expected, verbose):
134    """Check if the sha256 sum of the given path is valid"""
135    if verbose:
136        eprint("verifying", path)
137    with open(path, "rb") as source:
138        found = hashlib.sha256(source.read()).hexdigest()
139    verified = found == expected
140    if not verified:
141        eprint("invalid checksum:\n"
142              "    found:    {}\n"
143              "    expected: {}".format(found, expected))
144    return verified
145
146
147def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
148    """Unpack the given tarball file"""
149    eprint("extracting", tarball)
150    fname = os.path.basename(tarball).replace(tarball_suffix, "")
151    with contextlib.closing(tarfile.open(tarball)) as tar:
152        for member in tar.getnames():
153            if "/" not in member:
154                continue
155            name = member.replace(fname + "/", "", 1)
156            if match is not None and not name.startswith(match):
157                continue
158            name = name[len(match) + 1:]
159
160            dst_path = os.path.join(dst, name)
161            if verbose:
162                eprint("  extracting", member)
163            tar.extract(member, dst)
164            src_path = os.path.join(dst, member)
165            if os.path.isdir(src_path) and os.path.exists(dst_path):
166                continue
167            shutil.move(src_path, dst_path)
168    shutil.rmtree(os.path.join(dst, fname))
169
170
171def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs):
172    """Run a child program in a new process"""
173    if verbose:
174        eprint("running: " + ' '.join(args))
175    sys.stdout.flush()
176    # Ensure that the .exe is used on Windows just in case a Linux ELF has been
177    # compiled in the same directory.
178    if os.name == 'nt' and not args[0].endswith('.exe'):
179        args[0] += '.exe'
180    # Use Popen here instead of call() as it apparently allows powershell on
181    # Windows to not lock up waiting for input presumably.
182    ret = subprocess.Popen(args, **kwargs)
183    code = ret.wait()
184    if code != 0:
185        err = "failed to run: " + ' '.join(args)
186        if verbose or exception:
187            raise RuntimeError(err)
188        # For most failures, we definitely do want to print this error, or the user will have no
189        # idea what went wrong. But when we've successfully built bootstrap and it failed, it will
190        # have already printed an error above, so there's no need to print the exact command we're
191        # running.
192        if is_bootstrap:
193            sys.exit(1)
194        else:
195            sys.exit(err)
196
197def run_powershell(script, *args, **kwargs):
198    """Run a powershell script"""
199    run(["PowerShell.exe", "/nologo", "-Command"] + script, *args, **kwargs)
200
201
202def require(cmd, exit=True, exception=False):
203    '''Run a command, returning its output.
204    On error,
205        If `exception` is `True`, raise the error
206        Otherwise If `exit` is `True`, exit the process
207        Else return None.'''
208    try:
209        return subprocess.check_output(cmd).strip()
210    except (subprocess.CalledProcessError, OSError) as exc:
211        if exception:
212            raise
213        elif exit:
214            eprint("error: unable to run `{}`: {}".format(' '.join(cmd), exc))
215            eprint("Please make sure it's installed and in the path.")
216            sys.exit(1)
217        return None
218
219
220
221def format_build_time(duration):
222    """Return a nicer format for build time
223
224    >>> format_build_time('300')
225    '0:05:00'
226    """
227    return str(datetime.timedelta(seconds=int(duration)))
228
229
230def default_build_triple(verbose):
231    """Build triple as in LLVM"""
232    # If we're on Windows and have an existing `rustc` toolchain, use `rustc --version --verbose`
233    # to find our host target triple. This fixes an issue with Windows builds being detected
234    # as GNU instead of MSVC.
235    # Otherwise, detect it via `uname`
236    default_encoding = sys.getdefaultencoding()
237
238    if platform_is_win32():
239        try:
240            version = subprocess.check_output(["rustc", "--version", "--verbose"],
241                    stderr=subprocess.DEVNULL)
242            version = version.decode(default_encoding)
243            host = next(x for x in version.split('\n') if x.startswith("host: "))
244            triple = host.split("host: ")[1]
245            if verbose:
246                eprint("detected default triple {} from pre-installed rustc".format(triple))
247            return triple
248        except Exception as e:
249            if verbose:
250                eprint("pre-installed rustc not detected: {}".format(e))
251                eprint("falling back to auto-detect")
252
253    required = not platform_is_win32()
254    uname = require(["uname", "-smp"], exit=required)
255
256    # If we do not have `uname`, assume Windows.
257    if uname is None:
258        return 'x86_64-pc-windows-msvc'
259
260    kernel, cputype, processor = uname.decode(default_encoding).split(maxsplit=2)
261
262    # The goal here is to come up with the same triple as LLVM would,
263    # at least for the subset of platforms we're willing to target.
264    kerneltype_mapper = {
265        'Darwin': 'apple-darwin',
266        'DragonFly': 'unknown-dragonfly',
267        'FreeBSD': 'unknown-freebsd',
268        'Haiku': 'unknown-haiku',
269        'NetBSD': 'unknown-netbsd',
270        'OpenBSD': 'unknown-openbsd',
271        'GNU': 'unknown-hurd',
272    }
273
274    # Consider the direct transformation first and then the special cases
275    if kernel in kerneltype_mapper:
276        kernel = kerneltype_mapper[kernel]
277    elif kernel == 'Linux':
278        # Apple doesn't support `-o` so this can't be used in the combined
279        # uname invocation above
280        ostype = require(["uname", "-o"], exit=required).decode(default_encoding)
281        if ostype == 'Android':
282            kernel = 'linux-android'
283        else:
284            kernel = 'unknown-linux-gnu'
285    elif kernel == 'SunOS':
286        kernel = 'pc-solaris'
287        # On Solaris, uname -m will return a machine classification instead
288        # of a cpu type, so uname -p is recommended instead.  However, the
289        # output from that option is too generic for our purposes (it will
290        # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
291        # must be used instead.
292        cputype = require(['isainfo', '-k']).decode(default_encoding)
293        # sparc cpus have sun as a target vendor
294        if 'sparc' in cputype:
295            kernel = 'sun-solaris'
296    elif kernel.startswith('MINGW'):
297        # msys' `uname` does not print gcc configuration, but prints msys
298        # configuration. so we cannot believe `uname -m`:
299        # msys1 is always i686 and msys2 is always x86_64.
300        # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
301        # MINGW64 on x86_64.
302        kernel = 'pc-windows-gnu'
303        cputype = 'i686'
304        if os.environ.get('MSYSTEM') == 'MINGW64':
305            cputype = 'x86_64'
306    elif kernel.startswith('MSYS'):
307        kernel = 'pc-windows-gnu'
308    elif kernel.startswith('CYGWIN_NT'):
309        cputype = 'i686'
310        if kernel.endswith('WOW64'):
311            cputype = 'x86_64'
312        kernel = 'pc-windows-gnu'
313    elif platform_is_win32():
314        # Some Windows platforms might have a `uname` command that returns a
315        # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
316        # these cases, fall back to using sys.platform.
317        return 'x86_64-pc-windows-msvc'
318    elif kernel == 'AIX':
319        # `uname -m` returns the machine ID rather than machine hardware on AIX,
320        # so we are unable to use cputype to form triple. AIX 7.2 and
321        # above supports 32-bit and 64-bit mode simultaneously and `uname -p`
322        # returns `powerpc`, however we only supports `powerpc64-ibm-aix` in
323        # rust on AIX. For above reasons, kerneltype_mapper and cputype_mapper
324        # are not used to infer AIX's triple.
325        return 'powerpc64-ibm-aix'
326    else:
327        err = "unknown OS type: {}".format(kernel)
328        sys.exit(err)
329
330    if cputype in ['powerpc', 'riscv'] and kernel == 'unknown-freebsd':
331        cputype = subprocess.check_output(
332              ['uname', '-p']).strip().decode(default_encoding)
333    cputype_mapper = {
334        'BePC': 'i686',
335        'aarch64': 'aarch64',
336        'aarch64eb': 'aarch64',
337        'amd64': 'x86_64',
338        'arm64': 'aarch64',
339        'i386': 'i686',
340        'i486': 'i686',
341        'i686': 'i686',
342        'i686-AT386': 'i686',
343        'i786': 'i686',
344        'loongarch64': 'loongarch64',
345        'm68k': 'm68k',
346        'csky': 'csky',
347        'powerpc': 'powerpc',
348        'powerpc64': 'powerpc64',
349        'powerpc64le': 'powerpc64le',
350        'ppc': 'powerpc',
351        'ppc64': 'powerpc64',
352        'ppc64le': 'powerpc64le',
353        'riscv64': 'riscv64gc',
354        's390x': 's390x',
355        'x64': 'x86_64',
356        'x86': 'i686',
357        'x86-64': 'x86_64',
358        'x86_64': 'x86_64'
359    }
360
361    # Consider the direct transformation first and then the special cases
362    if cputype in cputype_mapper:
363        cputype = cputype_mapper[cputype]
364    elif cputype in {'xscale', 'arm'}:
365        cputype = 'arm'
366        if kernel == 'linux-android':
367            kernel = 'linux-androideabi'
368        elif kernel == 'unknown-freebsd':
369            cputype = processor
370            kernel = 'unknown-freebsd'
371    elif cputype == 'armv6l':
372        cputype = 'arm'
373        if kernel == 'linux-android':
374            kernel = 'linux-androideabi'
375        else:
376            kernel += 'eabihf'
377    elif cputype in {'armv7l', 'armv8l'}:
378        cputype = 'armv7'
379        if kernel == 'linux-android':
380            kernel = 'linux-androideabi'
381        else:
382            kernel += 'eabihf'
383    elif cputype == 'mips':
384        if sys.byteorder == 'big':
385            cputype = 'mips'
386        elif sys.byteorder == 'little':
387            cputype = 'mipsel'
388        else:
389            raise ValueError("unknown byteorder: {}".format(sys.byteorder))
390    elif cputype == 'mips64':
391        if sys.byteorder == 'big':
392            cputype = 'mips64'
393        elif sys.byteorder == 'little':
394            cputype = 'mips64el'
395        else:
396            raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
397        # only the n64 ABI is supported, indicate it
398        kernel += 'abi64'
399    elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
400        pass
401    else:
402        err = "unknown cpu type: {}".format(cputype)
403        sys.exit(err)
404
405    return "{}-{}".format(cputype, kernel)
406
407
408@contextlib.contextmanager
409def output(filepath):
410    tmp = filepath + '.tmp'
411    with open(tmp, 'w') as f:
412        yield f
413    try:
414        if os.path.exists(filepath):
415            os.remove(filepath)  # PermissionError/OSError on Win32 if in use
416    except OSError:
417        shutil.copy2(tmp, filepath)
418        os.remove(tmp)
419        return
420    os.rename(tmp, filepath)
421
422
423class Stage0Toolchain:
424    def __init__(self, stage0_payload):
425        self.date = stage0_payload["date"]
426        self.version = stage0_payload["version"]
427
428    def channel(self):
429        return self.version + "-" + self.date
430
431
432class DownloadInfo:
433    """A helper class that can be pickled into a parallel subprocess"""
434
435    def __init__(
436        self,
437        base_download_url,
438        download_path,
439        bin_root,
440        tarball_path,
441        tarball_suffix,
442        checksums_sha256,
443        pattern,
444        verbose,
445    ):
446        self.base_download_url = base_download_url
447        self.download_path = download_path
448        self.bin_root = bin_root
449        self.tarball_path = tarball_path
450        self.tarball_suffix = tarball_suffix
451        self.checksums_sha256 = checksums_sha256
452        self.pattern = pattern
453        self.verbose = verbose
454
455def download_component(download_info):
456    if not os.path.exists(download_info.tarball_path):
457        get(
458            download_info.base_download_url,
459            download_info.download_path,
460            download_info.tarball_path,
461            download_info.checksums_sha256,
462            verbose=download_info.verbose,
463        )
464
465def unpack_component(download_info):
466    unpack(
467        download_info.tarball_path,
468        download_info.tarball_suffix,
469        download_info.bin_root,
470        match=download_info.pattern,
471        verbose=download_info.verbose,
472    )
473
474class FakeArgs:
475    """Used for unit tests to avoid updating all call sites"""
476    def __init__(self):
477        self.build = ''
478        self.build_dir = ''
479        self.clean = False
480        self.verbose = False
481        self.json_output = False
482        self.color = 'auto'
483        self.warnings = 'default'
484
485class RustBuild(object):
486    """Provide all the methods required to build Rust"""
487    def __init__(self, config_toml="", args=None):
488        if args is None:
489            args = FakeArgs()
490        self.git_version = None
491        self.nix_deps_dir = None
492        self._should_fix_bins_and_dylibs = None
493        self.rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
494
495        self.config_toml = config_toml
496
497        self.clean = args.clean
498        self.json_output = args.json_output
499        self.verbose = args.verbose
500        self.color = args.color
501        self.warnings = args.warnings
502
503        config_verbose_count = self.get_toml('verbose', 'build')
504        if config_verbose_count is not None:
505            self.verbose = max(self.verbose, int(config_verbose_count))
506
507        self.use_vendored_sources = self.get_toml('vendor', 'build') == 'true'
508        self.use_locked_deps = self.get_toml('locked-deps', 'build') == 'true'
509
510        build_dir = args.build_dir or self.get_toml('build-dir', 'build') or 'build'
511        self.build_dir = os.path.abspath(build_dir)
512
513        with open(os.path.join(self.rust_root, "src", "stage0.json")) as f:
514            data = json.load(f)
515        self.checksums_sha256 = data["checksums_sha256"]
516        self.stage0_compiler = Stage0Toolchain(data["compiler"])
517        self.download_url = os.getenv("RUSTUP_DIST_SERVER") or data["config"]["dist_server"]
518
519        self.build = args.build or self.build_triple()
520
521
522    def download_toolchain(self):
523        """Fetch the build system for Rust, written in Rust
524
525        This method will build a cache directory, then it will fetch the
526        tarball which has the stage0 compiler used to then bootstrap the Rust
527        compiler itself.
528
529        Each downloaded tarball is extracted, after that, the script
530        will move all the content to the right place.
531        """
532        rustc_channel = self.stage0_compiler.version
533        bin_root = self.bin_root()
534
535        key = self.stage0_compiler.date
536        if self.rustc().startswith(bin_root) and \
537                (not os.path.exists(self.rustc()) or
538                 self.program_out_of_date(self.rustc_stamp(), key)):
539            if os.path.exists(bin_root):
540                # HACK: On Windows, we can't delete rust-analyzer-proc-macro-server while it's
541                # running. Kill it.
542                if platform_is_win32():
543                    print("Killing rust-analyzer-proc-macro-srv before deleting stage0 toolchain")
544                    regex =  '{}\\\\(host|{})\\\\stage0\\\\libexec'.format(
545                        os.path.basename(self.build_dir),
546                        self.build
547                    )
548                    script = (
549                        # NOTE: can't use `taskkill` or `Get-Process -Name` because they error if
550                        # the server isn't running.
551                        'Get-Process | ' +
552                        'Where-Object {$_.Name -eq "rust-analyzer-proc-macro-srv"} |' +
553                        'Where-Object {{$_.Path -match "{}"}} |'.format(regex) +
554                        'Stop-Process'
555                    )
556                    run_powershell([script])
557                shutil.rmtree(bin_root)
558
559            key = self.stage0_compiler.date
560            cache_dst = os.path.join(self.build_dir, "cache")
561            rustc_cache = os.path.join(cache_dst, key)
562            if not os.path.exists(rustc_cache):
563                os.makedirs(rustc_cache)
564
565            tarball_suffix = '.tar.gz' if lzma is None else '.tar.xz'
566
567            toolchain_suffix = "{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
568
569            tarballs_to_download = [
570                ("rust-std-{}".format(toolchain_suffix), "rust-std-{}".format(self.build)),
571                ("rustc-{}".format(toolchain_suffix), "rustc"),
572                ("cargo-{}".format(toolchain_suffix), "cargo"),
573            ]
574
575            tarballs_download_info = [
576                DownloadInfo(
577                    base_download_url=self.download_url,
578                    download_path="dist/{}/{}".format(self.stage0_compiler.date, filename),
579                    bin_root=self.bin_root(),
580                    tarball_path=os.path.join(rustc_cache, filename),
581                    tarball_suffix=tarball_suffix,
582                    checksums_sha256=self.checksums_sha256,
583                    pattern=pattern,
584                    verbose=self.verbose,
585                )
586                for filename, pattern in tarballs_to_download
587            ]
588
589            # Download the components serially to show the progress bars properly.
590            for download_info in tarballs_download_info:
591                download_component(download_info)
592
593            # Unpack the tarballs in parallle.
594            # In Python 2.7, Pool cannot be used as a context manager.
595            pool_size = min(len(tarballs_download_info), get_cpus())
596            if self.verbose:
597                print('Choosing a pool size of', pool_size, 'for the unpacking of the tarballs')
598            p = Pool(pool_size)
599            try:
600                p.map(unpack_component, tarballs_download_info)
601            finally:
602                p.close()
603            p.join()
604
605            if self.should_fix_bins_and_dylibs():
606                self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
607
608                self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
609                self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
610                self.fix_bin_or_dylib("{}/libexec/rust-analyzer-proc-macro-srv".format(bin_root))
611                lib_dir = "{}/lib".format(bin_root)
612                for lib in os.listdir(lib_dir):
613                    if lib.endswith(".so"):
614                        self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
615
616            with output(self.rustc_stamp()) as rust_stamp:
617                rust_stamp.write(key)
618
619    def _download_component_helper(
620        self, filename, pattern, tarball_suffix, rustc_cache,
621    ):
622        key = self.stage0_compiler.date
623
624        tarball = os.path.join(rustc_cache, filename)
625        if not os.path.exists(tarball):
626            get(
627                self.download_url,
628                "dist/{}/{}".format(key, filename),
629                tarball,
630                self.checksums_sha256,
631                verbose=self.verbose,
632            )
633        unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
634
635    def should_fix_bins_and_dylibs(self):
636        """Whether or not `fix_bin_or_dylib` needs to be run; can only be True
637        on NixOS or if config.toml has `build.patch-binaries-for-nix` set.
638        """
639        if self._should_fix_bins_and_dylibs is not None:
640            return self._should_fix_bins_and_dylibs
641
642        def get_answer():
643            default_encoding = sys.getdefaultencoding()
644            try:
645                ostype = subprocess.check_output(
646                    ['uname', '-s']).strip().decode(default_encoding)
647            except subprocess.CalledProcessError:
648                return False
649            except OSError as reason:
650                if getattr(reason, 'winerror', None) is not None:
651                    return False
652                raise reason
653
654            if ostype != "Linux":
655                return False
656
657            # If the user has explicitly indicated whether binaries should be
658            # patched for Nix, then don't check for NixOS.
659            if self.get_toml("patch-binaries-for-nix", "build") == "true":
660                return True
661            if self.get_toml("patch-binaries-for-nix", "build") == "false":
662                return False
663
664            # Use `/etc/os-release` instead of `/etc/NIXOS`.
665            # The latter one does not exist on NixOS when using tmpfs as root.
666            try:
667                with open("/etc/os-release", "r") as f:
668                    is_nixos = any(ln.strip() in ("ID=nixos", "ID='nixos'", 'ID="nixos"')
669                                   for ln in f)
670            except FileNotFoundError:
671                is_nixos = False
672
673            # If not on NixOS, then warn if user seems to be atop Nix shell
674            if not is_nixos:
675                in_nix_shell = os.getenv('IN_NIX_SHELL')
676                if in_nix_shell:
677                    eprint("The IN_NIX_SHELL environment variable is `{}`;".format(in_nix_shell),
678                          "you may need to set `patch-binaries-for-nix=true` in config.toml")
679
680            return is_nixos
681
682        answer = self._should_fix_bins_and_dylibs = get_answer()
683        if answer:
684            eprint("info: You seem to be using Nix.")
685        return answer
686
687    def fix_bin_or_dylib(self, fname):
688        """Modifies the interpreter section of 'fname' to fix the dynamic linker,
689        or the RPATH section, to fix the dynamic library search path
690
691        This method is only required on NixOS and uses the PatchELF utility to
692        change the interpreter/RPATH of ELF executables.
693
694        Please see https://nixos.org/patchelf.html for more information
695        """
696        assert self._should_fix_bins_and_dylibs is True
697        eprint("attempting to patch", fname)
698
699        # Only build `.nix-deps` once.
700        nix_deps_dir = self.nix_deps_dir
701        if not nix_deps_dir:
702            # Run `nix-build` to "build" each dependency (which will likely reuse
703            # the existing `/nix/store` copy, or at most download a pre-built copy).
704            #
705            # Importantly, we create a gc-root called `.nix-deps` in the `build/`
706            # directory, but still reference the actual `/nix/store` path in the rpath
707            # as it makes it significantly more robust against changes to the location of
708            # the `.nix-deps` location.
709            #
710            # bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
711            # zlib: Needed as a system dependency of `libLLVM-*.so`.
712            # patchelf: Needed for patching ELF binaries (see doc comment above).
713            nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
714            nix_expr = '''
715            with (import <nixpkgs> {});
716            symlinkJoin {
717              name = "rust-stage0-dependencies";
718              paths = [
719                zlib
720                patchelf
721                stdenv.cc.bintools
722              ];
723            }
724            '''
725            try:
726                subprocess.check_output([
727                    "nix-build", "-E", nix_expr, "-o", nix_deps_dir,
728                ])
729            except subprocess.CalledProcessError as reason:
730                eprint("warning: failed to call nix-build:", reason)
731                return
732            self.nix_deps_dir = nix_deps_dir
733
734        patchelf = "{}/bin/patchelf".format(nix_deps_dir)
735        rpath_entries = [
736            # Relative default, all binary and dynamic libraries we ship
737            # appear to have this (even when `../lib` is redundant).
738            "$ORIGIN/../lib",
739            os.path.join(os.path.realpath(nix_deps_dir), "lib")
740        ]
741        patchelf_args = ["--set-rpath", ":".join(rpath_entries)]
742        if not fname.endswith(".so"):
743            # Finally, set the correct .interp for binaries
744            with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker:
745                patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
746
747        try:
748            subprocess.check_output([patchelf] + patchelf_args + [fname])
749        except subprocess.CalledProcessError as reason:
750            eprint("warning: failed to call patchelf:", reason)
751            return
752
753    def rustc_stamp(self):
754        """Return the path for .rustc-stamp at the given stage
755
756        >>> rb = RustBuild()
757        >>> rb.build = "host"
758        >>> rb.build_dir = "build"
759        >>> expected = os.path.join("build", "host", "stage0", ".rustc-stamp")
760        >>> assert rb.rustc_stamp() == expected, rb.rustc_stamp()
761        """
762        return os.path.join(self.bin_root(), '.rustc-stamp')
763
764    def program_out_of_date(self, stamp_path, key):
765        """Check if the given program stamp is out of date"""
766        if not os.path.exists(stamp_path) or self.clean:
767            return True
768        with open(stamp_path, 'r') as stamp:
769            return key != stamp.read()
770
771    def bin_root(self):
772        """Return the binary root directory for the given stage
773
774        >>> rb = RustBuild()
775        >>> rb.build = "devel"
776        >>> expected = os.path.abspath(os.path.join("build", "devel", "stage0"))
777        >>> assert rb.bin_root() == expected, rb.bin_root()
778        """
779        subdir = "stage0"
780        return os.path.join(self.build_dir, self.build, subdir)
781
782    def get_toml(self, key, section=None):
783        """Returns the value of the given key in config.toml, otherwise returns None
784
785        >>> rb = RustBuild()
786        >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
787        >>> rb.get_toml("key2")
788        'value2'
789
790        If the key does not exist, the result is None:
791
792        >>> rb.get_toml("key3") is None
793        True
794
795        Optionally also matches the section the key appears in
796
797        >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
798        >>> rb.get_toml('key', 'a')
799        'value1'
800        >>> rb.get_toml('key', 'b')
801        'value2'
802        >>> rb.get_toml('key', 'c') is None
803        True
804
805        >>> rb.config_toml = 'key1 = true'
806        >>> rb.get_toml("key1")
807        'true'
808        """
809        return RustBuild.get_toml_static(self.config_toml, key, section)
810
811    @staticmethod
812    def get_toml_static(config_toml, key, section=None):
813        cur_section = None
814        for line in config_toml.splitlines():
815            section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
816            if section_match is not None:
817                cur_section = section_match.group(1)
818
819            match = re.match(r'^{}\s*=(.*)$'.format(key), line)
820            if match is not None:
821                value = match.group(1)
822                if section is None or section == cur_section:
823                    return RustBuild.get_string(value) or value.strip()
824        return None
825
826    def cargo(self):
827        """Return config path for cargo"""
828        return self.program_config('cargo')
829
830    def rustc(self):
831        """Return config path for rustc"""
832        return self.program_config('rustc')
833
834    def program_config(self, program):
835        """Return config path for the given program at the given stage
836
837        >>> rb = RustBuild()
838        >>> rb.config_toml = 'rustc = "rustc"\\n'
839        >>> rb.program_config('rustc')
840        'rustc'
841        >>> rb.config_toml = ''
842        >>> cargo_path = rb.program_config('cargo')
843        >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
844        ... "bin", "cargo")
845        True
846        """
847        config = self.get_toml(program)
848        if config:
849            return os.path.expanduser(config)
850        return os.path.join(self.bin_root(), "bin", "{}{}".format(program, EXE_SUFFIX))
851
852    @staticmethod
853    def get_string(line):
854        """Return the value between double quotes
855
856        >>> RustBuild.get_string('    "devel"   ')
857        'devel'
858        >>> RustBuild.get_string("    'devel'   ")
859        'devel'
860        >>> RustBuild.get_string('devel') is None
861        True
862        >>> RustBuild.get_string('    "devel   ')
863        ''
864        """
865        start = line.find('"')
866        if start != -1:
867            end = start + 1 + line[start + 1:].find('"')
868            return line[start + 1:end]
869        start = line.find('\'')
870        if start != -1:
871            end = start + 1 + line[start + 1:].find('\'')
872            return line[start + 1:end]
873        return None
874
875    def bootstrap_binary(self):
876        """Return the path of the bootstrap binary
877
878        >>> rb = RustBuild()
879        >>> rb.build_dir = "build"
880        >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
881        ... "debug", "bootstrap")
882        True
883        """
884        return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
885
886    def build_bootstrap(self):
887        """Build bootstrap"""
888        env = os.environ.copy()
889        if "GITHUB_ACTIONS" in env:
890            print("::group::Building bootstrap")
891        else:
892            eprint("Building bootstrap")
893
894        args = self.build_bootstrap_cmd(env)
895        # Run this from the source directory so cargo finds .cargo/config
896        run(args, env=env, verbose=self.verbose, cwd=self.rust_root)
897
898        if "GITHUB_ACTIONS" in env:
899            print("::endgroup::")
900
901    def build_bootstrap_cmd(self, env):
902        """For tests."""
903        build_dir = os.path.join(self.build_dir, "bootstrap")
904        if self.clean and os.path.exists(build_dir):
905            shutil.rmtree(build_dir)
906        # `CARGO_BUILD_TARGET` breaks bootstrap build.
907        # See also: <https://github.com/rust-lang/rust/issues/70208>.
908        if "CARGO_BUILD_TARGET" in env:
909            del env["CARGO_BUILD_TARGET"]
910        env["CARGO_TARGET_DIR"] = build_dir
911        env["RUSTC"] = self.rustc()
912        env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
913            (os.pathsep + env["LD_LIBRARY_PATH"]) \
914            if "LD_LIBRARY_PATH" in env else ""
915        env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
916            (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
917            if "DYLD_LIBRARY_PATH" in env else ""
918        env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
919            (os.pathsep + env["LIBRARY_PATH"]) \
920            if "LIBRARY_PATH" in env else ""
921        env["LIBPATH"] = os.path.join(self.bin_root(), "lib") + \
922            (os.pathsep + env["LIBPATH"]) \
923            if "LIBPATH" in env else ""
924
925        # Export Stage0 snapshot compiler related env variables
926        build_section = "target.{}".format(self.build)
927        host_triple_sanitized = self.build.replace("-", "_")
928        var_data = {
929            "CC": "cc", "CXX": "cxx", "LD": "linker", "AR": "ar", "RANLIB": "ranlib"
930        }
931        for var_name, toml_key in var_data.items():
932            toml_val = self.get_toml(toml_key, build_section)
933            if toml_val is not None:
934                env["{}_{}".format(var_name, host_triple_sanitized)] = toml_val
935
936        # preserve existing RUSTFLAGS
937        env.setdefault("RUSTFLAGS", "")
938
939        target_features = []
940        if self.get_toml("crt-static", build_section) == "true":
941            target_features += ["+crt-static"]
942        elif self.get_toml("crt-static", build_section) == "false":
943            target_features += ["-crt-static"]
944        if target_features:
945            env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
946        target_linker = self.get_toml("linker", build_section)
947        if target_linker is not None:
948            env["RUSTFLAGS"] += " -C linker=" + target_linker
949        env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
950        if self.warnings == "default":
951            deny_warnings = self.get_toml("deny-warnings", "rust") != "false"
952        else:
953            deny_warnings = self.warnings == "deny"
954        if deny_warnings:
955            env["RUSTFLAGS"] += " -Dwarnings"
956
957        env["PATH"] = os.path.join(self.bin_root(), "bin") + \
958            os.pathsep + env["PATH"]
959        if not os.path.isfile(self.cargo()):
960            raise Exception("no cargo executable found at `{}`".format(
961                self.cargo()))
962        args = [self.cargo(), "build", "--manifest-path",
963                os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
964        args.extend("--verbose" for _ in range(self.verbose))
965        if self.use_locked_deps:
966            args.append("--locked")
967        if self.use_vendored_sources:
968            args.append("--frozen")
969        if self.get_toml("metrics", "build"):
970            args.append("--features")
971            args.append("build-metrics")
972        if self.json_output:
973            args.append("--message-format=json")
974        if self.color == "always":
975            args.append("--color=always")
976        elif self.color == "never":
977            args.append("--color=never")
978        try:
979            args += env["CARGOFLAGS"].split()
980        except KeyError:
981            pass
982
983        return args
984
985    def build_triple(self):
986        """Build triple as in LLVM
987
988        Note that `default_build_triple` is moderately expensive,
989        so use `self.build` where possible.
990        """
991        config = self.get_toml('build')
992        return config or default_build_triple(self.verbose)
993
994    def check_vendored_status(self):
995        """Check that vendoring is configured properly"""
996        # keep this consistent with the equivalent check in rustbuild:
997        # https://github.com/rust-lang/rust/blob/a8a33cf27166d3eabaffc58ed3799e054af3b0c6/src/bootstrap/lib.rs#L399-L405
998        if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
999            if os.getuid() == 0:
1000                self.use_vendored_sources = True
1001                eprint('info: looks like you\'re trying to run this command as root')
1002                eprint('      and so in order to preserve your $HOME this will now')
1003                eprint('      use vendored sources by default.')
1004
1005        cargo_dir = os.path.join(self.rust_root, '.cargo')
1006        if self.use_vendored_sources:
1007            vendor_dir = os.path.join(self.rust_root, 'vendor')
1008            if not os.path.exists(vendor_dir):
1009                sync_dirs = "--sync ./src/tools/cargo/Cargo.toml " \
1010                            "--sync ./src/tools/rust-analyzer/Cargo.toml " \
1011                            "--sync ./compiler/rustc_codegen_cranelift/Cargo.toml " \
1012                            "--sync ./src/bootstrap/Cargo.toml "
1013                eprint('error: vendoring required, but vendor directory does not exist.')
1014                eprint('       Run `cargo vendor {}` to initialize the '
1015                      'vendor directory.'.format(sync_dirs))
1016                eprint('Alternatively, use the pre-vendored `rustc-src` dist component.')
1017                raise Exception("{} not found".format(vendor_dir))
1018
1019            if not os.path.exists(cargo_dir):
1020                eprint('error: vendoring required, but .cargo/config does not exist.')
1021                raise Exception("{} not found".format(cargo_dir))
1022        else:
1023            if os.path.exists(cargo_dir):
1024                shutil.rmtree(cargo_dir)
1025
1026def parse_args(args):
1027    """Parse the command line arguments that the python script needs."""
1028    parser = argparse.ArgumentParser(add_help=False)
1029    parser.add_argument('-h', '--help', action='store_true')
1030    parser.add_argument('--config')
1031    parser.add_argument('--build-dir')
1032    parser.add_argument('--build')
1033    parser.add_argument('--color', choices=['always', 'never', 'auto'])
1034    parser.add_argument('--clean', action='store_true')
1035    parser.add_argument('--json-output', action='store_true')
1036    parser.add_argument('--warnings', choices=['deny', 'warn', 'default'], default='default')
1037    parser.add_argument('-v', '--verbose', action='count', default=0)
1038
1039    return parser.parse_known_args(args)[0]
1040
1041def bootstrap(args):
1042    """Configure, fetch, build and run the initial bootstrap"""
1043    rust_root = os.path.abspath(os.path.join(__file__, '../../..'))
1044
1045    if not os.path.exists(os.path.join(rust_root, '.git')) and \
1046            os.path.exists(os.path.join(rust_root, '.github')):
1047        eprint("warn: Looks like you are trying to bootstrap Rust from a source that is neither a "
1048               "git clone nor distributed tarball.\nThis build may fail due to missing submodules "
1049               "unless you put them in place manually.")
1050
1051    # Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`,
1052    # then `config.toml` in the root directory.
1053    toml_path = args.config or os.getenv('RUST_BOOTSTRAP_CONFIG')
1054    using_default_path = toml_path is None
1055    if using_default_path:
1056        toml_path = 'config.toml'
1057        if not os.path.exists(toml_path):
1058            toml_path = os.path.join(rust_root, toml_path)
1059
1060    # Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path,
1061    # but not if `config.toml` hasn't been created.
1062    if not using_default_path or os.path.exists(toml_path):
1063        with open(toml_path) as config:
1064            config_toml = config.read()
1065    else:
1066        config_toml = ''
1067
1068    profile = RustBuild.get_toml_static(config_toml, 'profile')
1069    if profile is not None:
1070        # Allows creating alias for profile names, allowing
1071        # profiles to be renamed while maintaining back compatibility
1072        # Keep in sync with `profile_aliases` in config.rs
1073        profile_aliases = {
1074            "user": "dist"
1075        }
1076        include_file = 'config.{}.toml'.format(profile_aliases.get(profile) or profile)
1077        include_dir = os.path.join(rust_root, 'src', 'bootstrap', 'defaults')
1078        include_path = os.path.join(include_dir, include_file)
1079        # HACK: This works because `self.get_toml()` returns the first match it finds for a
1080        # specific key, so appending our defaults at the end allows the user to override them
1081        with open(include_path) as included_toml:
1082            config_toml += os.linesep + included_toml.read()
1083
1084    # Configure initial bootstrap
1085    build = RustBuild(config_toml, args)
1086    build.check_vendored_status()
1087
1088    if not os.path.exists(build.build_dir):
1089        os.makedirs(build.build_dir)
1090
1091    # Fetch/build the bootstrap
1092    build.download_toolchain()
1093    sys.stdout.flush()
1094    build.build_bootstrap()
1095    sys.stdout.flush()
1096
1097    # Run the bootstrap
1098    args = [build.bootstrap_binary()]
1099    args.extend(sys.argv[1:])
1100    env = os.environ.copy()
1101    env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
1102    env["BOOTSTRAP_PYTHON"] = sys.executable
1103    run(args, env=env, verbose=build.verbose, is_bootstrap=True)
1104
1105
1106def main():
1107    """Entry point for the bootstrap process"""
1108    start_time = time()
1109
1110    # x.py help <cmd> ...
1111    if len(sys.argv) > 1 and sys.argv[1] == 'help':
1112        sys.argv[1] = '-h'
1113
1114    args = parse_args(sys.argv)
1115    help_triggered = args.help or len(sys.argv) == 1
1116
1117    # If the user is asking for help, let them know that the whole download-and-build
1118    # process has to happen before anything is printed out.
1119    if help_triggered:
1120        eprint(
1121            "info: Downloading and building bootstrap before processing --help command.\n"
1122            "      See src/bootstrap/README.md for help with common commands.")
1123
1124    exit_code = 0
1125    success_word = "successfully"
1126    try:
1127        bootstrap(args)
1128    except (SystemExit, KeyboardInterrupt) as error:
1129        if hasattr(error, 'code') and isinstance(error.code, int):
1130            exit_code = error.code
1131        else:
1132            exit_code = 1
1133            eprint(error)
1134        success_word = "unsuccessfully"
1135
1136    if not help_triggered:
1137        eprint("Build completed", success_word, "in", format_build_time(time() - start_time))
1138    sys.exit(exit_code)
1139
1140
1141if __name__ == '__main__':
1142    main()