fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #72883 - Mark-Simulacrum:stable-next, r=Mark-Simulacrum49cae55760d

1.44.0
35.6 KiB971 linesraw
1from __future__ import absolute_import, division, print_function
2import argparse
3import contextlib
4import datetime
5import hashlib
6import os
7import re
8import shutil
9import subprocess
10import sys
11import tarfile
12import tempfile
13
14from time import time
15
16
17def get(url, path, verbose=False):
18    suffix = '.sha256'
19    sha_url = url + suffix
20    with tempfile.NamedTemporaryFile(delete=False) as temp_file:
21        temp_path = temp_file.name
22    with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
23        sha_path = sha_file.name
24
25    try:
26        download(sha_path, sha_url, False, verbose)
27        if os.path.exists(path):
28            if verify(path, sha_path, False):
29                if verbose:
30                    print("using already-download file", path)
31                return
32            else:
33                if verbose:
34                    print("ignoring already-download file",
35                          path, "due to failed verification")
36                os.unlink(path)
37        download(temp_path, url, True, verbose)
38        if not verify(temp_path, sha_path, verbose):
39            raise RuntimeError("failed verification")
40        if verbose:
41            print("moving {} to {}".format(temp_path, path))
42        shutil.move(temp_path, path)
43    finally:
44        delete_if_present(sha_path, verbose)
45        delete_if_present(temp_path, verbose)
46
47
48def delete_if_present(path, verbose):
49    """Remove the given file if present"""
50    if os.path.isfile(path):
51        if verbose:
52            print("removing", path)
53        os.unlink(path)
54
55
56def download(path, url, probably_big, verbose):
57    for _ in range(0, 4):
58        try:
59            _download(path, url, probably_big, verbose, True)
60            return
61        except RuntimeError:
62            print("\nspurious failure, trying again")
63    _download(path, url, probably_big, verbose, False)
64
65
66def _download(path, url, probably_big, verbose, exception):
67    if probably_big or verbose:
68        print("downloading {}".format(url))
69    # see http://serverfault.com/questions/301128/how-to-download
70    if sys.platform == 'win32':
71        run(["PowerShell.exe", "/nologo", "-Command",
72             "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
73             "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
74            verbose=verbose,
75            exception=exception)
76    else:
77        if probably_big or verbose:
78            option = "-#"
79        else:
80            option = "-s"
81        run(["curl", option,
82             "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
83             "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
84             "--retry", "3", "-Sf", "-o", path, url],
85            verbose=verbose,
86            exception=exception)
87
88
89def verify(path, sha_path, verbose):
90    """Check if the sha256 sum of the given path is valid"""
91    if verbose:
92        print("verifying", path)
93    with open(path, "rb") as source:
94        found = hashlib.sha256(source.read()).hexdigest()
95    with open(sha_path, "r") as sha256sum:
96        expected = sha256sum.readline().split()[0]
97    verified = found == expected
98    if not verified:
99        print("invalid checksum:\n"
100              "    found:    {}\n"
101              "    expected: {}".format(found, expected))
102    return verified
103
104
105def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
106    """Unpack the given tarball file"""
107    print("extracting", tarball)
108    fname = os.path.basename(tarball).replace(tarball_suffix, "")
109    with contextlib.closing(tarfile.open(tarball)) as tar:
110        for member in tar.getnames():
111            if "/" not in member:
112                continue
113            name = member.replace(fname + "/", "", 1)
114            if match is not None and not name.startswith(match):
115                continue
116            name = name[len(match) + 1:]
117
118            dst_path = os.path.join(dst, name)
119            if verbose:
120                print("  extracting", member)
121            tar.extract(member, dst)
122            src_path = os.path.join(dst, member)
123            if os.path.isdir(src_path) and os.path.exists(dst_path):
124                continue
125            shutil.move(src_path, dst_path)
126    shutil.rmtree(os.path.join(dst, fname))
127
128
129def run(args, verbose=False, exception=False, **kwargs):
130    """Run a child program in a new process"""
131    if verbose:
132        print("running: " + ' '.join(args))
133    sys.stdout.flush()
134    # Use Popen here instead of call() as it apparently allows powershell on
135    # Windows to not lock up waiting for input presumably.
136    ret = subprocess.Popen(args, **kwargs)
137    code = ret.wait()
138    if code != 0:
139        err = "failed to run: " + ' '.join(args)
140        if verbose or exception:
141            raise RuntimeError(err)
142        sys.exit(err)
143
144
145def stage0_data(rust_root):
146    """Build a dictionary from stage0.txt"""
147    nightlies = os.path.join(rust_root, "src/stage0.txt")
148    with open(nightlies, 'r') as nightlies:
149        lines = [line.rstrip() for line in nightlies
150                 if not line.startswith("#")]
151        return dict([line.split(": ", 1) for line in lines if line])
152
153
154def format_build_time(duration):
155    """Return a nicer format for build time
156
157    >>> format_build_time('300')
158    '0:05:00'
159    """
160    return str(datetime.timedelta(seconds=int(duration)))
161
162
163def default_build_triple():
164    """Build triple as in LLVM"""
165    default_encoding = sys.getdefaultencoding()
166    try:
167        ostype = subprocess.check_output(
168            ['uname', '-s']).strip().decode(default_encoding)
169        cputype = subprocess.check_output(
170            ['uname', '-m']).strip().decode(default_encoding)
171    except (subprocess.CalledProcessError, OSError):
172        if sys.platform == 'win32':
173            return 'x86_64-pc-windows-msvc'
174        err = "uname not found"
175        sys.exit(err)
176
177    # The goal here is to come up with the same triple as LLVM would,
178    # at least for the subset of platforms we're willing to target.
179    ostype_mapper = {
180        'Darwin': 'apple-darwin',
181        'DragonFly': 'unknown-dragonfly',
182        'FreeBSD': 'unknown-freebsd',
183        'Haiku': 'unknown-haiku',
184        'NetBSD': 'unknown-netbsd',
185        'OpenBSD': 'unknown-openbsd'
186    }
187
188    # Consider the direct transformation first and then the special cases
189    if ostype in ostype_mapper:
190        ostype = ostype_mapper[ostype]
191    elif ostype == 'Linux':
192        os_from_sp = subprocess.check_output(
193            ['uname', '-o']).strip().decode(default_encoding)
194        if os_from_sp == 'Android':
195            ostype = 'linux-android'
196        else:
197            ostype = 'unknown-linux-gnu'
198    elif ostype == 'SunOS':
199        ostype = 'sun-solaris'
200        # On Solaris, uname -m will return a machine classification instead
201        # of a cpu type, so uname -p is recommended instead.  However, the
202        # output from that option is too generic for our purposes (it will
203        # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
204        # must be used instead.
205        try:
206            cputype = subprocess.check_output(
207                ['isainfo', '-k']).strip().decode(default_encoding)
208        except (subprocess.CalledProcessError, OSError):
209            err = "isainfo not found"
210            sys.exit(err)
211    elif ostype.startswith('MINGW'):
212        # msys' `uname` does not print gcc configuration, but prints msys
213        # configuration. so we cannot believe `uname -m`:
214        # msys1 is always i686 and msys2 is always x86_64.
215        # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
216        # MINGW64 on x86_64.
217        ostype = 'pc-windows-gnu'
218        cputype = 'i686'
219        if os.environ.get('MSYSTEM') == 'MINGW64':
220            cputype = 'x86_64'
221    elif ostype.startswith('MSYS'):
222        ostype = 'pc-windows-gnu'
223    elif ostype.startswith('CYGWIN_NT'):
224        cputype = 'i686'
225        if ostype.endswith('WOW64'):
226            cputype = 'x86_64'
227        ostype = 'pc-windows-gnu'
228    else:
229        err = "unknown OS type: {}".format(ostype)
230        sys.exit(err)
231
232    if cputype == 'powerpc' and ostype == 'unknown-freebsd':
233        cputype = subprocess.check_output(
234              ['uname', '-p']).strip().decode(default_encoding)
235    cputype_mapper = {
236        'BePC': 'i686',
237        'aarch64': 'aarch64',
238        'amd64': 'x86_64',
239        'arm64': 'aarch64',
240        'i386': 'i686',
241        'i486': 'i686',
242        'i686': 'i686',
243        'i786': 'i686',
244        'powerpc': 'powerpc',
245        'powerpc64': 'powerpc64',
246        'powerpc64le': 'powerpc64le',
247        'ppc': 'powerpc',
248        'ppc64': 'powerpc64',
249        'ppc64le': 'powerpc64le',
250        's390x': 's390x',
251        'x64': 'x86_64',
252        'x86': 'i686',
253        'x86-64': 'x86_64',
254        'x86_64': 'x86_64'
255    }
256
257    # Consider the direct transformation first and then the special cases
258    if cputype in cputype_mapper:
259        cputype = cputype_mapper[cputype]
260    elif cputype in {'xscale', 'arm'}:
261        cputype = 'arm'
262        if ostype == 'linux-android':
263            ostype = 'linux-androideabi'
264        elif ostype == 'unknown-freebsd':
265            cputype = subprocess.check_output(
266                ['uname', '-p']).strip().decode(default_encoding)
267            ostype = 'unknown-freebsd'
268    elif cputype == 'armv6l':
269        cputype = 'arm'
270        if ostype == 'linux-android':
271            ostype = 'linux-androideabi'
272        else:
273            ostype += 'eabihf'
274    elif cputype in {'armv7l', 'armv8l'}:
275        cputype = 'armv7'
276        if ostype == 'linux-android':
277            ostype = 'linux-androideabi'
278        else:
279            ostype += 'eabihf'
280    elif cputype == 'mips':
281        if sys.byteorder == 'big':
282            cputype = 'mips'
283        elif sys.byteorder == 'little':
284            cputype = 'mipsel'
285        else:
286            raise ValueError("unknown byteorder: {}".format(sys.byteorder))
287    elif cputype == 'mips64':
288        if sys.byteorder == 'big':
289            cputype = 'mips64'
290        elif sys.byteorder == 'little':
291            cputype = 'mips64el'
292        else:
293            raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
294        # only the n64 ABI is supported, indicate it
295        ostype += 'abi64'
296    elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
297        pass
298    else:
299        err = "unknown cpu type: {}".format(cputype)
300        sys.exit(err)
301
302    return "{}-{}".format(cputype, ostype)
303
304
305@contextlib.contextmanager
306def output(filepath):
307    tmp = filepath + '.tmp'
308    with open(tmp, 'w') as f:
309        yield f
310    try:
311        os.remove(filepath)  # PermissionError/OSError on Win32 if in use
312        os.rename(tmp, filepath)
313    except OSError:
314        shutil.copy2(tmp, filepath)
315        os.remove(tmp)
316
317
318class RustBuild(object):
319    """Provide all the methods required to build Rust"""
320    def __init__(self):
321        self.cargo_channel = ''
322        self.date = ''
323        self._download_url = ''
324        self.rustc_channel = ''
325        self.rustfmt_channel = ''
326        self.build = ''
327        self.build_dir = os.path.join(os.getcwd(), "build")
328        self.clean = False
329        self.config_toml = ''
330        self.rust_root = ''
331        self.use_locked_deps = ''
332        self.use_vendored_sources = ''
333        self.verbose = False
334
335    def download_stage0(self):
336        """Fetch the build system for Rust, written in Rust
337
338        This method will build a cache directory, then it will fetch the
339        tarball which has the stage0 compiler used to then bootstrap the Rust
340        compiler itself.
341
342        Each downloaded tarball is extracted, after that, the script
343        will move all the content to the right place.
344        """
345        rustc_channel = self.rustc_channel
346        cargo_channel = self.cargo_channel
347        rustfmt_channel = self.rustfmt_channel
348
349        def support_xz():
350            try:
351                with tempfile.NamedTemporaryFile(delete=False) as temp_file:
352                    temp_path = temp_file.name
353                with tarfile.open(temp_path, "w:xz"):
354                    pass
355                return True
356            except tarfile.CompressionError:
357                return False
358
359        if self.rustc().startswith(self.bin_root()) and \
360                (not os.path.exists(self.rustc()) or
361                 self.program_out_of_date(self.rustc_stamp())):
362            if os.path.exists(self.bin_root()):
363                shutil.rmtree(self.bin_root())
364            tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
365            filename = "rust-std-{}-{}{}".format(
366                rustc_channel, self.build, tarball_suffix)
367            pattern = "rust-std-{}".format(self.build)
368            self._download_stage0_helper(filename, pattern, tarball_suffix)
369
370            filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
371                                              tarball_suffix)
372            self._download_stage0_helper(filename, "rustc", tarball_suffix)
373            self.fix_executable("{}/bin/rustc".format(self.bin_root()))
374            self.fix_executable("{}/bin/rustdoc".format(self.bin_root()))
375            with output(self.rustc_stamp()) as rust_stamp:
376                rust_stamp.write(self.date)
377
378            # This is required so that we don't mix incompatible MinGW
379            # libraries/binaries that are included in rust-std with
380            # the system MinGW ones.
381            if "pc-windows-gnu" in self.build:
382                filename = "rust-mingw-{}-{}{}".format(
383                    rustc_channel, self.build, tarball_suffix)
384                self._download_stage0_helper(filename, "rust-mingw", tarball_suffix)
385
386        if self.cargo().startswith(self.bin_root()) and \
387                (not os.path.exists(self.cargo()) or
388                 self.program_out_of_date(self.cargo_stamp())):
389            tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
390            filename = "cargo-{}-{}{}".format(cargo_channel, self.build,
391                                              tarball_suffix)
392            self._download_stage0_helper(filename, "cargo", tarball_suffix)
393            self.fix_executable("{}/bin/cargo".format(self.bin_root()))
394            with output(self.cargo_stamp()) as cargo_stamp:
395                cargo_stamp.write(self.date)
396
397        if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
398            not os.path.exists(self.rustfmt())
399            or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
400        ):
401            if rustfmt_channel:
402                tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
403                [channel, date] = rustfmt_channel.split('-', 1)
404                filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
405                self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
406                self.fix_executable("{}/bin/rustfmt".format(self.bin_root()))
407                self.fix_executable("{}/bin/cargo-fmt".format(self.bin_root()))
408                with output(self.rustfmt_stamp()) as rustfmt_stamp:
409                    rustfmt_stamp.write(self.date + self.rustfmt_channel)
410
411    def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
412        if date is None:
413            date = self.date
414        cache_dst = os.path.join(self.build_dir, "cache")
415        rustc_cache = os.path.join(cache_dst, date)
416        if not os.path.exists(rustc_cache):
417            os.makedirs(rustc_cache)
418
419        url = "{}/dist/{}".format(self._download_url, date)
420        tarball = os.path.join(rustc_cache, filename)
421        if not os.path.exists(tarball):
422            get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
423        unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
424
425    @staticmethod
426    def fix_executable(fname):
427        """Modifies the interpreter section of 'fname' to fix the dynamic linker
428
429        This method is only required on NixOS and uses the PatchELF utility to
430        change the dynamic linker of ELF executables.
431
432        Please see https://nixos.org/patchelf.html for more information
433        """
434        default_encoding = sys.getdefaultencoding()
435        try:
436            ostype = subprocess.check_output(
437                ['uname', '-s']).strip().decode(default_encoding)
438        except subprocess.CalledProcessError:
439            return
440        except OSError as reason:
441            if getattr(reason, 'winerror', None) is not None:
442                return
443            raise reason
444
445        if ostype != "Linux":
446            return
447
448        if not os.path.exists("/etc/NIXOS"):
449            return
450        if os.path.exists("/lib"):
451            return
452
453        # At this point we're pretty sure the user is running NixOS
454        nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
455        print(nix_os_msg, fname)
456
457        try:
458            interpreter = subprocess.check_output(
459                ["patchelf", "--print-interpreter", fname])
460            interpreter = interpreter.strip().decode(default_encoding)
461        except subprocess.CalledProcessError as reason:
462            print("warning: failed to call patchelf:", reason)
463            return
464
465        loader = interpreter.split("/")[-1]
466
467        try:
468            ldd_output = subprocess.check_output(
469                ['ldd', '/run/current-system/sw/bin/sh'])
470            ldd_output = ldd_output.strip().decode(default_encoding)
471        except subprocess.CalledProcessError as reason:
472            print("warning: unable to call ldd:", reason)
473            return
474
475        for line in ldd_output.splitlines():
476            libname = line.split()[0]
477            if libname.endswith(loader):
478                loader_path = libname[:len(libname) - len(loader)]
479                break
480        else:
481            print("warning: unable to find the path to the dynamic linker")
482            return
483
484        correct_interpreter = loader_path + loader
485
486        try:
487            subprocess.check_output(
488                ["patchelf", "--set-interpreter", correct_interpreter, fname])
489        except subprocess.CalledProcessError as reason:
490            print("warning: failed to call patchelf:", reason)
491            return
492
493    def rustc_stamp(self):
494        """Return the path for .rustc-stamp
495
496        >>> rb = RustBuild()
497        >>> rb.build_dir = "build"
498        >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
499        True
500        """
501        return os.path.join(self.bin_root(), '.rustc-stamp')
502
503    def cargo_stamp(self):
504        """Return the path for .cargo-stamp
505
506        >>> rb = RustBuild()
507        >>> rb.build_dir = "build"
508        >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
509        True
510        """
511        return os.path.join(self.bin_root(), '.cargo-stamp')
512
513    def rustfmt_stamp(self):
514        """Return the path for .rustfmt-stamp
515
516        >>> rb = RustBuild()
517        >>> rb.build_dir = "build"
518        >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
519        True
520        """
521        return os.path.join(self.bin_root(), '.rustfmt-stamp')
522
523    def program_out_of_date(self, stamp_path, extra=""):
524        """Check if the given program stamp is out of date"""
525        if not os.path.exists(stamp_path) or self.clean:
526            return True
527        with open(stamp_path, 'r') as stamp:
528            return (self.date + extra) != stamp.read()
529
530    def bin_root(self):
531        """Return the binary root directory
532
533        >>> rb = RustBuild()
534        >>> rb.build_dir = "build"
535        >>> rb.bin_root() == os.path.join("build", "stage0")
536        True
537
538        When the 'build' property is given should be a nested directory:
539
540        >>> rb.build = "devel"
541        >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
542        True
543        """
544        return os.path.join(self.build_dir, self.build, "stage0")
545
546    def get_toml(self, key, section=None):
547        """Returns the value of the given key in config.toml, otherwise returns None
548
549        >>> rb = RustBuild()
550        >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
551        >>> rb.get_toml("key2")
552        'value2'
553
554        If the key does not exists, the result is None:
555
556        >>> rb.get_toml("key3") is None
557        True
558
559        Optionally also matches the section the key appears in
560
561        >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
562        >>> rb.get_toml('key', 'a')
563        'value1'
564        >>> rb.get_toml('key', 'b')
565        'value2'
566        >>> rb.get_toml('key', 'c') is None
567        True
568
569        >>> rb.config_toml = 'key1 = true'
570        >>> rb.get_toml("key1")
571        'true'
572        """
573
574        cur_section = None
575        for line in self.config_toml.splitlines():
576            section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
577            if section_match is not None:
578                cur_section = section_match.group(1)
579
580            match = re.match(r'^{}\s*=(.*)$'.format(key), line)
581            if match is not None:
582                value = match.group(1)
583                if section is None or section == cur_section:
584                    return self.get_string(value) or value.strip()
585        return None
586
587    def cargo(self):
588        """Return config path for cargo"""
589        return self.program_config('cargo')
590
591    def rustc(self):
592        """Return config path for rustc"""
593        return self.program_config('rustc')
594
595    def rustfmt(self):
596        """Return config path for rustfmt"""
597        if not self.rustfmt_channel:
598            return None
599        return self.program_config('rustfmt')
600
601    def program_config(self, program):
602        """Return config path for the given program
603
604        >>> rb = RustBuild()
605        >>> rb.config_toml = 'rustc = "rustc"\\n'
606        >>> rb.program_config('rustc')
607        'rustc'
608        >>> rb.config_toml = ''
609        >>> cargo_path = rb.program_config('cargo')
610        >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
611        ... "bin", "cargo")
612        True
613        """
614        config = self.get_toml(program)
615        if config:
616            return os.path.expanduser(config)
617        return os.path.join(self.bin_root(), "bin", "{}{}".format(
618            program, self.exe_suffix()))
619
620    @staticmethod
621    def get_string(line):
622        """Return the value between double quotes
623
624        >>> RustBuild.get_string('    "devel"   ')
625        'devel'
626        >>> RustBuild.get_string("    'devel'   ")
627        'devel'
628        >>> RustBuild.get_string('devel') is None
629        True
630        >>> RustBuild.get_string('    "devel   ')
631        ''
632        """
633        start = line.find('"')
634        if start != -1:
635            end = start + 1 + line[start + 1:].find('"')
636            return line[start + 1:end]
637        start = line.find('\'')
638        if start != -1:
639            end = start + 1 + line[start + 1:].find('\'')
640            return line[start + 1:end]
641        return None
642
643    @staticmethod
644    def exe_suffix():
645        """Return a suffix for executables"""
646        if sys.platform == 'win32':
647            return '.exe'
648        return ''
649
650    def bootstrap_binary(self):
651        """Return the path of the bootstrap binary
652
653        >>> rb = RustBuild()
654        >>> rb.build_dir = "build"
655        >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
656        ... "debug", "bootstrap")
657        True
658        """
659        return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
660
661    def build_bootstrap(self):
662        """Build bootstrap"""
663        build_dir = os.path.join(self.build_dir, "bootstrap")
664        if self.clean and os.path.exists(build_dir):
665            shutil.rmtree(build_dir)
666        env = os.environ.copy()
667        # `CARGO_BUILD_TARGET` breaks bootstrap build.
668        # See also: <https://github.com/rust-lang/rust/issues/70208>.
669        if "CARGO_BUILD_TARGET" in env:
670            del env["CARGO_BUILD_TARGET"]
671        env["RUSTC_BOOTSTRAP"] = '1'
672        env["CARGO_TARGET_DIR"] = build_dir
673        env["RUSTC"] = self.rustc()
674        env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
675            (os.pathsep + env["LD_LIBRARY_PATH"]) \
676            if "LD_LIBRARY_PATH" in env else ""
677        env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
678            (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
679            if "DYLD_LIBRARY_PATH" in env else ""
680        env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
681            (os.pathsep + env["LIBRARY_PATH"]) \
682            if "LIBRARY_PATH" in env else ""
683        # preserve existing RUSTFLAGS
684        env.setdefault("RUSTFLAGS", "")
685        env["RUSTFLAGS"] += " -Cdebuginfo=2"
686
687        build_section = "target.{}".format(self.build_triple())
688        target_features = []
689        if self.get_toml("crt-static", build_section) == "true":
690            target_features += ["+crt-static"]
691        elif self.get_toml("crt-static", build_section) == "false":
692            target_features += ["-crt-static"]
693        if target_features:
694            env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
695        target_linker = self.get_toml("linker", build_section)
696        if target_linker is not None:
697            env["RUSTFLAGS"] += " -C linker=" + target_linker
698        env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
699        if self.get_toml("deny-warnings", "rust") != "false":
700            env["RUSTFLAGS"] += " -Dwarnings"
701
702        env["PATH"] = os.path.join(self.bin_root(), "bin") + \
703            os.pathsep + env["PATH"]
704        if not os.path.isfile(self.cargo()):
705            raise Exception("no cargo executable found at `{}`".format(
706                self.cargo()))
707        args = [self.cargo(), "build", "--manifest-path",
708                os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
709        for _ in range(1, self.verbose):
710            args.append("--verbose")
711        if self.use_locked_deps:
712            args.append("--locked")
713        if self.use_vendored_sources:
714            args.append("--frozen")
715        run(args, env=env, verbose=self.verbose)
716
717    def build_triple(self):
718        """Build triple as in LLVM"""
719        config = self.get_toml('build')
720        if config:
721            return config
722        return default_build_triple()
723
724    def check_submodule(self, module, slow_submodules):
725        if not slow_submodules:
726            checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
727                                           cwd=os.path.join(self.rust_root, module),
728                                           stdout=subprocess.PIPE)
729            return checked_out
730        else:
731            return None
732
733    def update_submodule(self, module, checked_out, recorded_submodules):
734        module_path = os.path.join(self.rust_root, module)
735
736        if checked_out is not None:
737            default_encoding = sys.getdefaultencoding()
738            checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
739            if recorded_submodules[module] == checked_out:
740                return
741
742        print("Updating submodule", module)
743
744        run(["git", "submodule", "-q", "sync", module],
745            cwd=self.rust_root, verbose=self.verbose)
746        try:
747            run(["git", "submodule", "update",
748                 "--init", "--recursive", "--progress", module],
749                cwd=self.rust_root, verbose=self.verbose, exception=True)
750        except RuntimeError:
751            # Some versions of git don't support --progress.
752            run(["git", "submodule", "update",
753                 "--init", "--recursive", module],
754                cwd=self.rust_root, verbose=self.verbose)
755        run(["git", "reset", "-q", "--hard"],
756            cwd=module_path, verbose=self.verbose)
757        run(["git", "clean", "-qdfx"],
758            cwd=module_path, verbose=self.verbose)
759
760    def update_submodules(self):
761        """Update submodules"""
762        if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
763                self.get_toml('submodules') == "false":
764            return
765
766        # check the existence of 'git' command
767        try:
768            subprocess.check_output(['git', '--version'])
769        except (subprocess.CalledProcessError, OSError):
770            print("error: `git` is not found, please make sure it's installed and in the path.")
771            sys.exit(1)
772
773        slow_submodules = self.get_toml('fast-submodules') == "false"
774        start_time = time()
775        if slow_submodules:
776            print('Unconditionally updating all submodules')
777        else:
778            print('Updating only changed submodules')
779        default_encoding = sys.getdefaultencoding()
780        submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
781            ["git", "config", "--file",
782             os.path.join(self.rust_root, ".gitmodules"),
783             "--get-regexp", "path"]
784        ).decode(default_encoding).splitlines()]
785        filtered_submodules = []
786        submodules_names = []
787        for module in submodules:
788            if module.endswith("llvm-project"):
789                if self.get_toml('llvm-config') and self.get_toml('lld') != 'true':
790                    continue
791            check = self.check_submodule(module, slow_submodules)
792            filtered_submodules.append((module, check))
793            submodules_names.append(module)
794        recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
795                                    cwd=self.rust_root, stdout=subprocess.PIPE)
796        recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
797        recorded_submodules = {}
798        for data in recorded:
799            data = data.split()
800            recorded_submodules[data[3]] = data[2]
801        for module in filtered_submodules:
802            self.update_submodule(module[0], module[1], recorded_submodules)
803        print("Submodules updated in %.2f seconds" % (time() - start_time))
804
805    def set_normal_environment(self):
806        """Set download URL for normal environment"""
807        if 'RUSTUP_DIST_SERVER' in os.environ:
808            self._download_url = os.environ['RUSTUP_DIST_SERVER']
809        else:
810            self._download_url = 'https://static.rust-lang.org'
811
812    def set_dev_environment(self):
813        """Set download URL for development environment"""
814        if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
815            self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
816        else:
817            self._download_url = 'https://dev-static.rust-lang.org'
818
819    def check_vendored_status(self):
820        """Check that vendoring is configured properly"""
821        vendor_dir = os.path.join(self.rust_root, 'vendor')
822        if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
823            if os.environ.get('USER') != os.environ['SUDO_USER']:
824                self.use_vendored_sources = True
825                print('info: looks like you are running this command under `sudo`')
826                print('      and so in order to preserve your $HOME this will now')
827                print('      use vendored sources by default.')
828                if not os.path.exists(vendor_dir):
829                    print('error: vendoring required, but vendor directory does not exist.')
830                    print('       Run `cargo vendor` without sudo to initialize the '
831                          'vendor directory.')
832                    raise Exception("{} not found".format(vendor_dir))
833
834        if self.use_vendored_sources:
835            if not os.path.exists('.cargo'):
836                os.makedirs('.cargo')
837            with output('.cargo/config') as cargo_config:
838                cargo_config.write(
839                    "[source.crates-io]\n"
840                    "replace-with = 'vendored-sources'\n"
841                    "registry = 'https://example.com'\n"
842                    "\n"
843                    "[source.vendored-sources]\n"
844                    "directory = '{}/vendor'\n"
845                    .format(self.rust_root))
846        else:
847            if os.path.exists('.cargo'):
848                shutil.rmtree('.cargo')
849
850    def ensure_vendored(self):
851        """Ensure that the vendored sources are available if needed"""
852        vendor_dir = os.path.join(self.rust_root, 'vendor')
853        # Note that this does not handle updating the vendored dependencies if
854        # the rust git repository is updated. Normal development usually does
855        # not use vendoring, so hopefully this isn't too much of a problem.
856        if self.use_vendored_sources and not os.path.exists(vendor_dir):
857            run([self.cargo(), "vendor"],
858                verbose=self.verbose, cwd=self.rust_root)
859
860
861def bootstrap(help_triggered):
862    """Configure, fetch, build and run the initial bootstrap"""
863
864    # If the user is asking for help, let them know that the whole download-and-build
865    # process has to happen before anything is printed out.
866    if help_triggered:
867        print("info: Downloading and building bootstrap before processing --help")
868        print("      command. See src/bootstrap/README.md for help with common")
869        print("      commands.")
870
871    parser = argparse.ArgumentParser(description='Build rust')
872    parser.add_argument('--config')
873    parser.add_argument('--build')
874    parser.add_argument('--src')
875    parser.add_argument('--clean', action='store_true')
876    parser.add_argument('-v', '--verbose', action='count', default=0)
877
878    args = [a for a in sys.argv if a != '-h' and a != '--help']
879    args, _ = parser.parse_known_args(args)
880
881    # Configure initial bootstrap
882    build = RustBuild()
883    build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..'))
884    build.verbose = args.verbose
885    build.clean = args.clean
886
887    try:
888        with open(args.config or 'config.toml') as config:
889            build.config_toml = config.read()
890    except (OSError, IOError):
891        pass
892
893    config_verbose = build.get_toml('verbose', 'build')
894    if config_verbose is not None:
895        build.verbose = max(build.verbose, int(config_verbose))
896
897    build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
898
899    build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
900
901    build.check_vendored_status()
902
903    data = stage0_data(build.rust_root)
904    build.date = data['date']
905    build.rustc_channel = data['rustc']
906    build.cargo_channel = data['cargo']
907
908    if "rustfmt" in data:
909        build.rustfmt_channel = data['rustfmt']
910
911    if 'dev' in data:
912        build.set_dev_environment()
913    else:
914        build.set_normal_environment()
915
916    build.update_submodules()
917
918    # Fetch/build the bootstrap
919    build.build = args.build or build.build_triple()
920    build.download_stage0()
921    sys.stdout.flush()
922    build.ensure_vendored()
923    build.build_bootstrap()
924    sys.stdout.flush()
925
926    # Run the bootstrap
927    args = [build.bootstrap_binary()]
928    args.extend(sys.argv[1:])
929    env = os.environ.copy()
930    env["BUILD"] = build.build
931    env["SRC"] = build.rust_root
932    env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
933    env["BOOTSTRAP_PYTHON"] = sys.executable
934    env["BUILD_DIR"] = build.build_dir
935    env["RUSTC_BOOTSTRAP"] = '1'
936    env["CARGO"] = build.cargo()
937    env["RUSTC"] = build.rustc()
938    if build.rustfmt():
939        env["RUSTFMT"] = build.rustfmt()
940    run(args, env=env, verbose=build.verbose)
941
942
943def main():
944    """Entry point for the bootstrap process"""
945    start_time = time()
946
947    # x.py help <cmd> ...
948    if len(sys.argv) > 1 and sys.argv[1] == 'help':
949        sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
950
951    help_triggered = (
952        '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
953    try:
954        bootstrap(help_triggered)
955        if not help_triggered:
956            print("Build completed successfully in {}".format(
957                format_build_time(time() - start_time)))
958    except (SystemExit, KeyboardInterrupt) as error:
959        if hasattr(error, 'code') and isinstance(error.code, int):
960            exit_code = error.code
961        else:
962            exit_code = 1
963            print(error)
964        if not help_triggered:
965            print("Build completed unsuccessfully in {}".format(
966                format_build_time(time() - start_time)))
967        sys.exit(exit_code)
968
969
970if __name__ == '__main__':
971    main()