fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #71868 - Mark-Simulacrum:stable-1.43.1, r=pietroalbini8d69840ab92

1.43.1
35.4 KiB967 linesraw
1from __future__ import absolute_import, division, print_function
2import argparse
3import contextlib
4import datetime
5import hashlib
6import os
7import re
8import shutil
9import subprocess
10import sys
11import tarfile
12import tempfile
13
14from time import time
15
16
17def get(url, path, verbose=False):
18    suffix = '.sha256'
19    sha_url = url + suffix
20    with tempfile.NamedTemporaryFile(delete=False) as temp_file:
21        temp_path = temp_file.name
22    with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
23        sha_path = sha_file.name
24
25    try:
26        download(sha_path, sha_url, False, verbose)
27        if os.path.exists(path):
28            if verify(path, sha_path, False):
29                if verbose:
30                    print("using already-download file", path)
31                return
32            else:
33                if verbose:
34                    print("ignoring already-download file",
35                          path, "due to failed verification")
36                os.unlink(path)
37        download(temp_path, url, True, verbose)
38        if not verify(temp_path, sha_path, verbose):
39            raise RuntimeError("failed verification")
40        if verbose:
41            print("moving {} to {}".format(temp_path, path))
42        shutil.move(temp_path, path)
43    finally:
44        delete_if_present(sha_path, verbose)
45        delete_if_present(temp_path, verbose)
46
47
48def delete_if_present(path, verbose):
49    """Remove the given file if present"""
50    if os.path.isfile(path):
51        if verbose:
52            print("removing", path)
53        os.unlink(path)
54
55
56def download(path, url, probably_big, verbose):
57    for _ in range(0, 4):
58        try:
59            _download(path, url, probably_big, verbose, True)
60            return
61        except RuntimeError:
62            print("\nspurious failure, trying again")
63    _download(path, url, probably_big, verbose, False)
64
65
66def _download(path, url, probably_big, verbose, exception):
67    if probably_big or verbose:
68        print("downloading {}".format(url))
69    # see http://serverfault.com/questions/301128/how-to-download
70    if sys.platform == 'win32':
71        run(["PowerShell.exe", "/nologo", "-Command",
72             "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
73             "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
74            verbose=verbose,
75            exception=exception)
76    else:
77        if probably_big or verbose:
78            option = "-#"
79        else:
80            option = "-s"
81        run(["curl", option,
82             "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
83             "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
84             "--retry", "3", "-Sf", "-o", path, url],
85            verbose=verbose,
86            exception=exception)
87
88
89def verify(path, sha_path, verbose):
90    """Check if the sha256 sum of the given path is valid"""
91    if verbose:
92        print("verifying", path)
93    with open(path, "rb") as source:
94        found = hashlib.sha256(source.read()).hexdigest()
95    with open(sha_path, "r") as sha256sum:
96        expected = sha256sum.readline().split()[0]
97    verified = found == expected
98    if not verified:
99        print("invalid checksum:\n"
100              "    found:    {}\n"
101              "    expected: {}".format(found, expected))
102    return verified
103
104
105def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
106    """Unpack the given tarball file"""
107    print("extracting", tarball)
108    fname = os.path.basename(tarball).replace(tarball_suffix, "")
109    with contextlib.closing(tarfile.open(tarball)) as tar:
110        for member in tar.getnames():
111            if "/" not in member:
112                continue
113            name = member.replace(fname + "/", "", 1)
114            if match is not None and not name.startswith(match):
115                continue
116            name = name[len(match) + 1:]
117
118            dst_path = os.path.join(dst, name)
119            if verbose:
120                print("  extracting", member)
121            tar.extract(member, dst)
122            src_path = os.path.join(dst, member)
123            if os.path.isdir(src_path) and os.path.exists(dst_path):
124                continue
125            shutil.move(src_path, dst_path)
126    shutil.rmtree(os.path.join(dst, fname))
127
128
129def run(args, verbose=False, exception=False, **kwargs):
130    """Run a child program in a new process"""
131    if verbose:
132        print("running: " + ' '.join(args))
133    sys.stdout.flush()
134    # Use Popen here instead of call() as it apparently allows powershell on
135    # Windows to not lock up waiting for input presumably.
136    ret = subprocess.Popen(args, **kwargs)
137    code = ret.wait()
138    if code != 0:
139        err = "failed to run: " + ' '.join(args)
140        if verbose or exception:
141            raise RuntimeError(err)
142        sys.exit(err)
143
144
145def stage0_data(rust_root):
146    """Build a dictionary from stage0.txt"""
147    nightlies = os.path.join(rust_root, "src/stage0.txt")
148    with open(nightlies, 'r') as nightlies:
149        lines = [line.rstrip() for line in nightlies
150                 if not line.startswith("#")]
151        return dict([line.split(": ", 1) for line in lines if line])
152
153
154def format_build_time(duration):
155    """Return a nicer format for build time
156
157    >>> format_build_time('300')
158    '0:05:00'
159    """
160    return str(datetime.timedelta(seconds=int(duration)))
161
162
163def default_build_triple():
164    """Build triple as in LLVM"""
165    default_encoding = sys.getdefaultencoding()
166    try:
167        ostype = subprocess.check_output(
168            ['uname', '-s']).strip().decode(default_encoding)
169        cputype = subprocess.check_output(
170            ['uname', '-m']).strip().decode(default_encoding)
171    except (subprocess.CalledProcessError, OSError):
172        if sys.platform == 'win32':
173            return 'x86_64-pc-windows-msvc'
174        err = "uname not found"
175        sys.exit(err)
176
177    # The goal here is to come up with the same triple as LLVM would,
178    # at least for the subset of platforms we're willing to target.
179    ostype_mapper = {
180        'Darwin': 'apple-darwin',
181        'DragonFly': 'unknown-dragonfly',
182        'FreeBSD': 'unknown-freebsd',
183        'Haiku': 'unknown-haiku',
184        'NetBSD': 'unknown-netbsd',
185        'OpenBSD': 'unknown-openbsd'
186    }
187
188    # Consider the direct transformation first and then the special cases
189    if ostype in ostype_mapper:
190        ostype = ostype_mapper[ostype]
191    elif ostype == 'Linux':
192        os_from_sp = subprocess.check_output(
193            ['uname', '-o']).strip().decode(default_encoding)
194        if os_from_sp == 'Android':
195            ostype = 'linux-android'
196        else:
197            ostype = 'unknown-linux-gnu'
198    elif ostype == 'SunOS':
199        ostype = 'sun-solaris'
200        # On Solaris, uname -m will return a machine classification instead
201        # of a cpu type, so uname -p is recommended instead.  However, the
202        # output from that option is too generic for our purposes (it will
203        # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
204        # must be used instead.
205        try:
206            cputype = subprocess.check_output(
207                ['isainfo', '-k']).strip().decode(default_encoding)
208        except (subprocess.CalledProcessError, OSError):
209            err = "isainfo not found"
210            sys.exit(err)
211    elif ostype.startswith('MINGW'):
212        # msys' `uname` does not print gcc configuration, but prints msys
213        # configuration. so we cannot believe `uname -m`:
214        # msys1 is always i686 and msys2 is always x86_64.
215        # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
216        # MINGW64 on x86_64.
217        ostype = 'pc-windows-gnu'
218        cputype = 'i686'
219        if os.environ.get('MSYSTEM') == 'MINGW64':
220            cputype = 'x86_64'
221    elif ostype.startswith('MSYS'):
222        ostype = 'pc-windows-gnu'
223    elif ostype.startswith('CYGWIN_NT'):
224        cputype = 'i686'
225        if ostype.endswith('WOW64'):
226            cputype = 'x86_64'
227        ostype = 'pc-windows-gnu'
228    else:
229        err = "unknown OS type: {}".format(ostype)
230        sys.exit(err)
231
232    if cputype == 'powerpc' and ostype == 'unknown-freebsd':
233        cputype = subprocess.check_output(
234              ['uname', '-p']).strip().decode(default_encoding)
235    cputype_mapper = {
236        'BePC': 'i686',
237        'aarch64': 'aarch64',
238        'amd64': 'x86_64',
239        'arm64': 'aarch64',
240        'i386': 'i686',
241        'i486': 'i686',
242        'i686': 'i686',
243        'i786': 'i686',
244        'powerpc': 'powerpc',
245        'powerpc64': 'powerpc64',
246        'powerpc64le': 'powerpc64le',
247        'ppc': 'powerpc',
248        'ppc64': 'powerpc64',
249        'ppc64le': 'powerpc64le',
250        's390x': 's390x',
251        'x64': 'x86_64',
252        'x86': 'i686',
253        'x86-64': 'x86_64',
254        'x86_64': 'x86_64'
255    }
256
257    # Consider the direct transformation first and then the special cases
258    if cputype in cputype_mapper:
259        cputype = cputype_mapper[cputype]
260    elif cputype in {'xscale', 'arm'}:
261        cputype = 'arm'
262        if ostype == 'linux-android':
263            ostype = 'linux-androideabi'
264        elif ostype == 'unknown-freebsd':
265            cputype = subprocess.check_output(
266                ['uname', '-p']).strip().decode(default_encoding)
267            ostype = 'unknown-freebsd'
268    elif cputype == 'armv6l':
269        cputype = 'arm'
270        if ostype == 'linux-android':
271            ostype = 'linux-androideabi'
272        else:
273            ostype += 'eabihf'
274    elif cputype in {'armv7l', 'armv8l'}:
275        cputype = 'armv7'
276        if ostype == 'linux-android':
277            ostype = 'linux-androideabi'
278        else:
279            ostype += 'eabihf'
280    elif cputype == 'mips':
281        if sys.byteorder == 'big':
282            cputype = 'mips'
283        elif sys.byteorder == 'little':
284            cputype = 'mipsel'
285        else:
286            raise ValueError("unknown byteorder: {}".format(sys.byteorder))
287    elif cputype == 'mips64':
288        if sys.byteorder == 'big':
289            cputype = 'mips64'
290        elif sys.byteorder == 'little':
291            cputype = 'mips64el'
292        else:
293            raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
294        # only the n64 ABI is supported, indicate it
295        ostype += 'abi64'
296    elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
297        pass
298    else:
299        err = "unknown cpu type: {}".format(cputype)
300        sys.exit(err)
301
302    return "{}-{}".format(cputype, ostype)
303
304
305@contextlib.contextmanager
306def output(filepath):
307    tmp = filepath + '.tmp'
308    with open(tmp, 'w') as f:
309        yield f
310    try:
311        os.remove(filepath)  # PermissionError/OSError on Win32 if in use
312        os.rename(tmp, filepath)
313    except OSError:
314        shutil.copy2(tmp, filepath)
315        os.remove(tmp)
316
317
318class RustBuild(object):
319    """Provide all the methods required to build Rust"""
320    def __init__(self):
321        self.cargo_channel = ''
322        self.date = ''
323        self._download_url = ''
324        self.rustc_channel = ''
325        self.rustfmt_channel = ''
326        self.build = ''
327        self.build_dir = os.path.join(os.getcwd(), "build")
328        self.clean = False
329        self.config_toml = ''
330        self.rust_root = ''
331        self.use_locked_deps = ''
332        self.use_vendored_sources = ''
333        self.verbose = False
334
335    def download_stage0(self):
336        """Fetch the build system for Rust, written in Rust
337
338        This method will build a cache directory, then it will fetch the
339        tarball which has the stage0 compiler used to then bootstrap the Rust
340        compiler itself.
341
342        Each downloaded tarball is extracted, after that, the script
343        will move all the content to the right place.
344        """
345        rustc_channel = self.rustc_channel
346        cargo_channel = self.cargo_channel
347        rustfmt_channel = self.rustfmt_channel
348
349        def support_xz():
350            try:
351                with tempfile.NamedTemporaryFile(delete=False) as temp_file:
352                    temp_path = temp_file.name
353                with tarfile.open(temp_path, "w:xz"):
354                    pass
355                return True
356            except tarfile.CompressionError:
357                return False
358
359        if self.rustc().startswith(self.bin_root()) and \
360                (not os.path.exists(self.rustc()) or
361                 self.program_out_of_date(self.rustc_stamp())):
362            if os.path.exists(self.bin_root()):
363                shutil.rmtree(self.bin_root())
364            tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
365            filename = "rust-std-{}-{}{}".format(
366                rustc_channel, self.build, tarball_suffix)
367            pattern = "rust-std-{}".format(self.build)
368            self._download_stage0_helper(filename, pattern, tarball_suffix)
369
370            filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
371                                              tarball_suffix)
372            self._download_stage0_helper(filename, "rustc", tarball_suffix)
373            self.fix_executable("{}/bin/rustc".format(self.bin_root()))
374            self.fix_executable("{}/bin/rustdoc".format(self.bin_root()))
375            with output(self.rustc_stamp()) as rust_stamp:
376                rust_stamp.write(self.date)
377
378            # This is required so that we don't mix incompatible MinGW
379            # libraries/binaries that are included in rust-std with
380            # the system MinGW ones.
381            if "pc-windows-gnu" in self.build:
382                filename = "rust-mingw-{}-{}{}".format(
383                    rustc_channel, self.build, tarball_suffix)
384                self._download_stage0_helper(filename, "rust-mingw", tarball_suffix)
385
386        if self.cargo().startswith(self.bin_root()) and \
387                (not os.path.exists(self.cargo()) or
388                 self.program_out_of_date(self.cargo_stamp())):
389            tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
390            filename = "cargo-{}-{}{}".format(cargo_channel, self.build,
391                                              tarball_suffix)
392            self._download_stage0_helper(filename, "cargo", tarball_suffix)
393            self.fix_executable("{}/bin/cargo".format(self.bin_root()))
394            with output(self.cargo_stamp()) as cargo_stamp:
395                cargo_stamp.write(self.date)
396
397        if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
398            not os.path.exists(self.rustfmt())
399            or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
400        ):
401            if rustfmt_channel:
402                tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
403                [channel, date] = rustfmt_channel.split('-', 1)
404                filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
405                self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
406                self.fix_executable("{}/bin/rustfmt".format(self.bin_root()))
407                self.fix_executable("{}/bin/cargo-fmt".format(self.bin_root()))
408                with output(self.rustfmt_stamp()) as rustfmt_stamp:
409                    rustfmt_stamp.write(self.date + self.rustfmt_channel)
410
411    def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
412        if date is None:
413            date = self.date
414        cache_dst = os.path.join(self.build_dir, "cache")
415        rustc_cache = os.path.join(cache_dst, date)
416        if not os.path.exists(rustc_cache):
417            os.makedirs(rustc_cache)
418
419        url = "{}/dist/{}".format(self._download_url, date)
420        tarball = os.path.join(rustc_cache, filename)
421        if not os.path.exists(tarball):
422            get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
423        unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
424
425    @staticmethod
426    def fix_executable(fname):
427        """Modifies the interpreter section of 'fname' to fix the dynamic linker
428
429        This method is only required on NixOS and uses the PatchELF utility to
430        change the dynamic linker of ELF executables.
431
432        Please see https://nixos.org/patchelf.html for more information
433        """
434        default_encoding = sys.getdefaultencoding()
435        try:
436            ostype = subprocess.check_output(
437                ['uname', '-s']).strip().decode(default_encoding)
438        except subprocess.CalledProcessError:
439            return
440        except OSError as reason:
441            if getattr(reason, 'winerror', None) is not None:
442                return
443            raise reason
444
445        if ostype != "Linux":
446            return
447
448        if not os.path.exists("/etc/NIXOS"):
449            return
450        if os.path.exists("/lib"):
451            return
452
453        # At this point we're pretty sure the user is running NixOS
454        nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
455        print(nix_os_msg, fname)
456
457        try:
458            interpreter = subprocess.check_output(
459                ["patchelf", "--print-interpreter", fname])
460            interpreter = interpreter.strip().decode(default_encoding)
461        except subprocess.CalledProcessError as reason:
462            print("warning: failed to call patchelf:", reason)
463            return
464
465        loader = interpreter.split("/")[-1]
466
467        try:
468            ldd_output = subprocess.check_output(
469                ['ldd', '/run/current-system/sw/bin/sh'])
470            ldd_output = ldd_output.strip().decode(default_encoding)
471        except subprocess.CalledProcessError as reason:
472            print("warning: unable to call ldd:", reason)
473            return
474
475        for line in ldd_output.splitlines():
476            libname = line.split()[0]
477            if libname.endswith(loader):
478                loader_path = libname[:len(libname) - len(loader)]
479                break
480        else:
481            print("warning: unable to find the path to the dynamic linker")
482            return
483
484        correct_interpreter = loader_path + loader
485
486        try:
487            subprocess.check_output(
488                ["patchelf", "--set-interpreter", correct_interpreter, fname])
489        except subprocess.CalledProcessError as reason:
490            print("warning: failed to call patchelf:", reason)
491            return
492
493    def rustc_stamp(self):
494        """Return the path for .rustc-stamp
495
496        >>> rb = RustBuild()
497        >>> rb.build_dir = "build"
498        >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
499        True
500        """
501        return os.path.join(self.bin_root(), '.rustc-stamp')
502
503    def cargo_stamp(self):
504        """Return the path for .cargo-stamp
505
506        >>> rb = RustBuild()
507        >>> rb.build_dir = "build"
508        >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
509        True
510        """
511        return os.path.join(self.bin_root(), '.cargo-stamp')
512
513    def rustfmt_stamp(self):
514        """Return the path for .rustfmt-stamp
515
516        >>> rb = RustBuild()
517        >>> rb.build_dir = "build"
518        >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
519        True
520        """
521        return os.path.join(self.bin_root(), '.rustfmt-stamp')
522
523    def program_out_of_date(self, stamp_path, extra=""):
524        """Check if the given program stamp is out of date"""
525        if not os.path.exists(stamp_path) or self.clean:
526            return True
527        with open(stamp_path, 'r') as stamp:
528            return (self.date + extra) != stamp.read()
529
530    def bin_root(self):
531        """Return the binary root directory
532
533        >>> rb = RustBuild()
534        >>> rb.build_dir = "build"
535        >>> rb.bin_root() == os.path.join("build", "stage0")
536        True
537
538        When the 'build' property is given should be a nested directory:
539
540        >>> rb.build = "devel"
541        >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
542        True
543        """
544        return os.path.join(self.build_dir, self.build, "stage0")
545
546    def get_toml(self, key, section=None):
547        """Returns the value of the given key in config.toml, otherwise returns None
548
549        >>> rb = RustBuild()
550        >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
551        >>> rb.get_toml("key2")
552        'value2'
553
554        If the key does not exists, the result is None:
555
556        >>> rb.get_toml("key3") is None
557        True
558
559        Optionally also matches the section the key appears in
560
561        >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
562        >>> rb.get_toml('key', 'a')
563        'value1'
564        >>> rb.get_toml('key', 'b')
565        'value2'
566        >>> rb.get_toml('key', 'c') is None
567        True
568
569        >>> rb.config_toml = 'key1 = true'
570        >>> rb.get_toml("key1")
571        'true'
572        """
573
574        cur_section = None
575        for line in self.config_toml.splitlines():
576            section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
577            if section_match is not None:
578                cur_section = section_match.group(1)
579
580            match = re.match(r'^{}\s*=(.*)$'.format(key), line)
581            if match is not None:
582                value = match.group(1)
583                if section is None or section == cur_section:
584                    return self.get_string(value) or value.strip()
585        return None
586
587    def cargo(self):
588        """Return config path for cargo"""
589        return self.program_config('cargo')
590
591    def rustc(self):
592        """Return config path for rustc"""
593        return self.program_config('rustc')
594
595    def rustfmt(self):
596        """Return config path for rustfmt"""
597        if not self.rustfmt_channel:
598            return None
599        return self.program_config('rustfmt')
600
601    def program_config(self, program):
602        """Return config path for the given program
603
604        >>> rb = RustBuild()
605        >>> rb.config_toml = 'rustc = "rustc"\\n'
606        >>> rb.program_config('rustc')
607        'rustc'
608        >>> rb.config_toml = ''
609        >>> cargo_path = rb.program_config('cargo')
610        >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
611        ... "bin", "cargo")
612        True
613        """
614        config = self.get_toml(program)
615        if config:
616            return os.path.expanduser(config)
617        return os.path.join(self.bin_root(), "bin", "{}{}".format(
618            program, self.exe_suffix()))
619
620    @staticmethod
621    def get_string(line):
622        """Return the value between double quotes
623
624        >>> RustBuild.get_string('    "devel"   ')
625        'devel'
626        >>> RustBuild.get_string("    'devel'   ")
627        'devel'
628        >>> RustBuild.get_string('devel') is None
629        True
630        >>> RustBuild.get_string('    "devel   ')
631        ''
632        """
633        start = line.find('"')
634        if start != -1:
635            end = start + 1 + line[start + 1:].find('"')
636            return line[start + 1:end]
637        start = line.find('\'')
638        if start != -1:
639            end = start + 1 + line[start + 1:].find('\'')
640            return line[start + 1:end]
641        return None
642
643    @staticmethod
644    def exe_suffix():
645        """Return a suffix for executables"""
646        if sys.platform == 'win32':
647            return '.exe'
648        return ''
649
650    def bootstrap_binary(self):
651        """Return the path of the bootstrap binary
652
653        >>> rb = RustBuild()
654        >>> rb.build_dir = "build"
655        >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
656        ... "debug", "bootstrap")
657        True
658        """
659        return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
660
661    def build_bootstrap(self):
662        """Build bootstrap"""
663        build_dir = os.path.join(self.build_dir, "bootstrap")
664        if self.clean and os.path.exists(build_dir):
665            shutil.rmtree(build_dir)
666        env = os.environ.copy()
667        env["RUSTC_BOOTSTRAP"] = '1'
668        env["CARGO_TARGET_DIR"] = build_dir
669        env["RUSTC"] = self.rustc()
670        env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
671            (os.pathsep + env["LD_LIBRARY_PATH"]) \
672            if "LD_LIBRARY_PATH" in env else ""
673        env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
674            (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
675            if "DYLD_LIBRARY_PATH" in env else ""
676        env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
677            (os.pathsep + env["LIBRARY_PATH"]) \
678            if "LIBRARY_PATH" in env else ""
679        # preserve existing RUSTFLAGS
680        env.setdefault("RUSTFLAGS", "")
681        env["RUSTFLAGS"] += " -Cdebuginfo=2"
682
683        build_section = "target.{}".format(self.build_triple())
684        target_features = []
685        if self.get_toml("crt-static", build_section) == "true":
686            target_features += ["+crt-static"]
687        elif self.get_toml("crt-static", build_section) == "false":
688            target_features += ["-crt-static"]
689        if target_features:
690            env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
691        target_linker = self.get_toml("linker", build_section)
692        if target_linker is not None:
693            env["RUSTFLAGS"] += " -C linker=" + target_linker
694        env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
695        if self.get_toml("deny-warnings", "rust") != "false":
696            env["RUSTFLAGS"] += " -Dwarnings"
697
698        env["PATH"] = os.path.join(self.bin_root(), "bin") + \
699            os.pathsep + env["PATH"]
700        if not os.path.isfile(self.cargo()):
701            raise Exception("no cargo executable found at `{}`".format(
702                self.cargo()))
703        args = [self.cargo(), "build", "--manifest-path",
704                os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
705        for _ in range(1, self.verbose):
706            args.append("--verbose")
707        if self.use_locked_deps:
708            args.append("--locked")
709        if self.use_vendored_sources:
710            args.append("--frozen")
711        run(args, env=env, verbose=self.verbose)
712
713    def build_triple(self):
714        """Build triple as in LLVM"""
715        config = self.get_toml('build')
716        if config:
717            return config
718        return default_build_triple()
719
720    def check_submodule(self, module, slow_submodules):
721        if not slow_submodules:
722            checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
723                                           cwd=os.path.join(self.rust_root, module),
724                                           stdout=subprocess.PIPE)
725            return checked_out
726        else:
727            return None
728
729    def update_submodule(self, module, checked_out, recorded_submodules):
730        module_path = os.path.join(self.rust_root, module)
731
732        if checked_out is not None:
733            default_encoding = sys.getdefaultencoding()
734            checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
735            if recorded_submodules[module] == checked_out:
736                return
737
738        print("Updating submodule", module)
739
740        run(["git", "submodule", "-q", "sync", module],
741            cwd=self.rust_root, verbose=self.verbose)
742        try:
743            run(["git", "submodule", "update",
744                 "--init", "--recursive", "--progress", module],
745                cwd=self.rust_root, verbose=self.verbose, exception=True)
746        except RuntimeError:
747            # Some versions of git don't support --progress.
748            run(["git", "submodule", "update",
749                 "--init", "--recursive", module],
750                cwd=self.rust_root, verbose=self.verbose)
751        run(["git", "reset", "-q", "--hard"],
752            cwd=module_path, verbose=self.verbose)
753        run(["git", "clean", "-qdfx"],
754            cwd=module_path, verbose=self.verbose)
755
756    def update_submodules(self):
757        """Update submodules"""
758        if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
759                self.get_toml('submodules') == "false":
760            return
761
762        # check the existence of 'git' command
763        try:
764            subprocess.check_output(['git', '--version'])
765        except (subprocess.CalledProcessError, OSError):
766            print("error: `git` is not found, please make sure it's installed and in the path.")
767            sys.exit(1)
768
769        slow_submodules = self.get_toml('fast-submodules') == "false"
770        start_time = time()
771        if slow_submodules:
772            print('Unconditionally updating all submodules')
773        else:
774            print('Updating only changed submodules')
775        default_encoding = sys.getdefaultencoding()
776        submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
777            ["git", "config", "--file",
778             os.path.join(self.rust_root, ".gitmodules"),
779             "--get-regexp", "path"]
780        ).decode(default_encoding).splitlines()]
781        filtered_submodules = []
782        submodules_names = []
783        for module in submodules:
784            if module.endswith("llvm-project"):
785                if self.get_toml('llvm-config') and self.get_toml('lld') != 'true':
786                    continue
787            check = self.check_submodule(module, slow_submodules)
788            filtered_submodules.append((module, check))
789            submodules_names.append(module)
790        recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
791                                    cwd=self.rust_root, stdout=subprocess.PIPE)
792        recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
793        recorded_submodules = {}
794        for data in recorded:
795            data = data.split()
796            recorded_submodules[data[3]] = data[2]
797        for module in filtered_submodules:
798            self.update_submodule(module[0], module[1], recorded_submodules)
799        print("Submodules updated in %.2f seconds" % (time() - start_time))
800
801    def set_normal_environment(self):
802        """Set download URL for normal environment"""
803        if 'RUSTUP_DIST_SERVER' in os.environ:
804            self._download_url = os.environ['RUSTUP_DIST_SERVER']
805        else:
806            self._download_url = 'https://static.rust-lang.org'
807
808    def set_dev_environment(self):
809        """Set download URL for development environment"""
810        if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
811            self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
812        else:
813            self._download_url = 'https://dev-static.rust-lang.org'
814
815    def check_vendored_status(self):
816        """Check that vendoring is configured properly"""
817        vendor_dir = os.path.join(self.rust_root, 'vendor')
818        if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
819            if os.environ.get('USER') != os.environ['SUDO_USER']:
820                self.use_vendored_sources = True
821                print('info: looks like you are running this command under `sudo`')
822                print('      and so in order to preserve your $HOME this will now')
823                print('      use vendored sources by default.')
824                if not os.path.exists(vendor_dir):
825                    print('error: vendoring required, but vendor directory does not exist.')
826                    print('       Run `cargo vendor` without sudo to initialize the '
827                          'vendor directory.')
828                    raise Exception("{} not found".format(vendor_dir))
829
830        if self.use_vendored_sources:
831            if not os.path.exists('.cargo'):
832                os.makedirs('.cargo')
833            with output('.cargo/config') as cargo_config:
834                cargo_config.write(
835                    "[source.crates-io]\n"
836                    "replace-with = 'vendored-sources'\n"
837                    "registry = 'https://example.com'\n"
838                    "\n"
839                    "[source.vendored-sources]\n"
840                    "directory = '{}/vendor'\n"
841                    .format(self.rust_root))
842        else:
843            if os.path.exists('.cargo'):
844                shutil.rmtree('.cargo')
845
846    def ensure_vendored(self):
847        """Ensure that the vendored sources are available if needed"""
848        vendor_dir = os.path.join(self.rust_root, 'vendor')
849        # Note that this does not handle updating the vendored dependencies if
850        # the rust git repository is updated. Normal development usually does
851        # not use vendoring, so hopefully this isn't too much of a problem.
852        if self.use_vendored_sources and not os.path.exists(vendor_dir):
853            run([self.cargo(), "vendor"],
854                verbose=self.verbose, cwd=self.rust_root)
855
856
857def bootstrap(help_triggered):
858    """Configure, fetch, build and run the initial bootstrap"""
859
860    # If the user is asking for help, let them know that the whole download-and-build
861    # process has to happen before anything is printed out.
862    if help_triggered:
863        print("info: Downloading and building bootstrap before processing --help")
864        print("      command. See src/bootstrap/README.md for help with common")
865        print("      commands.")
866
867    parser = argparse.ArgumentParser(description='Build rust')
868    parser.add_argument('--config')
869    parser.add_argument('--build')
870    parser.add_argument('--src')
871    parser.add_argument('--clean', action='store_true')
872    parser.add_argument('-v', '--verbose', action='count', default=0)
873
874    args = [a for a in sys.argv if a != '-h' and a != '--help']
875    args, _ = parser.parse_known_args(args)
876
877    # Configure initial bootstrap
878    build = RustBuild()
879    build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..'))
880    build.verbose = args.verbose
881    build.clean = args.clean
882
883    try:
884        with open(args.config or 'config.toml') as config:
885            build.config_toml = config.read()
886    except (OSError, IOError):
887        pass
888
889    config_verbose = build.get_toml('verbose', 'build')
890    if config_verbose is not None:
891        build.verbose = max(build.verbose, int(config_verbose))
892
893    build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
894
895    build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
896
897    build.check_vendored_status()
898
899    data = stage0_data(build.rust_root)
900    build.date = data['date']
901    build.rustc_channel = data['rustc']
902    build.cargo_channel = data['cargo']
903
904    if "rustfmt" in data:
905        build.rustfmt_channel = data['rustfmt']
906
907    if 'dev' in data:
908        build.set_dev_environment()
909    else:
910        build.set_normal_environment()
911
912    build.update_submodules()
913
914    # Fetch/build the bootstrap
915    build.build = args.build or build.build_triple()
916    build.download_stage0()
917    sys.stdout.flush()
918    build.ensure_vendored()
919    build.build_bootstrap()
920    sys.stdout.flush()
921
922    # Run the bootstrap
923    args = [build.bootstrap_binary()]
924    args.extend(sys.argv[1:])
925    env = os.environ.copy()
926    env["BUILD"] = build.build
927    env["SRC"] = build.rust_root
928    env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
929    env["BOOTSTRAP_PYTHON"] = sys.executable
930    env["BUILD_DIR"] = build.build_dir
931    env["RUSTC_BOOTSTRAP"] = '1'
932    env["CARGO"] = build.cargo()
933    env["RUSTC"] = build.rustc()
934    if build.rustfmt():
935        env["RUSTFMT"] = build.rustfmt()
936    run(args, env=env, verbose=build.verbose)
937
938
939def main():
940    """Entry point for the bootstrap process"""
941    start_time = time()
942
943    # x.py help <cmd> ...
944    if len(sys.argv) > 1 and sys.argv[1] == 'help':
945        sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
946
947    help_triggered = (
948        '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
949    try:
950        bootstrap(help_triggered)
951        if not help_triggered:
952            print("Build completed successfully in {}".format(
953                format_build_time(time() - start_time)))
954    except (SystemExit, KeyboardInterrupt) as error:
955        if hasattr(error, 'code') and isinstance(error.code, int):
956            exit_code = error.code
957        else:
958            exit_code = 1
959            print(error)
960        if not help_triggered:
961            print("Build completed unsuccessfully in {}".format(
962                format_build_time(time() - start_time)))
963        sys.exit(exit_code)
964
965
966if __name__ == '__main__':
967    main()