fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #69854 - pietroalbini:stable-next, r=Centrilb8cedc00407

1.42.0
35.4 KiB968 linesraw
1from __future__ import absolute_import, division, print_function
2import argparse
3import contextlib
4import datetime
5import hashlib
6import os
7import re
8import shutil
9import subprocess
10import sys
11import tarfile
12import tempfile
13
14from time import time
15
16
17def get(url, path, verbose=False):
18    suffix = '.sha256'
19    sha_url = url + suffix
20    with tempfile.NamedTemporaryFile(delete=False) as temp_file:
21        temp_path = temp_file.name
22    with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
23        sha_path = sha_file.name
24
25    try:
26        download(sha_path, sha_url, False, verbose)
27        if os.path.exists(path):
28            if verify(path, sha_path, False):
29                if verbose:
30                    print("using already-download file", path)
31                return
32            else:
33                if verbose:
34                    print("ignoring already-download file",
35                          path, "due to failed verification")
36                os.unlink(path)
37        download(temp_path, url, True, verbose)
38        if not verify(temp_path, sha_path, verbose):
39            raise RuntimeError("failed verification")
40        if verbose:
41            print("moving {} to {}".format(temp_path, path))
42        shutil.move(temp_path, path)
43    finally:
44        delete_if_present(sha_path, verbose)
45        delete_if_present(temp_path, verbose)
46
47
48def delete_if_present(path, verbose):
49    """Remove the given file if present"""
50    if os.path.isfile(path):
51        if verbose:
52            print("removing", path)
53        os.unlink(path)
54
55
56def download(path, url, probably_big, verbose):
57    for _ in range(0, 4):
58        try:
59            _download(path, url, probably_big, verbose, True)
60            return
61        except RuntimeError:
62            print("\nspurious failure, trying again")
63    _download(path, url, probably_big, verbose, False)
64
65
66def _download(path, url, probably_big, verbose, exception):
67    if probably_big or verbose:
68        print("downloading {}".format(url))
69    # see http://serverfault.com/questions/301128/how-to-download
70    if sys.platform == 'win32':
71        run(["PowerShell.exe", "/nologo", "-Command",
72             "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
73             "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
74            verbose=verbose,
75            exception=exception)
76    else:
77        if probably_big or verbose:
78            option = "-#"
79        else:
80            option = "-s"
81        run(["curl", option,
82             "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
83             "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
84             "--retry", "3", "-Sf", "-o", path, url],
85            verbose=verbose,
86            exception=exception)
87
88
89def verify(path, sha_path, verbose):
90    """Check if the sha256 sum of the given path is valid"""
91    if verbose:
92        print("verifying", path)
93    with open(path, "rb") as source:
94        found = hashlib.sha256(source.read()).hexdigest()
95    with open(sha_path, "r") as sha256sum:
96        expected = sha256sum.readline().split()[0]
97    verified = found == expected
98    if not verified:
99        print("invalid checksum:\n"
100              "    found:    {}\n"
101              "    expected: {}".format(found, expected))
102    return verified
103
104
105def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
106    """Unpack the given tarball file"""
107    print("extracting", tarball)
108    fname = os.path.basename(tarball).replace(tarball_suffix, "")
109    with contextlib.closing(tarfile.open(tarball)) as tar:
110        for member in tar.getnames():
111            if "/" not in member:
112                continue
113            name = member.replace(fname + "/", "", 1)
114            if match is not None and not name.startswith(match):
115                continue
116            name = name[len(match) + 1:]
117
118            dst_path = os.path.join(dst, name)
119            if verbose:
120                print("  extracting", member)
121            tar.extract(member, dst)
122            src_path = os.path.join(dst, member)
123            if os.path.isdir(src_path) and os.path.exists(dst_path):
124                continue
125            shutil.move(src_path, dst_path)
126    shutil.rmtree(os.path.join(dst, fname))
127
128
129def run(args, verbose=False, exception=False, **kwargs):
130    """Run a child program in a new process"""
131    if verbose:
132        print("running: " + ' '.join(args))
133    sys.stdout.flush()
134    # Use Popen here instead of call() as it apparently allows powershell on
135    # Windows to not lock up waiting for input presumably.
136    ret = subprocess.Popen(args, **kwargs)
137    code = ret.wait()
138    if code != 0:
139        err = "failed to run: " + ' '.join(args)
140        if verbose or exception:
141            raise RuntimeError(err)
142        sys.exit(err)
143
144
145def stage0_data(rust_root):
146    """Build a dictionary from stage0.txt"""
147    nightlies = os.path.join(rust_root, "src/stage0.txt")
148    with open(nightlies, 'r') as nightlies:
149        lines = [line.rstrip() for line in nightlies
150                 if not line.startswith("#")]
151        return dict([line.split(": ", 1) for line in lines if line])
152
153
154def format_build_time(duration):
155    """Return a nicer format for build time
156
157    >>> format_build_time('300')
158    '0:05:00'
159    """
160    return str(datetime.timedelta(seconds=int(duration)))
161
162
163def default_build_triple():
164    """Build triple as in LLVM"""
165    default_encoding = sys.getdefaultencoding()
166    try:
167        ostype = subprocess.check_output(
168            ['uname', '-s']).strip().decode(default_encoding)
169        cputype = subprocess.check_output(
170            ['uname', '-m']).strip().decode(default_encoding)
171    except (subprocess.CalledProcessError, OSError):
172        if sys.platform == 'win32':
173            return 'x86_64-pc-windows-msvc'
174        err = "uname not found"
175        sys.exit(err)
176
177    # The goal here is to come up with the same triple as LLVM would,
178    # at least for the subset of platforms we're willing to target.
179    ostype_mapper = {
180        'Darwin': 'apple-darwin',
181        'DragonFly': 'unknown-dragonfly',
182        'FreeBSD': 'unknown-freebsd',
183        'Haiku': 'unknown-haiku',
184        'NetBSD': 'unknown-netbsd',
185        'OpenBSD': 'unknown-openbsd'
186    }
187
188    # Consider the direct transformation first and then the special cases
189    if ostype in ostype_mapper:
190        ostype = ostype_mapper[ostype]
191    elif ostype == 'Linux':
192        os_from_sp = subprocess.check_output(
193            ['uname', '-o']).strip().decode(default_encoding)
194        if os_from_sp == 'Android':
195            ostype = 'linux-android'
196        else:
197            ostype = 'unknown-linux-gnu'
198    elif ostype == 'SunOS':
199        ostype = 'sun-solaris'
200        # On Solaris, uname -m will return a machine classification instead
201        # of a cpu type, so uname -p is recommended instead.  However, the
202        # output from that option is too generic for our purposes (it will
203        # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
204        # must be used instead.
205        try:
206            cputype = subprocess.check_output(
207                ['isainfo', '-k']).strip().decode(default_encoding)
208        except (subprocess.CalledProcessError, OSError):
209            err = "isainfo not found"
210            sys.exit(err)
211    elif ostype.startswith('MINGW'):
212        # msys' `uname` does not print gcc configuration, but prints msys
213        # configuration. so we cannot believe `uname -m`:
214        # msys1 is always i686 and msys2 is always x86_64.
215        # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
216        # MINGW64 on x86_64.
217        ostype = 'pc-windows-gnu'
218        cputype = 'i686'
219        if os.environ.get('MSYSTEM') == 'MINGW64':
220            cputype = 'x86_64'
221    elif ostype.startswith('MSYS'):
222        ostype = 'pc-windows-gnu'
223    elif ostype.startswith('CYGWIN_NT'):
224        cputype = 'i686'
225        if ostype.endswith('WOW64'):
226            cputype = 'x86_64'
227        ostype = 'pc-windows-gnu'
228    else:
229        err = "unknown OS type: {}".format(ostype)
230        sys.exit(err)
231
232    if cputype == 'powerpc' and ostype == 'unknown-freebsd':
233        cputype = subprocess.check_output(
234              ['uname', '-p']).strip().decode(default_encoding)
235    cputype_mapper = {
236        'BePC': 'i686',
237        'aarch64': 'aarch64',
238        'amd64': 'x86_64',
239        'arm64': 'aarch64',
240        'i386': 'i686',
241        'i486': 'i686',
242        'i686': 'i686',
243        'i786': 'i686',
244        'powerpc': 'powerpc',
245        'powerpc64': 'powerpc64',
246        'powerpc64le': 'powerpc64le',
247        'ppc': 'powerpc',
248        'ppc64': 'powerpc64',
249        'ppc64le': 'powerpc64le',
250        's390x': 's390x',
251        'x64': 'x86_64',
252        'x86': 'i686',
253        'x86-64': 'x86_64',
254        'x86_64': 'x86_64'
255    }
256
257    # Consider the direct transformation first and then the special cases
258    if cputype in cputype_mapper:
259        cputype = cputype_mapper[cputype]
260    elif cputype in {'xscale', 'arm'}:
261        cputype = 'arm'
262        if ostype == 'linux-android':
263            ostype = 'linux-androideabi'
264        elif ostype == 'unknown-freebsd':
265            cputype = subprocess.check_output(
266                ['uname', '-p']).strip().decode(default_encoding)
267            ostype = 'unknown-freebsd'
268    elif cputype == 'armv6l':
269        cputype = 'arm'
270        if ostype == 'linux-android':
271            ostype = 'linux-androideabi'
272        else:
273            ostype += 'eabihf'
274    elif cputype in {'armv7l', 'armv8l'}:
275        cputype = 'armv7'
276        if ostype == 'linux-android':
277            ostype = 'linux-androideabi'
278        else:
279            ostype += 'eabihf'
280    elif cputype == 'mips':
281        if sys.byteorder == 'big':
282            cputype = 'mips'
283        elif sys.byteorder == 'little':
284            cputype = 'mipsel'
285        else:
286            raise ValueError("unknown byteorder: {}".format(sys.byteorder))
287    elif cputype == 'mips64':
288        if sys.byteorder == 'big':
289            cputype = 'mips64'
290        elif sys.byteorder == 'little':
291            cputype = 'mips64el'
292        else:
293            raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
294        # only the n64 ABI is supported, indicate it
295        ostype += 'abi64'
296    elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
297        pass
298    else:
299        err = "unknown cpu type: {}".format(cputype)
300        sys.exit(err)
301
302    return "{}-{}".format(cputype, ostype)
303
304
305@contextlib.contextmanager
306def output(filepath):
307    tmp = filepath + '.tmp'
308    with open(tmp, 'w') as f:
309        yield f
310    try:
311        os.remove(filepath)  # PermissionError/OSError on Win32 if in use
312        os.rename(tmp, filepath)
313    except OSError:
314        shutil.copy2(tmp, filepath)
315        os.remove(tmp)
316
317
318class RustBuild(object):
319    """Provide all the methods required to build Rust"""
320    def __init__(self):
321        self.cargo_channel = ''
322        self.date = ''
323        self._download_url = ''
324        self.rustc_channel = ''
325        self.rustfmt_channel = ''
326        self.build = ''
327        self.build_dir = os.path.join(os.getcwd(), "build")
328        self.clean = False
329        self.config_toml = ''
330        self.rust_root = ''
331        self.use_locked_deps = ''
332        self.use_vendored_sources = ''
333        self.verbose = False
334
335
336    def download_stage0(self):
337        """Fetch the build system for Rust, written in Rust
338
339        This method will build a cache directory, then it will fetch the
340        tarball which has the stage0 compiler used to then bootstrap the Rust
341        compiler itself.
342
343        Each downloaded tarball is extracted, after that, the script
344        will move all the content to the right place.
345        """
346        rustc_channel = self.rustc_channel
347        cargo_channel = self.cargo_channel
348        rustfmt_channel = self.rustfmt_channel
349
350        def support_xz():
351            try:
352                with tempfile.NamedTemporaryFile(delete=False) as temp_file:
353                    temp_path = temp_file.name
354                with tarfile.open(temp_path, "w:xz") as tar:
355                    pass
356                return True
357            except tarfile.CompressionError:
358                return False
359
360        if self.rustc().startswith(self.bin_root()) and \
361                (not os.path.exists(self.rustc()) or
362                 self.program_out_of_date(self.rustc_stamp())):
363            if os.path.exists(self.bin_root()):
364                shutil.rmtree(self.bin_root())
365            tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
366            filename = "rust-std-{}-{}{}".format(
367                rustc_channel, self.build, tarball_suffix)
368            pattern = "rust-std-{}".format(self.build)
369            self._download_stage0_helper(filename, pattern, tarball_suffix)
370
371            filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
372                                              tarball_suffix)
373            self._download_stage0_helper(filename, "rustc", tarball_suffix)
374            self.fix_executable("{}/bin/rustc".format(self.bin_root()))
375            self.fix_executable("{}/bin/rustdoc".format(self.bin_root()))
376            with output(self.rustc_stamp()) as rust_stamp:
377                rust_stamp.write(self.date)
378
379            # This is required so that we don't mix incompatible MinGW
380            # libraries/binaries that are included in rust-std with
381            # the system MinGW ones.
382            if "pc-windows-gnu" in self.build:
383                filename = "rust-mingw-{}-{}{}".format(
384                    rustc_channel, self.build, tarball_suffix)
385                self._download_stage0_helper(filename, "rust-mingw", tarball_suffix)
386
387        if self.cargo().startswith(self.bin_root()) and \
388                (not os.path.exists(self.cargo()) or
389                 self.program_out_of_date(self.cargo_stamp())):
390            tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
391            filename = "cargo-{}-{}{}".format(cargo_channel, self.build,
392                                              tarball_suffix)
393            self._download_stage0_helper(filename, "cargo", tarball_suffix)
394            self.fix_executable("{}/bin/cargo".format(self.bin_root()))
395            with output(self.cargo_stamp()) as cargo_stamp:
396                cargo_stamp.write(self.date)
397
398        if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
399            not os.path.exists(self.rustfmt())
400            or self.program_out_of_date(self.rustfmt_stamp())
401        ):
402            if rustfmt_channel:
403                tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
404                [channel, date] = rustfmt_channel.split('-', 1)
405                filename = "rustfmt-{}-{}{}".format(channel, self.build, tarball_suffix)
406                self._download_stage0_helper(filename, "rustfmt-preview", tarball_suffix, date)
407                self.fix_executable("{}/bin/rustfmt".format(self.bin_root()))
408                self.fix_executable("{}/bin/cargo-fmt".format(self.bin_root()))
409                with output(self.rustfmt_stamp()) as rustfmt_stamp:
410                    rustfmt_stamp.write(self.date)
411
412    def _download_stage0_helper(self, filename, pattern, tarball_suffix, date=None):
413        if date is None:
414            date = self.date
415        cache_dst = os.path.join(self.build_dir, "cache")
416        rustc_cache = os.path.join(cache_dst, date)
417        if not os.path.exists(rustc_cache):
418            os.makedirs(rustc_cache)
419
420        url = "{}/dist/{}".format(self._download_url, date)
421        tarball = os.path.join(rustc_cache, filename)
422        if not os.path.exists(tarball):
423            get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
424        unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
425
426    @staticmethod
427    def fix_executable(fname):
428        """Modifies the interpreter section of 'fname' to fix the dynamic linker
429
430        This method is only required on NixOS and uses the PatchELF utility to
431        change the dynamic linker of ELF executables.
432
433        Please see https://nixos.org/patchelf.html for more information
434        """
435        default_encoding = sys.getdefaultencoding()
436        try:
437            ostype = subprocess.check_output(
438                ['uname', '-s']).strip().decode(default_encoding)
439        except subprocess.CalledProcessError:
440            return
441        except OSError as reason:
442            if getattr(reason, 'winerror', None) is not None:
443                return
444            raise reason
445
446        if ostype != "Linux":
447            return
448
449        if not os.path.exists("/etc/NIXOS"):
450            return
451        if os.path.exists("/lib"):
452            return
453
454        # At this point we're pretty sure the user is running NixOS
455        nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
456        print(nix_os_msg, fname)
457
458        try:
459            interpreter = subprocess.check_output(
460                ["patchelf", "--print-interpreter", fname])
461            interpreter = interpreter.strip().decode(default_encoding)
462        except subprocess.CalledProcessError as reason:
463            print("warning: failed to call patchelf:", reason)
464            return
465
466        loader = interpreter.split("/")[-1]
467
468        try:
469            ldd_output = subprocess.check_output(
470                ['ldd', '/run/current-system/sw/bin/sh'])
471            ldd_output = ldd_output.strip().decode(default_encoding)
472        except subprocess.CalledProcessError as reason:
473            print("warning: unable to call ldd:", reason)
474            return
475
476        for line in ldd_output.splitlines():
477            libname = line.split()[0]
478            if libname.endswith(loader):
479                loader_path = libname[:len(libname) - len(loader)]
480                break
481        else:
482            print("warning: unable to find the path to the dynamic linker")
483            return
484
485        correct_interpreter = loader_path + loader
486
487        try:
488            subprocess.check_output(
489                ["patchelf", "--set-interpreter", correct_interpreter, fname])
490        except subprocess.CalledProcessError as reason:
491            print("warning: failed to call patchelf:", reason)
492            return
493
494    def rustc_stamp(self):
495        """Return the path for .rustc-stamp
496
497        >>> rb = RustBuild()
498        >>> rb.build_dir = "build"
499        >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
500        True
501        """
502        return os.path.join(self.bin_root(), '.rustc-stamp')
503
504    def cargo_stamp(self):
505        """Return the path for .cargo-stamp
506
507        >>> rb = RustBuild()
508        >>> rb.build_dir = "build"
509        >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
510        True
511        """
512        return os.path.join(self.bin_root(), '.cargo-stamp')
513
514    def rustfmt_stamp(self):
515        """Return the path for .rustfmt-stamp
516
517        >>> rb = RustBuild()
518        >>> rb.build_dir = "build"
519        >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
520        True
521        """
522        return os.path.join(self.bin_root(), '.rustfmt-stamp')
523
524    def program_out_of_date(self, stamp_path):
525        """Check if the given program stamp is out of date"""
526        if not os.path.exists(stamp_path) or self.clean:
527            return True
528        with open(stamp_path, 'r') as stamp:
529            return self.date != stamp.read()
530
531    def bin_root(self):
532        """Return the binary root directory
533
534        >>> rb = RustBuild()
535        >>> rb.build_dir = "build"
536        >>> rb.bin_root() == os.path.join("build", "stage0")
537        True
538
539        When the 'build' property is given should be a nested directory:
540
541        >>> rb.build = "devel"
542        >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
543        True
544        """
545        return os.path.join(self.build_dir, self.build, "stage0")
546
547    def get_toml(self, key, section=None):
548        """Returns the value of the given key in config.toml, otherwise returns None
549
550        >>> rb = RustBuild()
551        >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
552        >>> rb.get_toml("key2")
553        'value2'
554
555        If the key does not exists, the result is None:
556
557        >>> rb.get_toml("key3") is None
558        True
559
560        Optionally also matches the section the key appears in
561
562        >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
563        >>> rb.get_toml('key', 'a')
564        'value1'
565        >>> rb.get_toml('key', 'b')
566        'value2'
567        >>> rb.get_toml('key', 'c') is None
568        True
569
570        >>> rb.config_toml = 'key1 = true'
571        >>> rb.get_toml("key1")
572        'true'
573        """
574
575        cur_section = None
576        for line in self.config_toml.splitlines():
577            section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
578            if section_match is not None:
579                cur_section = section_match.group(1)
580
581            match = re.match(r'^{}\s*=(.*)$'.format(key), line)
582            if match is not None:
583                value = match.group(1)
584                if section is None or section == cur_section:
585                    return self.get_string(value) or value.strip()
586        return None
587
588    def cargo(self):
589        """Return config path for cargo"""
590        return self.program_config('cargo')
591
592    def rustc(self):
593        """Return config path for rustc"""
594        return self.program_config('rustc')
595
596    def rustfmt(self):
597        """Return config path for rustfmt"""
598        if not self.rustfmt_channel:
599            return None
600        return self.program_config('rustfmt')
601
602    def program_config(self, program):
603        """Return config path for the given program
604
605        >>> rb = RustBuild()
606        >>> rb.config_toml = 'rustc = "rustc"\\n'
607        >>> rb.program_config('rustc')
608        'rustc'
609        >>> rb.config_toml = ''
610        >>> cargo_path = rb.program_config('cargo')
611        >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
612        ... "bin", "cargo")
613        True
614        """
615        config = self.get_toml(program)
616        if config:
617            return os.path.expanduser(config)
618        return os.path.join(self.bin_root(), "bin", "{}{}".format(
619            program, self.exe_suffix()))
620
621    @staticmethod
622    def get_string(line):
623        """Return the value between double quotes
624
625        >>> RustBuild.get_string('    "devel"   ')
626        'devel'
627        >>> RustBuild.get_string("    'devel'   ")
628        'devel'
629        >>> RustBuild.get_string('devel') is None
630        True
631        >>> RustBuild.get_string('    "devel   ')
632        ''
633        """
634        start = line.find('"')
635        if start != -1:
636            end = start + 1 + line[start + 1:].find('"')
637            return line[start + 1:end]
638        start = line.find('\'')
639        if start != -1:
640            end = start + 1 + line[start + 1:].find('\'')
641            return line[start + 1:end]
642        return None
643
644    @staticmethod
645    def exe_suffix():
646        """Return a suffix for executables"""
647        if sys.platform == 'win32':
648            return '.exe'
649        return ''
650
651    def bootstrap_binary(self):
652        """Return the path of the bootstrap binary
653
654        >>> rb = RustBuild()
655        >>> rb.build_dir = "build"
656        >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
657        ... "debug", "bootstrap")
658        True
659        """
660        return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
661
662    def build_bootstrap(self):
663        """Build bootstrap"""
664        build_dir = os.path.join(self.build_dir, "bootstrap")
665        if self.clean and os.path.exists(build_dir):
666            shutil.rmtree(build_dir)
667        env = os.environ.copy()
668        env["RUSTC_BOOTSTRAP"] = '1'
669        env["CARGO_TARGET_DIR"] = build_dir
670        env["RUSTC"] = self.rustc()
671        env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
672            (os.pathsep + env["LD_LIBRARY_PATH"]) \
673            if "LD_LIBRARY_PATH" in env else ""
674        env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
675            (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
676            if "DYLD_LIBRARY_PATH" in env else ""
677        env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
678            (os.pathsep + env["LIBRARY_PATH"]) \
679            if "LIBRARY_PATH" in env else ""
680        # preserve existing RUSTFLAGS
681        env.setdefault("RUSTFLAGS", "")
682        env["RUSTFLAGS"] += " -Cdebuginfo=2"
683
684        build_section = "target.{}".format(self.build_triple())
685        target_features = []
686        if self.get_toml("crt-static", build_section) == "true":
687            target_features += ["+crt-static"]
688        elif self.get_toml("crt-static", build_section) == "false":
689            target_features += ["-crt-static"]
690        if target_features:
691            env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
692        target_linker = self.get_toml("linker", build_section)
693        if target_linker is not None:
694            env["RUSTFLAGS"] += " -C linker=" + target_linker
695        env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
696        if self.get_toml("deny-warnings", "rust") != "false":
697            env["RUSTFLAGS"] += " -Dwarnings"
698
699        env["PATH"] = os.path.join(self.bin_root(), "bin") + \
700            os.pathsep + env["PATH"]
701        if not os.path.isfile(self.cargo()):
702            raise Exception("no cargo executable found at `{}`".format(
703                self.cargo()))
704        args = [self.cargo(), "build", "--manifest-path",
705                os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
706        for _ in range(1, self.verbose):
707            args.append("--verbose")
708        if self.use_locked_deps:
709            args.append("--locked")
710        if self.use_vendored_sources:
711            args.append("--frozen")
712        run(args, env=env, verbose=self.verbose)
713
714    def build_triple(self):
715        """Build triple as in LLVM"""
716        config = self.get_toml('build')
717        if config:
718            return config
719        return default_build_triple()
720
721    def check_submodule(self, module, slow_submodules):
722        if not slow_submodules:
723            checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
724                                           cwd=os.path.join(self.rust_root, module),
725                                           stdout=subprocess.PIPE)
726            return checked_out
727        else:
728            return None
729
730    def update_submodule(self, module, checked_out, recorded_submodules):
731        module_path = os.path.join(self.rust_root, module)
732
733        if checked_out is not None:
734            default_encoding = sys.getdefaultencoding()
735            checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
736            if recorded_submodules[module] == checked_out:
737                return
738
739        print("Updating submodule", module)
740
741        run(["git", "submodule", "-q", "sync", module],
742            cwd=self.rust_root, verbose=self.verbose)
743        try:
744            run(["git", "submodule", "update",
745                 "--init", "--recursive", "--progress", module],
746                cwd=self.rust_root, verbose=self.verbose, exception=True)
747        except RuntimeError:
748            # Some versions of git don't support --progress.
749            run(["git", "submodule", "update",
750                 "--init", "--recursive", module],
751                cwd=self.rust_root, verbose=self.verbose)
752        run(["git", "reset", "-q", "--hard"],
753            cwd=module_path, verbose=self.verbose)
754        run(["git", "clean", "-qdfx"],
755            cwd=module_path, verbose=self.verbose)
756
757    def update_submodules(self):
758        """Update submodules"""
759        if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
760                self.get_toml('submodules') == "false":
761            return
762
763        # check the existence of 'git' command
764        try:
765            subprocess.check_output(['git', '--version'])
766        except (subprocess.CalledProcessError, OSError):
767            print("error: `git` is not found, please make sure it's installed and in the path.")
768            sys.exit(1)
769
770        slow_submodules = self.get_toml('fast-submodules') == "false"
771        start_time = time()
772        if slow_submodules:
773            print('Unconditionally updating all submodules')
774        else:
775            print('Updating only changed submodules')
776        default_encoding = sys.getdefaultencoding()
777        submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
778            ["git", "config", "--file",
779             os.path.join(self.rust_root, ".gitmodules"),
780             "--get-regexp", "path"]
781        ).decode(default_encoding).splitlines()]
782        filtered_submodules = []
783        submodules_names = []
784        for module in submodules:
785            if module.endswith("llvm-project"):
786                if self.get_toml('llvm-config') and self.get_toml('lld') != 'true':
787                    continue
788            check = self.check_submodule(module, slow_submodules)
789            filtered_submodules.append((module, check))
790            submodules_names.append(module)
791        recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
792                                    cwd=self.rust_root, stdout=subprocess.PIPE)
793        recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
794        recorded_submodules = {}
795        for data in recorded:
796            data = data.split()
797            recorded_submodules[data[3]] = data[2]
798        for module in filtered_submodules:
799            self.update_submodule(module[0], module[1], recorded_submodules)
800        print("Submodules updated in %.2f seconds" % (time() - start_time))
801
802    def set_normal_environment(self):
803        """Set download URL for normal environment"""
804        if 'RUSTUP_DIST_SERVER' in os.environ:
805            self._download_url = os.environ['RUSTUP_DIST_SERVER']
806        else:
807            self._download_url = 'https://static.rust-lang.org'
808
809    def set_dev_environment(self):
810        """Set download URL for development environment"""
811        if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
812            self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
813        else:
814            self._download_url = 'https://dev-static.rust-lang.org'
815
816    def check_vendored_status(self):
817        """Check that vendoring is configured properly"""
818        vendor_dir = os.path.join(self.rust_root, 'vendor')
819        if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
820            if os.environ.get('USER') != os.environ['SUDO_USER']:
821                self.use_vendored_sources = True
822                print('info: looks like you are running this command under `sudo`')
823                print('      and so in order to preserve your $HOME this will now')
824                print('      use vendored sources by default.')
825                if not os.path.exists(vendor_dir):
826                    print('error: vendoring required, but vendor directory does not exist.')
827                    print('       Run `cargo vendor` without sudo to initialize the '
828                        'vendor directory.')
829                    raise Exception("{} not found".format(vendor_dir))
830
831        if self.use_vendored_sources:
832            if not os.path.exists('.cargo'):
833                os.makedirs('.cargo')
834            with output('.cargo/config') as cargo_config:
835                cargo_config.write(
836                    "[source.crates-io]\n"
837                    "replace-with = 'vendored-sources'\n"
838                    "registry = 'https://example.com'\n"
839                    "\n"
840                    "[source.vendored-sources]\n"
841                    "directory = '{}/vendor'\n"
842                .format(self.rust_root))
843        else:
844            if os.path.exists('.cargo'):
845                shutil.rmtree('.cargo')
846
847    def ensure_vendored(self):
848        """Ensure that the vendored sources are available if needed"""
849        vendor_dir = os.path.join(self.rust_root, 'vendor')
850        # Note that this does not handle updating the vendored dependencies if
851        # the rust git repository is updated. Normal development usually does
852        # not use vendoring, so hopefully this isn't too much of a problem.
853        if self.use_vendored_sources and not os.path.exists(vendor_dir):
854            run([self.cargo(), "vendor"],
855                verbose=self.verbose, cwd=self.rust_root)
856
857
858def bootstrap(help_triggered):
859    """Configure, fetch, build and run the initial bootstrap"""
860
861    # If the user is asking for help, let them know that the whole download-and-build
862    # process has to happen before anything is printed out.
863    if help_triggered:
864        print("info: Downloading and building bootstrap before processing --help")
865        print("      command. See src/bootstrap/README.md for help with common")
866        print("      commands.")
867
868    parser = argparse.ArgumentParser(description='Build rust')
869    parser.add_argument('--config')
870    parser.add_argument('--build')
871    parser.add_argument('--src')
872    parser.add_argument('--clean', action='store_true')
873    parser.add_argument('-v', '--verbose', action='count', default=0)
874
875    args = [a for a in sys.argv if a != '-h' and a != '--help']
876    args, _ = parser.parse_known_args(args)
877
878    # Configure initial bootstrap
879    build = RustBuild()
880    build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..'))
881    build.verbose = args.verbose
882    build.clean = args.clean
883
884    try:
885        with open(args.config or 'config.toml') as config:
886            build.config_toml = config.read()
887    except (OSError, IOError):
888        pass
889
890    config_verbose = build.get_toml('verbose', 'build')
891    if config_verbose is not None:
892        build.verbose = max(build.verbose, int(config_verbose))
893
894    build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
895
896    build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
897
898    build.check_vendored_status()
899
900    data = stage0_data(build.rust_root)
901    build.date = data['date']
902    build.rustc_channel = data['rustc']
903    build.cargo_channel = data['cargo']
904
905    if "rustfmt" in data:
906        build.rustfmt_channel = data['rustfmt']
907
908    if 'dev' in data:
909        build.set_dev_environment()
910    else:
911        build.set_normal_environment()
912
913    build.update_submodules()
914
915    # Fetch/build the bootstrap
916    build.build = args.build or build.build_triple()
917    build.download_stage0()
918    sys.stdout.flush()
919    build.ensure_vendored()
920    build.build_bootstrap()
921    sys.stdout.flush()
922
923    # Run the bootstrap
924    args = [build.bootstrap_binary()]
925    args.extend(sys.argv[1:])
926    env = os.environ.copy()
927    env["BUILD"] = build.build
928    env["SRC"] = build.rust_root
929    env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
930    env["BOOTSTRAP_PYTHON"] = sys.executable
931    env["BUILD_DIR"] = build.build_dir
932    env["RUSTC_BOOTSTRAP"] = '1'
933    env["CARGO"] = build.cargo()
934    env["RUSTC"] = build.rustc()
935    if build.rustfmt():
936        env["RUSTFMT"] = build.rustfmt()
937    run(args, env=env, verbose=build.verbose)
938
939
940def main():
941    """Entry point for the bootstrap process"""
942    start_time = time()
943
944    # x.py help <cmd> ...
945    if len(sys.argv) > 1 and sys.argv[1] == 'help':
946        sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
947
948    help_triggered = (
949        '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
950    try:
951        bootstrap(help_triggered)
952        if not help_triggered:
953            print("Build completed successfully in {}".format(
954                format_build_time(time() - start_time)))
955    except (SystemExit, KeyboardInterrupt) as error:
956        if hasattr(error, 'code') and isinstance(error.code, int):
957            exit_code = error.code
958        else:
959            exit_code = 1
960            print(error)
961        if not help_triggered:
962            print("Build completed unsuccessfully in {}".format(
963                format_build_time(time() - start_time)))
964        sys.exit(exit_code)
965
966
967if __name__ == '__main__':
968    main()