fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #67350 - Mark-Simulacrum:stable-next, r=Mark-Simulacrum73528e339aa

1.40.0
33.0 KiB912 linesraw
1from __future__ import absolute_import, division, print_function
2import argparse
3import contextlib
4import datetime
5import hashlib
6import os
7import re
8import shutil
9import subprocess
10import sys
11import tarfile
12import tempfile
13
14from time import time
15
16
17def get(url, path, verbose=False):
18    suffix = '.sha256'
19    sha_url = url + suffix
20    with tempfile.NamedTemporaryFile(delete=False) as temp_file:
21        temp_path = temp_file.name
22    with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
23        sha_path = sha_file.name
24
25    try:
26        download(sha_path, sha_url, False, verbose)
27        if os.path.exists(path):
28            if verify(path, sha_path, False):
29                if verbose:
30                    print("using already-download file", path)
31                return
32            else:
33                if verbose:
34                    print("ignoring already-download file",
35                          path, "due to failed verification")
36                os.unlink(path)
37        download(temp_path, url, True, verbose)
38        if not verify(temp_path, sha_path, verbose):
39            raise RuntimeError("failed verification")
40        if verbose:
41            print("moving {} to {}".format(temp_path, path))
42        shutil.move(temp_path, path)
43    finally:
44        delete_if_present(sha_path, verbose)
45        delete_if_present(temp_path, verbose)
46
47
48def delete_if_present(path, verbose):
49    """Remove the given file if present"""
50    if os.path.isfile(path):
51        if verbose:
52            print("removing", path)
53        os.unlink(path)
54
55
56def download(path, url, probably_big, verbose):
57    for _ in range(0, 4):
58        try:
59            _download(path, url, probably_big, verbose, True)
60            return
61        except RuntimeError:
62            print("\nspurious failure, trying again")
63    _download(path, url, probably_big, verbose, False)
64
65
66def _download(path, url, probably_big, verbose, exception):
67    if probably_big or verbose:
68        print("downloading {}".format(url))
69    # see http://serverfault.com/questions/301128/how-to-download
70    if sys.platform == 'win32':
71        run(["PowerShell.exe", "/nologo", "-Command",
72             "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
73             "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
74            verbose=verbose,
75            exception=exception)
76    else:
77        if probably_big or verbose:
78            option = "-#"
79        else:
80            option = "-s"
81        run(["curl", option,
82             "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
83             "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
84             "--retry", "3", "-Sf", "-o", path, url],
85            verbose=verbose,
86            exception=exception)
87
88
89def verify(path, sha_path, verbose):
90    """Check if the sha256 sum of the given path is valid"""
91    if verbose:
92        print("verifying", path)
93    with open(path, "rb") as source:
94        found = hashlib.sha256(source.read()).hexdigest()
95    with open(sha_path, "r") as sha256sum:
96        expected = sha256sum.readline().split()[0]
97    verified = found == expected
98    if not verified:
99        print("invalid checksum:\n"
100              "    found:    {}\n"
101              "    expected: {}".format(found, expected))
102    return verified
103
104
105def unpack(tarball, dst, verbose=False, match=None):
106    """Unpack the given tarball file"""
107    print("extracting", tarball)
108    fname = os.path.basename(tarball).replace(".tar.gz", "")
109    with contextlib.closing(tarfile.open(tarball)) as tar:
110        for member in tar.getnames():
111            if "/" not in member:
112                continue
113            name = member.replace(fname + "/", "", 1)
114            if match is not None and not name.startswith(match):
115                continue
116            name = name[len(match) + 1:]
117
118            dst_path = os.path.join(dst, name)
119            if verbose:
120                print("  extracting", member)
121            tar.extract(member, dst)
122            src_path = os.path.join(dst, member)
123            if os.path.isdir(src_path) and os.path.exists(dst_path):
124                continue
125            shutil.move(src_path, dst_path)
126    shutil.rmtree(os.path.join(dst, fname))
127
128
129def run(args, verbose=False, exception=False, **kwargs):
130    """Run a child program in a new process"""
131    if verbose:
132        print("running: " + ' '.join(args))
133    sys.stdout.flush()
134    # Use Popen here instead of call() as it apparently allows powershell on
135    # Windows to not lock up waiting for input presumably.
136    ret = subprocess.Popen(args, **kwargs)
137    code = ret.wait()
138    if code != 0:
139        err = "failed to run: " + ' '.join(args)
140        if verbose or exception:
141            raise RuntimeError(err)
142        sys.exit(err)
143
144
145def stage0_data(rust_root):
146    """Build a dictionary from stage0.txt"""
147    nightlies = os.path.join(rust_root, "src/stage0.txt")
148    with open(nightlies, 'r') as nightlies:
149        lines = [line.rstrip() for line in nightlies
150                 if not line.startswith("#")]
151        return dict([line.split(": ", 1) for line in lines if line])
152
153
154def format_build_time(duration):
155    """Return a nicer format for build time
156
157    >>> format_build_time('300')
158    '0:05:00'
159    """
160    return str(datetime.timedelta(seconds=int(duration)))
161
162
163def default_build_triple():
164    """Build triple as in LLVM"""
165    default_encoding = sys.getdefaultencoding()
166    try:
167        ostype = subprocess.check_output(
168            ['uname', '-s']).strip().decode(default_encoding)
169        cputype = subprocess.check_output(
170            ['uname', '-m']).strip().decode(default_encoding)
171    except (subprocess.CalledProcessError, OSError):
172        if sys.platform == 'win32':
173            return 'x86_64-pc-windows-msvc'
174        err = "uname not found"
175        sys.exit(err)
176
177    # The goal here is to come up with the same triple as LLVM would,
178    # at least for the subset of platforms we're willing to target.
179    ostype_mapper = {
180        'Darwin': 'apple-darwin',
181        'DragonFly': 'unknown-dragonfly',
182        'FreeBSD': 'unknown-freebsd',
183        'Haiku': 'unknown-haiku',
184        'NetBSD': 'unknown-netbsd',
185        'OpenBSD': 'unknown-openbsd'
186    }
187
188    # Consider the direct transformation first and then the special cases
189    if ostype in ostype_mapper:
190        ostype = ostype_mapper[ostype]
191    elif ostype == 'Linux':
192        os_from_sp = subprocess.check_output(
193            ['uname', '-o']).strip().decode(default_encoding)
194        if os_from_sp == 'Android':
195            ostype = 'linux-android'
196        else:
197            ostype = 'unknown-linux-gnu'
198    elif ostype == 'SunOS':
199        ostype = 'sun-solaris'
200        # On Solaris, uname -m will return a machine classification instead
201        # of a cpu type, so uname -p is recommended instead.  However, the
202        # output from that option is too generic for our purposes (it will
203        # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
204        # must be used instead.
205        try:
206            cputype = subprocess.check_output(
207                ['isainfo', '-k']).strip().decode(default_encoding)
208        except (subprocess.CalledProcessError, OSError):
209            err = "isainfo not found"
210            sys.exit(err)
211    elif ostype.startswith('MINGW'):
212        # msys' `uname` does not print gcc configuration, but prints msys
213        # configuration. so we cannot believe `uname -m`:
214        # msys1 is always i686 and msys2 is always x86_64.
215        # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
216        # MINGW64 on x86_64.
217        ostype = 'pc-windows-gnu'
218        cputype = 'i686'
219        if os.environ.get('MSYSTEM') == 'MINGW64':
220            cputype = 'x86_64'
221    elif ostype.startswith('MSYS'):
222        ostype = 'pc-windows-gnu'
223    elif ostype.startswith('CYGWIN_NT'):
224        cputype = 'i686'
225        if ostype.endswith('WOW64'):
226            cputype = 'x86_64'
227        ostype = 'pc-windows-gnu'
228    else:
229        err = "unknown OS type: {}".format(ostype)
230        sys.exit(err)
231
232    if cputype == 'powerpc' and ostype == 'unknown-freebsd':
233        cputype = subprocess.check_output(
234              ['uname', '-p']).strip().decode(default_encoding)
235    cputype_mapper = {
236        'BePC': 'i686',
237        'aarch64': 'aarch64',
238        'amd64': 'x86_64',
239        'arm64': 'aarch64',
240        'i386': 'i686',
241        'i486': 'i686',
242        'i686': 'i686',
243        'i786': 'i686',
244        'powerpc': 'powerpc',
245        'powerpc64': 'powerpc64',
246        'powerpc64le': 'powerpc64le',
247        'ppc': 'powerpc',
248        'ppc64': 'powerpc64',
249        'ppc64le': 'powerpc64le',
250        's390x': 's390x',
251        'x64': 'x86_64',
252        'x86': 'i686',
253        'x86-64': 'x86_64',
254        'x86_64': 'x86_64'
255    }
256
257    # Consider the direct transformation first and then the special cases
258    if cputype in cputype_mapper:
259        cputype = cputype_mapper[cputype]
260    elif cputype in {'xscale', 'arm'}:
261        cputype = 'arm'
262        if ostype == 'linux-android':
263            ostype = 'linux-androideabi'
264        elif ostype == 'unknown-freebsd':
265            cputype = subprocess.check_output(
266                ['uname', '-p']).strip().decode(default_encoding)
267            ostype = 'unknown-freebsd'
268    elif cputype == 'armv6l':
269        cputype = 'arm'
270        if ostype == 'linux-android':
271            ostype = 'linux-androideabi'
272        else:
273            ostype += 'eabihf'
274    elif cputype in {'armv7l', 'armv8l'}:
275        cputype = 'armv7'
276        if ostype == 'linux-android':
277            ostype = 'linux-androideabi'
278        else:
279            ostype += 'eabihf'
280    elif cputype == 'mips':
281        if sys.byteorder == 'big':
282            cputype = 'mips'
283        elif sys.byteorder == 'little':
284            cputype = 'mipsel'
285        else:
286            raise ValueError("unknown byteorder: {}".format(sys.byteorder))
287    elif cputype == 'mips64':
288        if sys.byteorder == 'big':
289            cputype = 'mips64'
290        elif sys.byteorder == 'little':
291            cputype = 'mips64el'
292        else:
293            raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
294        # only the n64 ABI is supported, indicate it
295        ostype += 'abi64'
296    elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
297        pass
298    else:
299        err = "unknown cpu type: {}".format(cputype)
300        sys.exit(err)
301
302    return "{}-{}".format(cputype, ostype)
303
304
305@contextlib.contextmanager
306def output(filepath):
307    tmp = filepath + '.tmp'
308    with open(tmp, 'w') as f:
309        yield f
310    try:
311        os.remove(filepath)  # PermissionError/OSError on Win32 if in use
312        os.rename(tmp, filepath)
313    except OSError:
314        shutil.copy2(tmp, filepath)
315        os.remove(tmp)
316
317
318class RustBuild(object):
319    """Provide all the methods required to build Rust"""
320    def __init__(self):
321        self.cargo_channel = ''
322        self.date = ''
323        self._download_url = ''
324        self.rustc_channel = ''
325        self.build = ''
326        self.build_dir = os.path.join(os.getcwd(), "build")
327        self.clean = False
328        self.config_toml = ''
329        self.rust_root = ''
330        self.use_locked_deps = ''
331        self.use_vendored_sources = ''
332        self.verbose = False
333
334    def download_stage0(self):
335        """Fetch the build system for Rust, written in Rust
336
337        This method will build a cache directory, then it will fetch the
338        tarball which has the stage0 compiler used to then bootstrap the Rust
339        compiler itself.
340
341        Each downloaded tarball is extracted, after that, the script
342        will move all the content to the right place.
343        """
344        rustc_channel = self.rustc_channel
345        cargo_channel = self.cargo_channel
346
347        if self.rustc().startswith(self.bin_root()) and \
348                (not os.path.exists(self.rustc()) or
349                 self.program_out_of_date(self.rustc_stamp())):
350            if os.path.exists(self.bin_root()):
351                shutil.rmtree(self.bin_root())
352            filename = "rust-std-{}-{}.tar.gz".format(
353                rustc_channel, self.build)
354            pattern = "rust-std-{}".format(self.build)
355            self._download_stage0_helper(filename, pattern)
356
357            filename = "rustc-{}-{}.tar.gz".format(rustc_channel, self.build)
358            self._download_stage0_helper(filename, "rustc")
359            self.fix_executable("{}/bin/rustc".format(self.bin_root()))
360            self.fix_executable("{}/bin/rustdoc".format(self.bin_root()))
361            with output(self.rustc_stamp()) as rust_stamp:
362                rust_stamp.write(self.date)
363
364            # This is required so that we don't mix incompatible MinGW
365            # libraries/binaries that are included in rust-std with
366            # the system MinGW ones.
367            if "pc-windows-gnu" in self.build:
368                filename = "rust-mingw-{}-{}.tar.gz".format(
369                    rustc_channel, self.build)
370                self._download_stage0_helper(filename, "rust-mingw")
371
372        if self.cargo().startswith(self.bin_root()) and \
373                (not os.path.exists(self.cargo()) or
374                 self.program_out_of_date(self.cargo_stamp())):
375            filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build)
376            self._download_stage0_helper(filename, "cargo")
377            self.fix_executable("{}/bin/cargo".format(self.bin_root()))
378            with output(self.cargo_stamp()) as cargo_stamp:
379                cargo_stamp.write(self.date)
380
381    def _download_stage0_helper(self, filename, pattern):
382        cache_dst = os.path.join(self.build_dir, "cache")
383        rustc_cache = os.path.join(cache_dst, self.date)
384        if not os.path.exists(rustc_cache):
385            os.makedirs(rustc_cache)
386
387        url = "{}/dist/{}".format(self._download_url, self.date)
388        tarball = os.path.join(rustc_cache, filename)
389        if not os.path.exists(tarball):
390            get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
391        unpack(tarball, self.bin_root(), match=pattern, verbose=self.verbose)
392
393    @staticmethod
394    def fix_executable(fname):
395        """Modifies the interpreter section of 'fname' to fix the dynamic linker
396
397        This method is only required on NixOS and uses the PatchELF utility to
398        change the dynamic linker of ELF executables.
399
400        Please see https://nixos.org/patchelf.html for more information
401        """
402        default_encoding = sys.getdefaultencoding()
403        try:
404            ostype = subprocess.check_output(
405                ['uname', '-s']).strip().decode(default_encoding)
406        except subprocess.CalledProcessError:
407            return
408        except OSError as reason:
409            if getattr(reason, 'winerror', None) is not None:
410                return
411            raise reason
412
413        if ostype != "Linux":
414            return
415
416        if not os.path.exists("/etc/NIXOS"):
417            return
418        if os.path.exists("/lib"):
419            return
420
421        # At this point we're pretty sure the user is running NixOS
422        nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
423        print(nix_os_msg, fname)
424
425        try:
426            interpreter = subprocess.check_output(
427                ["patchelf", "--print-interpreter", fname])
428            interpreter = interpreter.strip().decode(default_encoding)
429        except subprocess.CalledProcessError as reason:
430            print("warning: failed to call patchelf:", reason)
431            return
432
433        loader = interpreter.split("/")[-1]
434
435        try:
436            ldd_output = subprocess.check_output(
437                ['ldd', '/run/current-system/sw/bin/sh'])
438            ldd_output = ldd_output.strip().decode(default_encoding)
439        except subprocess.CalledProcessError as reason:
440            print("warning: unable to call ldd:", reason)
441            return
442
443        for line in ldd_output.splitlines():
444            libname = line.split()[0]
445            if libname.endswith(loader):
446                loader_path = libname[:len(libname) - len(loader)]
447                break
448        else:
449            print("warning: unable to find the path to the dynamic linker")
450            return
451
452        correct_interpreter = loader_path + loader
453
454        try:
455            subprocess.check_output(
456                ["patchelf", "--set-interpreter", correct_interpreter, fname])
457        except subprocess.CalledProcessError as reason:
458            print("warning: failed to call patchelf:", reason)
459            return
460
461    def rustc_stamp(self):
462        """Return the path for .rustc-stamp
463
464        >>> rb = RustBuild()
465        >>> rb.build_dir = "build"
466        >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
467        True
468        """
469        return os.path.join(self.bin_root(), '.rustc-stamp')
470
471    def cargo_stamp(self):
472        """Return the path for .cargo-stamp
473
474        >>> rb = RustBuild()
475        >>> rb.build_dir = "build"
476        >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
477        True
478        """
479        return os.path.join(self.bin_root(), '.cargo-stamp')
480
481    def program_out_of_date(self, stamp_path):
482        """Check if the given program stamp is out of date"""
483        if not os.path.exists(stamp_path) or self.clean:
484            return True
485        with open(stamp_path, 'r') as stamp:
486            return self.date != stamp.read()
487
488    def bin_root(self):
489        """Return the binary root directory
490
491        >>> rb = RustBuild()
492        >>> rb.build_dir = "build"
493        >>> rb.bin_root() == os.path.join("build", "stage0")
494        True
495
496        When the 'build' property is given should be a nested directory:
497
498        >>> rb.build = "devel"
499        >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
500        True
501        """
502        return os.path.join(self.build_dir, self.build, "stage0")
503
504    def get_toml(self, key, section=None):
505        """Returns the value of the given key in config.toml, otherwise returns None
506
507        >>> rb = RustBuild()
508        >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
509        >>> rb.get_toml("key2")
510        'value2'
511
512        If the key does not exists, the result is None:
513
514        >>> rb.get_toml("key3") is None
515        True
516
517        Optionally also matches the section the key appears in
518
519        >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
520        >>> rb.get_toml('key', 'a')
521        'value1'
522        >>> rb.get_toml('key', 'b')
523        'value2'
524        >>> rb.get_toml('key', 'c') is None
525        True
526
527        >>> rb.config_toml = 'key1 = true'
528        >>> rb.get_toml("key1")
529        'true'
530        """
531
532        cur_section = None
533        for line in self.config_toml.splitlines():
534            section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
535            if section_match is not None:
536                cur_section = section_match.group(1)
537
538            match = re.match(r'^{}\s*=(.*)$'.format(key), line)
539            if match is not None:
540                value = match.group(1)
541                if section is None or section == cur_section:
542                    return self.get_string(value) or value.strip()
543        return None
544
545    def cargo(self):
546        """Return config path for cargo"""
547        return self.program_config('cargo')
548
549    def rustc(self):
550        """Return config path for rustc"""
551        return self.program_config('rustc')
552
553    def program_config(self, program):
554        """Return config path for the given program
555
556        >>> rb = RustBuild()
557        >>> rb.config_toml = 'rustc = "rustc"\\n'
558        >>> rb.program_config('rustc')
559        'rustc'
560        >>> rb.config_toml = ''
561        >>> cargo_path = rb.program_config('cargo')
562        >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
563        ... "bin", "cargo")
564        True
565        """
566        config = self.get_toml(program)
567        if config:
568            return os.path.expanduser(config)
569        return os.path.join(self.bin_root(), "bin", "{}{}".format(
570            program, self.exe_suffix()))
571
572    @staticmethod
573    def get_string(line):
574        """Return the value between double quotes
575
576        >>> RustBuild.get_string('    "devel"   ')
577        'devel'
578        >>> RustBuild.get_string("    'devel'   ")
579        'devel'
580        >>> RustBuild.get_string('devel') is None
581        True
582        >>> RustBuild.get_string('    "devel   ')
583        ''
584        """
585        start = line.find('"')
586        if start != -1:
587            end = start + 1 + line[start + 1:].find('"')
588            return line[start + 1:end]
589        start = line.find('\'')
590        if start != -1:
591            end = start + 1 + line[start + 1:].find('\'')
592            return line[start + 1:end]
593        return None
594
595    @staticmethod
596    def exe_suffix():
597        """Return a suffix for executables"""
598        if sys.platform == 'win32':
599            return '.exe'
600        return ''
601
602    def bootstrap_binary(self):
603        """Return the path of the bootstrap binary
604
605        >>> rb = RustBuild()
606        >>> rb.build_dir = "build"
607        >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
608        ... "debug", "bootstrap")
609        True
610        """
611        return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
612
613    def build_bootstrap(self):
614        """Build bootstrap"""
615        build_dir = os.path.join(self.build_dir, "bootstrap")
616        if self.clean and os.path.exists(build_dir):
617            shutil.rmtree(build_dir)
618        env = os.environ.copy()
619        env["RUSTC_BOOTSTRAP"] = '1'
620        env["CARGO_TARGET_DIR"] = build_dir
621        env["RUSTC"] = self.rustc()
622        env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
623            (os.pathsep + env["LD_LIBRARY_PATH"]) \
624            if "LD_LIBRARY_PATH" in env else ""
625        env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
626            (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
627            if "DYLD_LIBRARY_PATH" in env else ""
628        env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
629            (os.pathsep + env["LIBRARY_PATH"]) \
630            if "LIBRARY_PATH" in env else ""
631        env["RUSTFLAGS"] = "-Cdebuginfo=2 "
632
633        build_section = "target.{}".format(self.build_triple())
634        target_features = []
635        if self.get_toml("crt-static", build_section) == "true":
636            target_features += ["+crt-static"]
637        elif self.get_toml("crt-static", build_section) == "false":
638            target_features += ["-crt-static"]
639        if target_features:
640            env["RUSTFLAGS"] += "-C target-feature=" + (",".join(target_features)) + " "
641        target_linker = self.get_toml("linker", build_section)
642        if target_linker is not None:
643            env["RUSTFLAGS"] += "-C linker=" + target_linker + " "
644        env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes "
645        if self.get_toml("deny-warnings", "rust") != "false":
646            env["RUSTFLAGS"] += "-Dwarnings "
647
648        env["PATH"] = os.path.join(self.bin_root(), "bin") + \
649            os.pathsep + env["PATH"]
650        if not os.path.isfile(self.cargo()):
651            raise Exception("no cargo executable found at `{}`".format(
652                self.cargo()))
653        args = [self.cargo(), "build", "--manifest-path",
654                os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
655        for _ in range(1, self.verbose):
656            args.append("--verbose")
657        if self.use_locked_deps:
658            args.append("--locked")
659        if self.use_vendored_sources:
660            args.append("--frozen")
661        run(args, env=env, verbose=self.verbose)
662
663    def build_triple(self):
664        """Build triple as in LLVM"""
665        config = self.get_toml('build')
666        if config:
667            return config
668        return default_build_triple()
669
670    def check_submodule(self, module, slow_submodules):
671        if not slow_submodules:
672            checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
673                                           cwd=os.path.join(self.rust_root, module),
674                                           stdout=subprocess.PIPE)
675            return checked_out
676        else:
677            return None
678
679    def update_submodule(self, module, checked_out, recorded_submodules):
680        module_path = os.path.join(self.rust_root, module)
681
682        if checked_out is not None:
683            default_encoding = sys.getdefaultencoding()
684            checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
685            if recorded_submodules[module] == checked_out:
686                return
687
688        print("Updating submodule", module)
689
690        run(["git", "submodule", "-q", "sync", module],
691            cwd=self.rust_root, verbose=self.verbose)
692        try:
693            run(["git", "submodule", "update",
694                 "--init", "--recursive", "--progress", module],
695                cwd=self.rust_root, verbose=self.verbose, exception=True)
696        except RuntimeError:
697            # Some versions of git don't support --progress.
698            run(["git", "submodule", "update",
699                 "--init", "--recursive", module],
700                cwd=self.rust_root, verbose=self.verbose)
701        run(["git", "reset", "-q", "--hard"],
702            cwd=module_path, verbose=self.verbose)
703        run(["git", "clean", "-qdfx"],
704            cwd=module_path, verbose=self.verbose)
705
706    def update_submodules(self):
707        """Update submodules"""
708        if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
709                self.get_toml('submodules') == "false":
710            return
711
712        # check the existence of 'git' command
713        try:
714            subprocess.check_output(['git', '--version'])
715        except (subprocess.CalledProcessError, OSError):
716            print("error: `git` is not found, please make sure it's installed and in the path.")
717            sys.exit(1)
718
719        slow_submodules = self.get_toml('fast-submodules') == "false"
720        start_time = time()
721        if slow_submodules:
722            print('Unconditionally updating all submodules')
723        else:
724            print('Updating only changed submodules')
725        default_encoding = sys.getdefaultencoding()
726        submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
727            ["git", "config", "--file",
728             os.path.join(self.rust_root, ".gitmodules"),
729             "--get-regexp", "path"]
730        ).decode(default_encoding).splitlines()]
731        filtered_submodules = []
732        submodules_names = []
733        for module in submodules:
734            if module.endswith("llvm-project"):
735                if self.get_toml('llvm-config') and self.get_toml('lld') != 'true':
736                    continue
737            check = self.check_submodule(module, slow_submodules)
738            filtered_submodules.append((module, check))
739            submodules_names.append(module)
740        recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
741                                    cwd=self.rust_root, stdout=subprocess.PIPE)
742        recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
743        recorded_submodules = {}
744        for data in recorded:
745            data = data.split()
746            recorded_submodules[data[3]] = data[2]
747        for module in filtered_submodules:
748            self.update_submodule(module[0], module[1], recorded_submodules)
749        print("Submodules updated in %.2f seconds" % (time() - start_time))
750
751    def set_normal_environment(self):
752        """Set download URL for normal environment"""
753        if 'RUSTUP_DIST_SERVER' in os.environ:
754            self._download_url = os.environ['RUSTUP_DIST_SERVER']
755        else:
756            self._download_url = 'https://static.rust-lang.org'
757
758    def set_dev_environment(self):
759        """Set download URL for development environment"""
760        if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
761            self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
762        else:
763            self._download_url = 'https://dev-static.rust-lang.org'
764
765    def check_vendored_status(self):
766        """Check that vendoring is configured properly"""
767        vendor_dir = os.path.join(self.rust_root, 'vendor')
768        if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
769            if os.environ.get('USER') != os.environ['SUDO_USER']:
770                self.use_vendored_sources = True
771                print('info: looks like you are running this command under `sudo`')
772                print('      and so in order to preserve your $HOME this will now')
773                print('      use vendored sources by default.')
774                if not os.path.exists(vendor_dir):
775                    print('error: vendoring required, but vendor directory does not exist.')
776                    print('       Run `cargo vendor` without sudo to initialize the '
777                        'vendor directory.')
778                    raise Exception("{} not found".format(vendor_dir))
779
780        if self.use_vendored_sources:
781            if not os.path.exists('.cargo'):
782                os.makedirs('.cargo')
783            with output('.cargo/config') as cargo_config:
784                cargo_config.write(
785                    "[source.crates-io]\n"
786                    "replace-with = 'vendored-sources'\n"
787                    "registry = 'https://example.com'\n"
788                    "\n"
789                    "[source.vendored-sources]\n"
790                    "directory = '{}/vendor'\n"
791                .format(self.rust_root))
792        else:
793            if os.path.exists('.cargo'):
794                shutil.rmtree('.cargo')
795
796    def ensure_vendored(self):
797        """Ensure that the vendored sources are available if needed"""
798        vendor_dir = os.path.join(self.rust_root, 'vendor')
799        # Note that this does not handle updating the vendored dependencies if
800        # the rust git repository is updated. Normal development usually does
801        # not use vendoring, so hopefully this isn't too much of a problem.
802        if self.use_vendored_sources and not os.path.exists(vendor_dir):
803            run([self.cargo(), "vendor"],
804                verbose=self.verbose, cwd=self.rust_root)
805
806
807def bootstrap(help_triggered):
808    """Configure, fetch, build and run the initial bootstrap"""
809
810    # If the user is asking for help, let them know that the whole download-and-build
811    # process has to happen before anything is printed out.
812    if help_triggered:
813        print("info: Downloading and building bootstrap before processing --help")
814        print("      command. See src/bootstrap/README.md for help with common")
815        print("      commands.")
816
817    parser = argparse.ArgumentParser(description='Build rust')
818    parser.add_argument('--config')
819    parser.add_argument('--build')
820    parser.add_argument('--src')
821    parser.add_argument('--clean', action='store_true')
822    parser.add_argument('-v', '--verbose', action='count', default=0)
823
824    args = [a for a in sys.argv if a != '-h' and a != '--help']
825    args, _ = parser.parse_known_args(args)
826
827    # Configure initial bootstrap
828    build = RustBuild()
829    build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..'))
830    build.verbose = args.verbose
831    build.clean = args.clean
832
833    try:
834        with open(args.config or 'config.toml') as config:
835            build.config_toml = config.read()
836    except (OSError, IOError):
837        pass
838
839    config_verbose = build.get_toml('verbose', 'build')
840    if config_verbose is not None:
841        build.verbose = max(build.verbose, int(config_verbose))
842
843    build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
844
845    build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
846
847    build.check_vendored_status()
848
849    data = stage0_data(build.rust_root)
850    build.date = data['date']
851    build.rustc_channel = data['rustc']
852    build.cargo_channel = data['cargo']
853
854    if 'dev' in data:
855        build.set_dev_environment()
856    else:
857        build.set_normal_environment()
858
859    build.update_submodules()
860
861    # Fetch/build the bootstrap
862    build.build = args.build or build.build_triple()
863    build.download_stage0()
864    sys.stdout.flush()
865    build.ensure_vendored()
866    build.build_bootstrap()
867    sys.stdout.flush()
868
869    # Run the bootstrap
870    args = [build.bootstrap_binary()]
871    args.extend(sys.argv[1:])
872    env = os.environ.copy()
873    env["BUILD"] = build.build
874    env["SRC"] = build.rust_root
875    env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
876    env["BOOTSTRAP_PYTHON"] = sys.executable
877    env["BUILD_DIR"] = build.build_dir
878    env["RUSTC_BOOTSTRAP"] = '1'
879    env["CARGO"] = build.cargo()
880    env["RUSTC"] = build.rustc()
881    run(args, env=env, verbose=build.verbose)
882
883
884def main():
885    """Entry point for the bootstrap process"""
886    start_time = time()
887
888    # x.py help <cmd> ...
889    if len(sys.argv) > 1 and sys.argv[1] == 'help':
890        sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
891
892    help_triggered = (
893        '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
894    try:
895        bootstrap(help_triggered)
896        if not help_triggered:
897            print("Build completed successfully in {}".format(
898                format_build_time(time() - start_time)))
899    except (SystemExit, KeyboardInterrupt) as error:
900        if hasattr(error, 'code') and isinstance(error.code, int):
901            exit_code = error.code
902        else:
903            exit_code = 1
904            print(error)
905        if not help_triggered:
906            print("Build completed unsuccessfully in {}".format(
907                format_build_time(time() - start_time)))
908        sys.exit(exit_code)
909
910
911if __name__ == '__main__':
912    main()