fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #66089 - Mark-Simulacrum:stable-next, r=Mark-Simulacrum4560ea788cb

1.39.0
33.2 KiB916 linesraw
1from __future__ import absolute_import, division, print_function
2import argparse
3import contextlib
4import datetime
5import hashlib
6import os
7import re
8import shutil
9import subprocess
10import sys
11import tarfile
12import tempfile
13
14from time import time
15
16
17def get(url, path, verbose=False):
18    suffix = '.sha256'
19    sha_url = url + suffix
20    with tempfile.NamedTemporaryFile(delete=False) as temp_file:
21        temp_path = temp_file.name
22    with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
23        sha_path = sha_file.name
24
25    try:
26        download(sha_path, sha_url, False, verbose)
27        if os.path.exists(path):
28            if verify(path, sha_path, False):
29                if verbose:
30                    print("using already-download file", path)
31                return
32            else:
33                if verbose:
34                    print("ignoring already-download file",
35                          path, "due to failed verification")
36                os.unlink(path)
37        download(temp_path, url, True, verbose)
38        if not verify(temp_path, sha_path, verbose):
39            raise RuntimeError("failed verification")
40        if verbose:
41            print("moving {} to {}".format(temp_path, path))
42        shutil.move(temp_path, path)
43    finally:
44        delete_if_present(sha_path, verbose)
45        delete_if_present(temp_path, verbose)
46
47
48def delete_if_present(path, verbose):
49    """Remove the given file if present"""
50    if os.path.isfile(path):
51        if verbose:
52            print("removing", path)
53        os.unlink(path)
54
55
56def download(path, url, probably_big, verbose):
57    for _ in range(0, 4):
58        try:
59            _download(path, url, probably_big, verbose, True)
60            return
61        except RuntimeError:
62            print("\nspurious failure, trying again")
63    _download(path, url, probably_big, verbose, False)
64
65
66def _download(path, url, probably_big, verbose, exception):
67    if probably_big or verbose:
68        print("downloading {}".format(url))
69    # see http://serverfault.com/questions/301128/how-to-download
70    if sys.platform == 'win32':
71        run(["PowerShell.exe", "/nologo", "-Command",
72             "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
73             "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
74            verbose=verbose,
75            exception=exception)
76    else:
77        if probably_big or verbose:
78            option = "-#"
79        else:
80            option = "-s"
81        run(["curl", option,
82             "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
83             "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
84             "--retry", "3", "-Sf", "-o", path, url],
85            verbose=verbose,
86            exception=exception)
87
88
89def verify(path, sha_path, verbose):
90    """Check if the sha256 sum of the given path is valid"""
91    if verbose:
92        print("verifying", path)
93    with open(path, "rb") as source:
94        found = hashlib.sha256(source.read()).hexdigest()
95    with open(sha_path, "r") as sha256sum:
96        expected = sha256sum.readline().split()[0]
97    verified = found == expected
98    if not verified:
99        print("invalid checksum:\n"
100              "    found:    {}\n"
101              "    expected: {}".format(found, expected))
102    return verified
103
104
105def unpack(tarball, dst, verbose=False, match=None):
106    """Unpack the given tarball file"""
107    print("extracting", tarball)
108    fname = os.path.basename(tarball).replace(".tar.gz", "")
109    with contextlib.closing(tarfile.open(tarball)) as tar:
110        for member in tar.getnames():
111            if "/" not in member:
112                continue
113            name = member.replace(fname + "/", "", 1)
114            if match is not None and not name.startswith(match):
115                continue
116            name = name[len(match) + 1:]
117
118            dst_path = os.path.join(dst, name)
119            if verbose:
120                print("  extracting", member)
121            tar.extract(member, dst)
122            src_path = os.path.join(dst, member)
123            if os.path.isdir(src_path) and os.path.exists(dst_path):
124                continue
125            shutil.move(src_path, dst_path)
126    shutil.rmtree(os.path.join(dst, fname))
127
128
129def run(args, verbose=False, exception=False, **kwargs):
130    """Run a child program in a new process"""
131    if verbose:
132        print("running: " + ' '.join(args))
133    sys.stdout.flush()
134    # Use Popen here instead of call() as it apparently allows powershell on
135    # Windows to not lock up waiting for input presumably.
136    ret = subprocess.Popen(args, **kwargs)
137    code = ret.wait()
138    if code != 0:
139        err = "failed to run: " + ' '.join(args)
140        if verbose or exception:
141            raise RuntimeError(err)
142        sys.exit(err)
143
144
145def stage0_data(rust_root):
146    """Build a dictionary from stage0.txt"""
147    nightlies = os.path.join(rust_root, "src/stage0.txt")
148    with open(nightlies, 'r') as nightlies:
149        lines = [line.rstrip() for line in nightlies
150                 if not line.startswith("#")]
151        return dict([line.split(": ", 1) for line in lines if line])
152
153
154def format_build_time(duration):
155    """Return a nicer format for build time
156
157    >>> format_build_time('300')
158    '0:05:00'
159    """
160    return str(datetime.timedelta(seconds=int(duration)))
161
162
163def default_build_triple():
164    """Build triple as in LLVM"""
165    default_encoding = sys.getdefaultencoding()
166    try:
167        ostype = subprocess.check_output(
168            ['uname', '-s']).strip().decode(default_encoding)
169        cputype = subprocess.check_output(
170            ['uname', '-m']).strip().decode(default_encoding)
171    except (subprocess.CalledProcessError, OSError):
172        if sys.platform == 'win32':
173            return 'x86_64-pc-windows-msvc'
174        err = "uname not found"
175        sys.exit(err)
176
177    # The goal here is to come up with the same triple as LLVM would,
178    # at least for the subset of platforms we're willing to target.
179    ostype_mapper = {
180        'Darwin': 'apple-darwin',
181        'DragonFly': 'unknown-dragonfly',
182        'FreeBSD': 'unknown-freebsd',
183        'Haiku': 'unknown-haiku',
184        'NetBSD': 'unknown-netbsd',
185        'OpenBSD': 'unknown-openbsd'
186    }
187
188    # Consider the direct transformation first and then the special cases
189    if ostype in ostype_mapper:
190        ostype = ostype_mapper[ostype]
191    elif ostype == 'Linux':
192        os_from_sp = subprocess.check_output(
193            ['uname', '-o']).strip().decode(default_encoding)
194        if os_from_sp == 'Android':
195            ostype = 'linux-android'
196        else:
197            ostype = 'unknown-linux-gnu'
198    elif ostype == 'SunOS':
199        ostype = 'sun-solaris'
200        # On Solaris, uname -m will return a machine classification instead
201        # of a cpu type, so uname -p is recommended instead.  However, the
202        # output from that option is too generic for our purposes (it will
203        # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
204        # must be used instead.
205        try:
206            cputype = subprocess.check_output(
207                ['isainfo', '-k']).strip().decode(default_encoding)
208        except (subprocess.CalledProcessError, OSError):
209            err = "isainfo not found"
210            sys.exit(err)
211    elif ostype.startswith('MINGW'):
212        # msys' `uname` does not print gcc configuration, but prints msys
213        # configuration. so we cannot believe `uname -m`:
214        # msys1 is always i686 and msys2 is always x86_64.
215        # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
216        # MINGW64 on x86_64.
217        ostype = 'pc-windows-gnu'
218        cputype = 'i686'
219        if os.environ.get('MSYSTEM') == 'MINGW64':
220            cputype = 'x86_64'
221    elif ostype.startswith('MSYS'):
222        ostype = 'pc-windows-gnu'
223    elif ostype.startswith('CYGWIN_NT'):
224        cputype = 'i686'
225        if ostype.endswith('WOW64'):
226            cputype = 'x86_64'
227        ostype = 'pc-windows-gnu'
228    else:
229        err = "unknown OS type: {}".format(ostype)
230        sys.exit(err)
231
232    if cputype == 'powerpc' and ostype == 'unknown-freebsd':
233        cputype = subprocess.check_output(
234              ['uname', '-p']).strip().decode(default_encoding)
235    cputype_mapper = {
236        'BePC': 'i686',
237        'aarch64': 'aarch64',
238        'amd64': 'x86_64',
239        'arm64': 'aarch64',
240        'i386': 'i686',
241        'i486': 'i686',
242        'i686': 'i686',
243        'i786': 'i686',
244        'powerpc': 'powerpc',
245        'powerpc64': 'powerpc64',
246        'powerpc64le': 'powerpc64le',
247        'ppc': 'powerpc',
248        'ppc64': 'powerpc64',
249        'ppc64le': 'powerpc64le',
250        's390x': 's390x',
251        'x64': 'x86_64',
252        'x86': 'i686',
253        'x86-64': 'x86_64',
254        'x86_64': 'x86_64'
255    }
256
257    # Consider the direct transformation first and then the special cases
258    if cputype in cputype_mapper:
259        cputype = cputype_mapper[cputype]
260    elif cputype in {'xscale', 'arm'}:
261        cputype = 'arm'
262        if ostype == 'linux-android':
263            ostype = 'linux-androideabi'
264        elif ostype == 'unknown-freebsd':
265            cputype = subprocess.check_output(
266                ['uname', '-p']).strip().decode(default_encoding)
267            ostype = 'unknown-freebsd'
268    elif cputype == 'armv6l':
269        cputype = 'arm'
270        if ostype == 'linux-android':
271            ostype = 'linux-androideabi'
272        else:
273            ostype += 'eabihf'
274    elif cputype in {'armv7l', 'armv8l'}:
275        cputype = 'armv7'
276        if ostype == 'linux-android':
277            ostype = 'linux-androideabi'
278        else:
279            ostype += 'eabihf'
280    elif cputype == 'mips':
281        if sys.byteorder == 'big':
282            cputype = 'mips'
283        elif sys.byteorder == 'little':
284            cputype = 'mipsel'
285        else:
286            raise ValueError("unknown byteorder: {}".format(sys.byteorder))
287    elif cputype == 'mips64':
288        if sys.byteorder == 'big':
289            cputype = 'mips64'
290        elif sys.byteorder == 'little':
291            cputype = 'mips64el'
292        else:
293            raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
294        # only the n64 ABI is supported, indicate it
295        ostype += 'abi64'
296    elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
297        pass
298    else:
299        err = "unknown cpu type: {}".format(cputype)
300        sys.exit(err)
301
302    return "{}-{}".format(cputype, ostype)
303
304
305@contextlib.contextmanager
306def output(filepath):
307    tmp = filepath + '.tmp'
308    with open(tmp, 'w') as f:
309        yield f
310    try:
311        os.remove(filepath)  # PermissionError/OSError on Win32 if in use
312        os.rename(tmp, filepath)
313    except OSError:
314        shutil.copy2(tmp, filepath)
315        os.remove(tmp)
316
317
318class RustBuild(object):
319    """Provide all the methods required to build Rust"""
320    def __init__(self):
321        self.cargo_channel = ''
322        self.date = ''
323        self._download_url = ''
324        self.rustc_channel = ''
325        self.build = ''
326        self.build_dir = os.path.join(os.getcwd(), "build")
327        self.clean = False
328        self.config_toml = ''
329        self.rust_root = ''
330        self.use_locked_deps = ''
331        self.use_vendored_sources = ''
332        self.verbose = False
333
334    def download_stage0(self):
335        """Fetch the build system for Rust, written in Rust
336
337        This method will build a cache directory, then it will fetch the
338        tarball which has the stage0 compiler used to then bootstrap the Rust
339        compiler itself.
340
341        Each downloaded tarball is extracted, after that, the script
342        will move all the content to the right place.
343        """
344        rustc_channel = self.rustc_channel
345        cargo_channel = self.cargo_channel
346
347        if self.rustc().startswith(self.bin_root()) and \
348                (not os.path.exists(self.rustc()) or
349                 self.program_out_of_date(self.rustc_stamp())):
350            if os.path.exists(self.bin_root()):
351                shutil.rmtree(self.bin_root())
352            filename = "rust-std-{}-{}.tar.gz".format(
353                rustc_channel, self.build)
354            pattern = "rust-std-{}".format(self.build)
355            self._download_stage0_helper(filename, pattern)
356
357            filename = "rustc-{}-{}.tar.gz".format(rustc_channel, self.build)
358            self._download_stage0_helper(filename, "rustc")
359            self.fix_executable("{}/bin/rustc".format(self.bin_root()))
360            self.fix_executable("{}/bin/rustdoc".format(self.bin_root()))
361            with output(self.rustc_stamp()) as rust_stamp:
362                rust_stamp.write(self.date)
363
364            # This is required so that we don't mix incompatible MinGW
365            # libraries/binaries that are included in rust-std with
366            # the system MinGW ones.
367            if "pc-windows-gnu" in self.build:
368                filename = "rust-mingw-{}-{}.tar.gz".format(
369                    rustc_channel, self.build)
370                self._download_stage0_helper(filename, "rust-mingw")
371
372        if self.cargo().startswith(self.bin_root()) and \
373                (not os.path.exists(self.cargo()) or
374                 self.program_out_of_date(self.cargo_stamp())):
375            filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build)
376            self._download_stage0_helper(filename, "cargo")
377            self.fix_executable("{}/bin/cargo".format(self.bin_root()))
378            with output(self.cargo_stamp()) as cargo_stamp:
379                cargo_stamp.write(self.date)
380
381    def _download_stage0_helper(self, filename, pattern):
382        cache_dst = os.path.join(self.build_dir, "cache")
383        rustc_cache = os.path.join(cache_dst, self.date)
384        if not os.path.exists(rustc_cache):
385            os.makedirs(rustc_cache)
386
387        url = "{}/dist/{}".format(self._download_url, self.date)
388        tarball = os.path.join(rustc_cache, filename)
389        if not os.path.exists(tarball):
390            get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
391        unpack(tarball, self.bin_root(), match=pattern, verbose=self.verbose)
392
393    @staticmethod
394    def fix_executable(fname):
395        """Modifies the interpreter section of 'fname' to fix the dynamic linker
396
397        This method is only required on NixOS and uses the PatchELF utility to
398        change the dynamic linker of ELF executables.
399
400        Please see https://nixos.org/patchelf.html for more information
401        """
402        default_encoding = sys.getdefaultencoding()
403        try:
404            ostype = subprocess.check_output(
405                ['uname', '-s']).strip().decode(default_encoding)
406        except subprocess.CalledProcessError:
407            return
408        except OSError as reason:
409            if getattr(reason, 'winerror', None) is not None:
410                return
411            raise reason
412
413        if ostype != "Linux":
414            return
415
416        if not os.path.exists("/etc/NIXOS"):
417            return
418        if os.path.exists("/lib"):
419            return
420
421        # At this point we're pretty sure the user is running NixOS
422        nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
423        print(nix_os_msg, fname)
424
425        try:
426            interpreter = subprocess.check_output(
427                ["patchelf", "--print-interpreter", fname])
428            interpreter = interpreter.strip().decode(default_encoding)
429        except subprocess.CalledProcessError as reason:
430            print("warning: failed to call patchelf:", reason)
431            return
432
433        loader = interpreter.split("/")[-1]
434
435        try:
436            ldd_output = subprocess.check_output(
437                ['ldd', '/run/current-system/sw/bin/sh'])
438            ldd_output = ldd_output.strip().decode(default_encoding)
439        except subprocess.CalledProcessError as reason:
440            print("warning: unable to call ldd:", reason)
441            return
442
443        for line in ldd_output.splitlines():
444            libname = line.split()[0]
445            if libname.endswith(loader):
446                loader_path = libname[:len(libname) - len(loader)]
447                break
448        else:
449            print("warning: unable to find the path to the dynamic linker")
450            return
451
452        correct_interpreter = loader_path + loader
453
454        try:
455            subprocess.check_output(
456                ["patchelf", "--set-interpreter", correct_interpreter, fname])
457        except subprocess.CalledProcessError as reason:
458            print("warning: failed to call patchelf:", reason)
459            return
460
461    def rustc_stamp(self):
462        """Return the path for .rustc-stamp
463
464        >>> rb = RustBuild()
465        >>> rb.build_dir = "build"
466        >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
467        True
468        """
469        return os.path.join(self.bin_root(), '.rustc-stamp')
470
471    def cargo_stamp(self):
472        """Return the path for .cargo-stamp
473
474        >>> rb = RustBuild()
475        >>> rb.build_dir = "build"
476        >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
477        True
478        """
479        return os.path.join(self.bin_root(), '.cargo-stamp')
480
481    def program_out_of_date(self, stamp_path):
482        """Check if the given program stamp is out of date"""
483        if not os.path.exists(stamp_path) or self.clean:
484            return True
485        with open(stamp_path, 'r') as stamp:
486            return self.date != stamp.read()
487
488    def bin_root(self):
489        """Return the binary root directory
490
491        >>> rb = RustBuild()
492        >>> rb.build_dir = "build"
493        >>> rb.bin_root() == os.path.join("build", "stage0")
494        True
495
496        When the 'build' property is given should be a nested directory:
497
498        >>> rb.build = "devel"
499        >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
500        True
501        """
502        return os.path.join(self.build_dir, self.build, "stage0")
503
504    def get_toml(self, key, section=None):
505        """Returns the value of the given key in config.toml, otherwise returns None
506
507        >>> rb = RustBuild()
508        >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
509        >>> rb.get_toml("key2")
510        'value2'
511
512        If the key does not exists, the result is None:
513
514        >>> rb.get_toml("key3") is None
515        True
516
517        Optionally also matches the section the key appears in
518
519        >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
520        >>> rb.get_toml('key', 'a')
521        'value1'
522        >>> rb.get_toml('key', 'b')
523        'value2'
524        >>> rb.get_toml('key', 'c') is None
525        True
526
527        >>> rb.config_toml = 'key1 = true'
528        >>> rb.get_toml("key1")
529        'true'
530        """
531
532        cur_section = None
533        for line in self.config_toml.splitlines():
534            section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
535            if section_match is not None:
536                cur_section = section_match.group(1)
537
538            match = re.match(r'^{}\s*=(.*)$'.format(key), line)
539            if match is not None:
540                value = match.group(1)
541                if section is None or section == cur_section:
542                    return self.get_string(value) or value.strip()
543        return None
544
545    def cargo(self):
546        """Return config path for cargo"""
547        return self.program_config('cargo')
548
549    def rustc(self):
550        """Return config path for rustc"""
551        return self.program_config('rustc')
552
553    def program_config(self, program):
554        """Return config path for the given program
555
556        >>> rb = RustBuild()
557        >>> rb.config_toml = 'rustc = "rustc"\\n'
558        >>> rb.program_config('rustc')
559        'rustc'
560        >>> rb.config_toml = ''
561        >>> cargo_path = rb.program_config('cargo')
562        >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
563        ... "bin", "cargo")
564        True
565        """
566        config = self.get_toml(program)
567        if config:
568            return os.path.expanduser(config)
569        return os.path.join(self.bin_root(), "bin", "{}{}".format(
570            program, self.exe_suffix()))
571
572    @staticmethod
573    def get_string(line):
574        """Return the value between double quotes
575
576        >>> RustBuild.get_string('    "devel"   ')
577        'devel'
578        >>> RustBuild.get_string("    'devel'   ")
579        'devel'
580        >>> RustBuild.get_string('devel') is None
581        True
582        >>> RustBuild.get_string('    "devel   ')
583        ''
584        """
585        start = line.find('"')
586        if start != -1:
587            end = start + 1 + line[start + 1:].find('"')
588            return line[start + 1:end]
589        start = line.find('\'')
590        if start != -1:
591            end = start + 1 + line[start + 1:].find('\'')
592            return line[start + 1:end]
593        return None
594
595    @staticmethod
596    def exe_suffix():
597        """Return a suffix for executables"""
598        if sys.platform == 'win32':
599            return '.exe'
600        return ''
601
602    def bootstrap_binary(self):
603        """Return the path of the bootstrap binary
604
605        >>> rb = RustBuild()
606        >>> rb.build_dir = "build"
607        >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
608        ... "debug", "bootstrap")
609        True
610        """
611        return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
612
613    def build_bootstrap(self):
614        """Build bootstrap"""
615        build_dir = os.path.join(self.build_dir, "bootstrap")
616        if self.clean and os.path.exists(build_dir):
617            shutil.rmtree(build_dir)
618        env = os.environ.copy()
619        env["RUSTC_BOOTSTRAP"] = '1'
620        env["CARGO_TARGET_DIR"] = build_dir
621        env["RUSTC"] = self.rustc()
622        env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
623            (os.pathsep + env["LD_LIBRARY_PATH"]) \
624            if "LD_LIBRARY_PATH" in env else ""
625        env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
626            (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
627            if "DYLD_LIBRARY_PATH" in env else ""
628        env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
629            (os.pathsep + env["LIBRARY_PATH"]) \
630            if "LIBRARY_PATH" in env else ""
631        env["RUSTFLAGS"] = "-Cdebuginfo=2 "
632
633        build_section = "target.{}".format(self.build_triple())
634        target_features = []
635        if self.get_toml("crt-static", build_section) == "true":
636            target_features += ["+crt-static"]
637        elif self.get_toml("crt-static", build_section) == "false":
638            target_features += ["-crt-static"]
639        if target_features:
640            env["RUSTFLAGS"] += "-C target-feature=" + (",".join(target_features)) + " "
641        target_linker = self.get_toml("linker", build_section)
642        if target_linker is not None:
643            env["RUSTFLAGS"] += "-C linker=" + target_linker + " "
644        env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes "
645        if self.get_toml("deny-warnings", "rust") != "false":
646            env["RUSTFLAGS"] += "-Dwarnings "
647
648        env["PATH"] = os.path.join(self.bin_root(), "bin") + \
649            os.pathsep + env["PATH"]
650        if not os.path.isfile(self.cargo()):
651            raise Exception("no cargo executable found at `{}`".format(
652                self.cargo()))
653        args = [self.cargo(), "build", "--manifest-path",
654                os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
655        for _ in range(1, self.verbose):
656            args.append("--verbose")
657        if self.use_locked_deps:
658            args.append("--locked")
659        if self.use_vendored_sources:
660            args.append("--frozen")
661        run(args, env=env, verbose=self.verbose)
662
663    def build_triple(self):
664        """Build triple as in LLVM"""
665        config = self.get_toml('build')
666        if config:
667            return config
668        return default_build_triple()
669
670    def check_submodule(self, module, slow_submodules):
671        if not slow_submodules:
672            checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
673                                           cwd=os.path.join(self.rust_root, module),
674                                           stdout=subprocess.PIPE)
675            return checked_out
676        else:
677            return None
678
679    def update_submodule(self, module, checked_out, recorded_submodules):
680        module_path = os.path.join(self.rust_root, module)
681
682        if checked_out is not None:
683            default_encoding = sys.getdefaultencoding()
684            checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
685            if recorded_submodules[module] == checked_out:
686                return
687
688        print("Updating submodule", module)
689
690        run(["git", "submodule", "-q", "sync", module],
691            cwd=self.rust_root, verbose=self.verbose)
692        try:
693            run(["git", "submodule", "update",
694                 "--init", "--recursive", "--progress", module],
695                cwd=self.rust_root, verbose=self.verbose, exception=True)
696        except RuntimeError:
697            # Some versions of git don't support --progress.
698            run(["git", "submodule", "update",
699                 "--init", "--recursive", module],
700                cwd=self.rust_root, verbose=self.verbose)
701        run(["git", "reset", "-q", "--hard"],
702            cwd=module_path, verbose=self.verbose)
703        run(["git", "clean", "-qdfx"],
704            cwd=module_path, verbose=self.verbose)
705
706    def update_submodules(self):
707        """Update submodules"""
708        if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
709                self.get_toml('submodules') == "false":
710            return
711
712        # check the existence of 'git' command
713        try:
714            subprocess.check_output(['git', '--version'])
715        except (subprocess.CalledProcessError, OSError):
716            print("error: `git` is not found, please make sure it's installed and in the path.")
717            sys.exit(1)
718
719        slow_submodules = self.get_toml('fast-submodules') == "false"
720        start_time = time()
721        if slow_submodules:
722            print('Unconditionally updating all submodules')
723        else:
724            print('Updating only changed submodules')
725        default_encoding = sys.getdefaultencoding()
726        submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
727            ["git", "config", "--file",
728             os.path.join(self.rust_root, ".gitmodules"),
729             "--get-regexp", "path"]
730        ).decode(default_encoding).splitlines()]
731        filtered_submodules = []
732        submodules_names = []
733        for module in submodules:
734            if module.endswith("llvm-project"):
735                if self.get_toml('llvm-config') and self.get_toml('lld') != 'true':
736                    continue
737            if module.endswith("llvm-emscripten"):
738                backends = self.get_toml('codegen-backends')
739                if backends is None or not 'emscripten' in backends:
740                    continue
741            check = self.check_submodule(module, slow_submodules)
742            filtered_submodules.append((module, check))
743            submodules_names.append(module)
744        recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
745                                    cwd=self.rust_root, stdout=subprocess.PIPE)
746        recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
747        recorded_submodules = {}
748        for data in recorded:
749            data = data.split()
750            recorded_submodules[data[3]] = data[2]
751        for module in filtered_submodules:
752            self.update_submodule(module[0], module[1], recorded_submodules)
753        print("Submodules updated in %.2f seconds" % (time() - start_time))
754
755    def set_normal_environment(self):
756        """Set download URL for normal environment"""
757        if 'RUSTUP_DIST_SERVER' in os.environ:
758            self._download_url = os.environ['RUSTUP_DIST_SERVER']
759        else:
760            self._download_url = 'https://static.rust-lang.org'
761
762    def set_dev_environment(self):
763        """Set download URL for development environment"""
764        if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
765            self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
766        else:
767            self._download_url = 'https://dev-static.rust-lang.org'
768
769    def check_vendored_status(self):
770        """Check that vendoring is configured properly"""
771        vendor_dir = os.path.join(self.rust_root, 'vendor')
772        if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
773            if os.environ.get('USER') != os.environ['SUDO_USER']:
774                self.use_vendored_sources = True
775                print('info: looks like you are running this command under `sudo`')
776                print('      and so in order to preserve your $HOME this will now')
777                print('      use vendored sources by default.')
778                if not os.path.exists(vendor_dir):
779                    print('error: vendoring required, but vendor directory does not exist.')
780                    print('       Run `cargo vendor` without sudo to initialize the '
781                        'vendor directory.')
782                    raise Exception("{} not found".format(vendor_dir))
783
784        if self.use_vendored_sources:
785            if not os.path.exists('.cargo'):
786                os.makedirs('.cargo')
787            with output('.cargo/config') as cargo_config:
788                cargo_config.write(
789                    "[source.crates-io]\n"
790                    "replace-with = 'vendored-sources'\n"
791                    "registry = 'https://example.com'\n"
792                    "\n"
793                    "[source.vendored-sources]\n"
794                    "directory = '{}/vendor'\n"
795                .format(self.rust_root))
796        else:
797            if os.path.exists('.cargo'):
798                shutil.rmtree('.cargo')
799
800    def ensure_vendored(self):
801        """Ensure that the vendored sources are available if needed"""
802        vendor_dir = os.path.join(self.rust_root, 'vendor')
803        # Note that this does not handle updating the vendored dependencies if
804        # the rust git repository is updated. Normal development usually does
805        # not use vendoring, so hopefully this isn't too much of a problem.
806        if self.use_vendored_sources and not os.path.exists(vendor_dir):
807            run([self.cargo(), "vendor"],
808                verbose=self.verbose, cwd=self.rust_root)
809
810
811def bootstrap(help_triggered):
812    """Configure, fetch, build and run the initial bootstrap"""
813
814    # If the user is asking for help, let them know that the whole download-and-build
815    # process has to happen before anything is printed out.
816    if help_triggered:
817        print("info: Downloading and building bootstrap before processing --help")
818        print("      command. See src/bootstrap/README.md for help with common")
819        print("      commands.")
820
821    parser = argparse.ArgumentParser(description='Build rust')
822    parser.add_argument('--config')
823    parser.add_argument('--build')
824    parser.add_argument('--src')
825    parser.add_argument('--clean', action='store_true')
826    parser.add_argument('-v', '--verbose', action='count', default=0)
827
828    args = [a for a in sys.argv if a != '-h' and a != '--help']
829    args, _ = parser.parse_known_args(args)
830
831    # Configure initial bootstrap
832    build = RustBuild()
833    build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..'))
834    build.verbose = args.verbose
835    build.clean = args.clean
836
837    try:
838        with open(args.config or 'config.toml') as config:
839            build.config_toml = config.read()
840    except (OSError, IOError):
841        pass
842
843    config_verbose = build.get_toml('verbose', 'build')
844    if config_verbose is not None:
845        build.verbose = max(build.verbose, int(config_verbose))
846
847    build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
848
849    build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
850
851    build.check_vendored_status()
852
853    data = stage0_data(build.rust_root)
854    build.date = data['date']
855    build.rustc_channel = data['rustc']
856    build.cargo_channel = data['cargo']
857
858    if 'dev' in data:
859        build.set_dev_environment()
860    else:
861        build.set_normal_environment()
862
863    build.update_submodules()
864
865    # Fetch/build the bootstrap
866    build.build = args.build or build.build_triple()
867    build.download_stage0()
868    sys.stdout.flush()
869    build.ensure_vendored()
870    build.build_bootstrap()
871    sys.stdout.flush()
872
873    # Run the bootstrap
874    args = [build.bootstrap_binary()]
875    args.extend(sys.argv[1:])
876    env = os.environ.copy()
877    env["BUILD"] = build.build
878    env["SRC"] = build.rust_root
879    env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
880    env["BOOTSTRAP_PYTHON"] = sys.executable
881    env["BUILD_DIR"] = build.build_dir
882    env["RUSTC_BOOTSTRAP"] = '1'
883    env["CARGO"] = build.cargo()
884    env["RUSTC"] = build.rustc()
885    run(args, env=env, verbose=build.verbose)
886
887
888def main():
889    """Entry point for the bootstrap process"""
890    start_time = time()
891
892    # x.py help <cmd> ...
893    if len(sys.argv) > 1 and sys.argv[1] == 'help':
894        sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
895
896    help_triggered = (
897        '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
898    try:
899        bootstrap(help_triggered)
900        if not help_triggered:
901            print("Build completed successfully in {}".format(
902                format_build_time(time() - start_time)))
903    except (SystemExit, KeyboardInterrupt) as error:
904        if hasattr(error, 'code') and isinstance(error.code, int):
905            exit_code = error.code
906        else:
907            exit_code = 1
908            print(error)
909        if not help_triggered:
910            print("Build completed unsuccessfully in {}".format(
911                format_build_time(time() - start_time)))
912        sys.exit(exit_code)
913
914
915if __name__ == '__main__':
916    main()