fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #69444 - pietroalbini:stable-next-final-v2-new, r=pietroalbinif3e1a954d2e

1.41.1
33.8 KiB929 linesraw
1from __future__ import absolute_import, division, print_function
2import argparse
3import contextlib
4import datetime
5import hashlib
6import os
7import re
8import shutil
9import subprocess
10import sys
11import tarfile
12import tempfile
13
14from time import time
15
16
17def get(url, path, verbose=False):
18    suffix = '.sha256'
19    sha_url = url + suffix
20    with tempfile.NamedTemporaryFile(delete=False) as temp_file:
21        temp_path = temp_file.name
22    with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as sha_file:
23        sha_path = sha_file.name
24
25    try:
26        download(sha_path, sha_url, False, verbose)
27        if os.path.exists(path):
28            if verify(path, sha_path, False):
29                if verbose:
30                    print("using already-download file", path)
31                return
32            else:
33                if verbose:
34                    print("ignoring already-download file",
35                          path, "due to failed verification")
36                os.unlink(path)
37        download(temp_path, url, True, verbose)
38        if not verify(temp_path, sha_path, verbose):
39            raise RuntimeError("failed verification")
40        if verbose:
41            print("moving {} to {}".format(temp_path, path))
42        shutil.move(temp_path, path)
43    finally:
44        delete_if_present(sha_path, verbose)
45        delete_if_present(temp_path, verbose)
46
47
48def delete_if_present(path, verbose):
49    """Remove the given file if present"""
50    if os.path.isfile(path):
51        if verbose:
52            print("removing", path)
53        os.unlink(path)
54
55
56def download(path, url, probably_big, verbose):
57    for _ in range(0, 4):
58        try:
59            _download(path, url, probably_big, verbose, True)
60            return
61        except RuntimeError:
62            print("\nspurious failure, trying again")
63    _download(path, url, probably_big, verbose, False)
64
65
66def _download(path, url, probably_big, verbose, exception):
67    if probably_big or verbose:
68        print("downloading {}".format(url))
69    # see http://serverfault.com/questions/301128/how-to-download
70    if sys.platform == 'win32':
71        run(["PowerShell.exe", "/nologo", "-Command",
72             "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
73             "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)],
74            verbose=verbose,
75            exception=exception)
76    else:
77        if probably_big or verbose:
78            option = "-#"
79        else:
80            option = "-s"
81        run(["curl", option,
82             "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
83             "--connect-timeout", "30", # timeout if cannot connect within 30 seconds
84             "--retry", "3", "-Sf", "-o", path, url],
85            verbose=verbose,
86            exception=exception)
87
88
89def verify(path, sha_path, verbose):
90    """Check if the sha256 sum of the given path is valid"""
91    if verbose:
92        print("verifying", path)
93    with open(path, "rb") as source:
94        found = hashlib.sha256(source.read()).hexdigest()
95    with open(sha_path, "r") as sha256sum:
96        expected = sha256sum.readline().split()[0]
97    verified = found == expected
98    if not verified:
99        print("invalid checksum:\n"
100              "    found:    {}\n"
101              "    expected: {}".format(found, expected))
102    return verified
103
104
105def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
106    """Unpack the given tarball file"""
107    print("extracting", tarball)
108    fname = os.path.basename(tarball).replace(tarball_suffix, "")
109    with contextlib.closing(tarfile.open(tarball)) as tar:
110        for member in tar.getnames():
111            if "/" not in member:
112                continue
113            name = member.replace(fname + "/", "", 1)
114            if match is not None and not name.startswith(match):
115                continue
116            name = name[len(match) + 1:]
117
118            dst_path = os.path.join(dst, name)
119            if verbose:
120                print("  extracting", member)
121            tar.extract(member, dst)
122            src_path = os.path.join(dst, member)
123            if os.path.isdir(src_path) and os.path.exists(dst_path):
124                continue
125            shutil.move(src_path, dst_path)
126    shutil.rmtree(os.path.join(dst, fname))
127
128
129def run(args, verbose=False, exception=False, **kwargs):
130    """Run a child program in a new process"""
131    if verbose:
132        print("running: " + ' '.join(args))
133    sys.stdout.flush()
134    # Use Popen here instead of call() as it apparently allows powershell on
135    # Windows to not lock up waiting for input presumably.
136    ret = subprocess.Popen(args, **kwargs)
137    code = ret.wait()
138    if code != 0:
139        err = "failed to run: " + ' '.join(args)
140        if verbose or exception:
141            raise RuntimeError(err)
142        sys.exit(err)
143
144
145def stage0_data(rust_root):
146    """Build a dictionary from stage0.txt"""
147    nightlies = os.path.join(rust_root, "src/stage0.txt")
148    with open(nightlies, 'r') as nightlies:
149        lines = [line.rstrip() for line in nightlies
150                 if not line.startswith("#")]
151        return dict([line.split(": ", 1) for line in lines if line])
152
153
154def format_build_time(duration):
155    """Return a nicer format for build time
156
157    >>> format_build_time('300')
158    '0:05:00'
159    """
160    return str(datetime.timedelta(seconds=int(duration)))
161
162
163def default_build_triple():
164    """Build triple as in LLVM"""
165    default_encoding = sys.getdefaultencoding()
166    try:
167        ostype = subprocess.check_output(
168            ['uname', '-s']).strip().decode(default_encoding)
169        cputype = subprocess.check_output(
170            ['uname', '-m']).strip().decode(default_encoding)
171    except (subprocess.CalledProcessError, OSError):
172        if sys.platform == 'win32':
173            return 'x86_64-pc-windows-msvc'
174        err = "uname not found"
175        sys.exit(err)
176
177    # The goal here is to come up with the same triple as LLVM would,
178    # at least for the subset of platforms we're willing to target.
179    ostype_mapper = {
180        'Darwin': 'apple-darwin',
181        'DragonFly': 'unknown-dragonfly',
182        'FreeBSD': 'unknown-freebsd',
183        'Haiku': 'unknown-haiku',
184        'NetBSD': 'unknown-netbsd',
185        'OpenBSD': 'unknown-openbsd'
186    }
187
188    # Consider the direct transformation first and then the special cases
189    if ostype in ostype_mapper:
190        ostype = ostype_mapper[ostype]
191    elif ostype == 'Linux':
192        os_from_sp = subprocess.check_output(
193            ['uname', '-o']).strip().decode(default_encoding)
194        if os_from_sp == 'Android':
195            ostype = 'linux-android'
196        else:
197            ostype = 'unknown-linux-gnu'
198    elif ostype == 'SunOS':
199        ostype = 'sun-solaris'
200        # On Solaris, uname -m will return a machine classification instead
201        # of a cpu type, so uname -p is recommended instead.  However, the
202        # output from that option is too generic for our purposes (it will
203        # always emit 'i386' on x86/amd64 systems).  As such, isainfo -k
204        # must be used instead.
205        try:
206            cputype = subprocess.check_output(
207                ['isainfo', '-k']).strip().decode(default_encoding)
208        except (subprocess.CalledProcessError, OSError):
209            err = "isainfo not found"
210            sys.exit(err)
211    elif ostype.startswith('MINGW'):
212        # msys' `uname` does not print gcc configuration, but prints msys
213        # configuration. so we cannot believe `uname -m`:
214        # msys1 is always i686 and msys2 is always x86_64.
215        # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
216        # MINGW64 on x86_64.
217        ostype = 'pc-windows-gnu'
218        cputype = 'i686'
219        if os.environ.get('MSYSTEM') == 'MINGW64':
220            cputype = 'x86_64'
221    elif ostype.startswith('MSYS'):
222        ostype = 'pc-windows-gnu'
223    elif ostype.startswith('CYGWIN_NT'):
224        cputype = 'i686'
225        if ostype.endswith('WOW64'):
226            cputype = 'x86_64'
227        ostype = 'pc-windows-gnu'
228    else:
229        err = "unknown OS type: {}".format(ostype)
230        sys.exit(err)
231
232    if cputype == 'powerpc' and ostype == 'unknown-freebsd':
233        cputype = subprocess.check_output(
234              ['uname', '-p']).strip().decode(default_encoding)
235    cputype_mapper = {
236        'BePC': 'i686',
237        'aarch64': 'aarch64',
238        'amd64': 'x86_64',
239        'arm64': 'aarch64',
240        'i386': 'i686',
241        'i486': 'i686',
242        'i686': 'i686',
243        'i786': 'i686',
244        'powerpc': 'powerpc',
245        'powerpc64': 'powerpc64',
246        'powerpc64le': 'powerpc64le',
247        'ppc': 'powerpc',
248        'ppc64': 'powerpc64',
249        'ppc64le': 'powerpc64le',
250        's390x': 's390x',
251        'x64': 'x86_64',
252        'x86': 'i686',
253        'x86-64': 'x86_64',
254        'x86_64': 'x86_64'
255    }
256
257    # Consider the direct transformation first and then the special cases
258    if cputype in cputype_mapper:
259        cputype = cputype_mapper[cputype]
260    elif cputype in {'xscale', 'arm'}:
261        cputype = 'arm'
262        if ostype == 'linux-android':
263            ostype = 'linux-androideabi'
264        elif ostype == 'unknown-freebsd':
265            cputype = subprocess.check_output(
266                ['uname', '-p']).strip().decode(default_encoding)
267            ostype = 'unknown-freebsd'
268    elif cputype == 'armv6l':
269        cputype = 'arm'
270        if ostype == 'linux-android':
271            ostype = 'linux-androideabi'
272        else:
273            ostype += 'eabihf'
274    elif cputype in {'armv7l', 'armv8l'}:
275        cputype = 'armv7'
276        if ostype == 'linux-android':
277            ostype = 'linux-androideabi'
278        else:
279            ostype += 'eabihf'
280    elif cputype == 'mips':
281        if sys.byteorder == 'big':
282            cputype = 'mips'
283        elif sys.byteorder == 'little':
284            cputype = 'mipsel'
285        else:
286            raise ValueError("unknown byteorder: {}".format(sys.byteorder))
287    elif cputype == 'mips64':
288        if sys.byteorder == 'big':
289            cputype = 'mips64'
290        elif sys.byteorder == 'little':
291            cputype = 'mips64el'
292        else:
293            raise ValueError('unknown byteorder: {}'.format(sys.byteorder))
294        # only the n64 ABI is supported, indicate it
295        ostype += 'abi64'
296    elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64':
297        pass
298    else:
299        err = "unknown cpu type: {}".format(cputype)
300        sys.exit(err)
301
302    return "{}-{}".format(cputype, ostype)
303
304
305@contextlib.contextmanager
306def output(filepath):
307    tmp = filepath + '.tmp'
308    with open(tmp, 'w') as f:
309        yield f
310    try:
311        os.remove(filepath)  # PermissionError/OSError on Win32 if in use
312        os.rename(tmp, filepath)
313    except OSError:
314        shutil.copy2(tmp, filepath)
315        os.remove(tmp)
316
317
318class RustBuild(object):
319    """Provide all the methods required to build Rust"""
320    def __init__(self):
321        self.cargo_channel = ''
322        self.date = ''
323        self._download_url = ''
324        self.rustc_channel = ''
325        self.build = ''
326        self.build_dir = os.path.join(os.getcwd(), "build")
327        self.clean = False
328        self.config_toml = ''
329        self.rust_root = ''
330        self.use_locked_deps = ''
331        self.use_vendored_sources = ''
332        self.verbose = False
333
334
335    def download_stage0(self):
336        """Fetch the build system for Rust, written in Rust
337
338        This method will build a cache directory, then it will fetch the
339        tarball which has the stage0 compiler used to then bootstrap the Rust
340        compiler itself.
341
342        Each downloaded tarball is extracted, after that, the script
343        will move all the content to the right place.
344        """
345        rustc_channel = self.rustc_channel
346        cargo_channel = self.cargo_channel
347
348        def support_xz():
349            try:
350                with tempfile.NamedTemporaryFile(delete=False) as temp_file:
351                    temp_path = temp_file.name
352                with tarfile.open(temp_path, "w:xz") as tar:
353                    pass
354                return True
355            except tarfile.CompressionError:
356                return False
357
358        if self.rustc().startswith(self.bin_root()) and \
359                (not os.path.exists(self.rustc()) or
360                 self.program_out_of_date(self.rustc_stamp())):
361            if os.path.exists(self.bin_root()):
362                shutil.rmtree(self.bin_root())
363            tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
364            filename = "rust-std-{}-{}{}".format(
365                rustc_channel, self.build, tarball_suffix)
366            pattern = "rust-std-{}".format(self.build)
367            self._download_stage0_helper(filename, pattern, tarball_suffix)
368
369            filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
370                                              tarball_suffix)
371            self._download_stage0_helper(filename, "rustc", tarball_suffix)
372            self.fix_executable("{}/bin/rustc".format(self.bin_root()))
373            self.fix_executable("{}/bin/rustdoc".format(self.bin_root()))
374            with output(self.rustc_stamp()) as rust_stamp:
375                rust_stamp.write(self.date)
376
377            # This is required so that we don't mix incompatible MinGW
378            # libraries/binaries that are included in rust-std with
379            # the system MinGW ones.
380            if "pc-windows-gnu" in self.build:
381                filename = "rust-mingw-{}-{}{}".format(
382                    rustc_channel, self.build, tarball_suffix)
383                self._download_stage0_helper(filename, "rust-mingw", tarball_suffix)
384
385        if self.cargo().startswith(self.bin_root()) and \
386                (not os.path.exists(self.cargo()) or
387                 self.program_out_of_date(self.cargo_stamp())):
388            tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
389            filename = "cargo-{}-{}{}".format(cargo_channel, self.build,
390                                              tarball_suffix)
391            self._download_stage0_helper(filename, "cargo", tarball_suffix)
392            self.fix_executable("{}/bin/cargo".format(self.bin_root()))
393            with output(self.cargo_stamp()) as cargo_stamp:
394                cargo_stamp.write(self.date)
395
396    def _download_stage0_helper(self, filename, pattern, tarball_suffix):
397        cache_dst = os.path.join(self.build_dir, "cache")
398        rustc_cache = os.path.join(cache_dst, self.date)
399        if not os.path.exists(rustc_cache):
400            os.makedirs(rustc_cache)
401
402        url = "{}/dist/{}".format(self._download_url, self.date)
403        tarball = os.path.join(rustc_cache, filename)
404        if not os.path.exists(tarball):
405            get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
406        unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
407
408    @staticmethod
409    def fix_executable(fname):
410        """Modifies the interpreter section of 'fname' to fix the dynamic linker
411
412        This method is only required on NixOS and uses the PatchELF utility to
413        change the dynamic linker of ELF executables.
414
415        Please see https://nixos.org/patchelf.html for more information
416        """
417        default_encoding = sys.getdefaultencoding()
418        try:
419            ostype = subprocess.check_output(
420                ['uname', '-s']).strip().decode(default_encoding)
421        except subprocess.CalledProcessError:
422            return
423        except OSError as reason:
424            if getattr(reason, 'winerror', None) is not None:
425                return
426            raise reason
427
428        if ostype != "Linux":
429            return
430
431        if not os.path.exists("/etc/NIXOS"):
432            return
433        if os.path.exists("/lib"):
434            return
435
436        # At this point we're pretty sure the user is running NixOS
437        nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
438        print(nix_os_msg, fname)
439
440        try:
441            interpreter = subprocess.check_output(
442                ["patchelf", "--print-interpreter", fname])
443            interpreter = interpreter.strip().decode(default_encoding)
444        except subprocess.CalledProcessError as reason:
445            print("warning: failed to call patchelf:", reason)
446            return
447
448        loader = interpreter.split("/")[-1]
449
450        try:
451            ldd_output = subprocess.check_output(
452                ['ldd', '/run/current-system/sw/bin/sh'])
453            ldd_output = ldd_output.strip().decode(default_encoding)
454        except subprocess.CalledProcessError as reason:
455            print("warning: unable to call ldd:", reason)
456            return
457
458        for line in ldd_output.splitlines():
459            libname = line.split()[0]
460            if libname.endswith(loader):
461                loader_path = libname[:len(libname) - len(loader)]
462                break
463        else:
464            print("warning: unable to find the path to the dynamic linker")
465            return
466
467        correct_interpreter = loader_path + loader
468
469        try:
470            subprocess.check_output(
471                ["patchelf", "--set-interpreter", correct_interpreter, fname])
472        except subprocess.CalledProcessError as reason:
473            print("warning: failed to call patchelf:", reason)
474            return
475
476    def rustc_stamp(self):
477        """Return the path for .rustc-stamp
478
479        >>> rb = RustBuild()
480        >>> rb.build_dir = "build"
481        >>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
482        True
483        """
484        return os.path.join(self.bin_root(), '.rustc-stamp')
485
486    def cargo_stamp(self):
487        """Return the path for .cargo-stamp
488
489        >>> rb = RustBuild()
490        >>> rb.build_dir = "build"
491        >>> rb.cargo_stamp() == os.path.join("build", "stage0", ".cargo-stamp")
492        True
493        """
494        return os.path.join(self.bin_root(), '.cargo-stamp')
495
496    def program_out_of_date(self, stamp_path):
497        """Check if the given program stamp is out of date"""
498        if not os.path.exists(stamp_path) or self.clean:
499            return True
500        with open(stamp_path, 'r') as stamp:
501            return self.date != stamp.read()
502
503    def bin_root(self):
504        """Return the binary root directory
505
506        >>> rb = RustBuild()
507        >>> rb.build_dir = "build"
508        >>> rb.bin_root() == os.path.join("build", "stage0")
509        True
510
511        When the 'build' property is given should be a nested directory:
512
513        >>> rb.build = "devel"
514        >>> rb.bin_root() == os.path.join("build", "devel", "stage0")
515        True
516        """
517        return os.path.join(self.build_dir, self.build, "stage0")
518
519    def get_toml(self, key, section=None):
520        """Returns the value of the given key in config.toml, otherwise returns None
521
522        >>> rb = RustBuild()
523        >>> rb.config_toml = 'key1 = "value1"\\nkey2 = "value2"'
524        >>> rb.get_toml("key2")
525        'value2'
526
527        If the key does not exists, the result is None:
528
529        >>> rb.get_toml("key3") is None
530        True
531
532        Optionally also matches the section the key appears in
533
534        >>> rb.config_toml = '[a]\\nkey = "value1"\\n[b]\\nkey = "value2"'
535        >>> rb.get_toml('key', 'a')
536        'value1'
537        >>> rb.get_toml('key', 'b')
538        'value2'
539        >>> rb.get_toml('key', 'c') is None
540        True
541
542        >>> rb.config_toml = 'key1 = true'
543        >>> rb.get_toml("key1")
544        'true'
545        """
546
547        cur_section = None
548        for line in self.config_toml.splitlines():
549            section_match = re.match(r'^\s*\[(.*)\]\s*$', line)
550            if section_match is not None:
551                cur_section = section_match.group(1)
552
553            match = re.match(r'^{}\s*=(.*)$'.format(key), line)
554            if match is not None:
555                value = match.group(1)
556                if section is None or section == cur_section:
557                    return self.get_string(value) or value.strip()
558        return None
559
560    def cargo(self):
561        """Return config path for cargo"""
562        return self.program_config('cargo')
563
564    def rustc(self):
565        """Return config path for rustc"""
566        return self.program_config('rustc')
567
568    def program_config(self, program):
569        """Return config path for the given program
570
571        >>> rb = RustBuild()
572        >>> rb.config_toml = 'rustc = "rustc"\\n'
573        >>> rb.program_config('rustc')
574        'rustc'
575        >>> rb.config_toml = ''
576        >>> cargo_path = rb.program_config('cargo')
577        >>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
578        ... "bin", "cargo")
579        True
580        """
581        config = self.get_toml(program)
582        if config:
583            return os.path.expanduser(config)
584        return os.path.join(self.bin_root(), "bin", "{}{}".format(
585            program, self.exe_suffix()))
586
587    @staticmethod
588    def get_string(line):
589        """Return the value between double quotes
590
591        >>> RustBuild.get_string('    "devel"   ')
592        'devel'
593        >>> RustBuild.get_string("    'devel'   ")
594        'devel'
595        >>> RustBuild.get_string('devel') is None
596        True
597        >>> RustBuild.get_string('    "devel   ')
598        ''
599        """
600        start = line.find('"')
601        if start != -1:
602            end = start + 1 + line[start + 1:].find('"')
603            return line[start + 1:end]
604        start = line.find('\'')
605        if start != -1:
606            end = start + 1 + line[start + 1:].find('\'')
607            return line[start + 1:end]
608        return None
609
610    @staticmethod
611    def exe_suffix():
612        """Return a suffix for executables"""
613        if sys.platform == 'win32':
614            return '.exe'
615        return ''
616
617    def bootstrap_binary(self):
618        """Return the path of the bootstrap binary
619
620        >>> rb = RustBuild()
621        >>> rb.build_dir = "build"
622        >>> rb.bootstrap_binary() == os.path.join("build", "bootstrap",
623        ... "debug", "bootstrap")
624        True
625        """
626        return os.path.join(self.build_dir, "bootstrap", "debug", "bootstrap")
627
628    def build_bootstrap(self):
629        """Build bootstrap"""
630        build_dir = os.path.join(self.build_dir, "bootstrap")
631        if self.clean and os.path.exists(build_dir):
632            shutil.rmtree(build_dir)
633        env = os.environ.copy()
634        env["RUSTC_BOOTSTRAP"] = '1'
635        env["CARGO_TARGET_DIR"] = build_dir
636        env["RUSTC"] = self.rustc()
637        env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
638            (os.pathsep + env["LD_LIBRARY_PATH"]) \
639            if "LD_LIBRARY_PATH" in env else ""
640        env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
641            (os.pathsep + env["DYLD_LIBRARY_PATH"]) \
642            if "DYLD_LIBRARY_PATH" in env else ""
643        env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
644            (os.pathsep + env["LIBRARY_PATH"]) \
645            if "LIBRARY_PATH" in env else ""
646        # preserve existing RUSTFLAGS
647        env.setdefault("RUSTFLAGS", "")
648        env["RUSTFLAGS"] += " -Cdebuginfo=2"
649
650        build_section = "target.{}".format(self.build_triple())
651        target_features = []
652        if self.get_toml("crt-static", build_section) == "true":
653            target_features += ["+crt-static"]
654        elif self.get_toml("crt-static", build_section) == "false":
655            target_features += ["-crt-static"]
656        if target_features:
657            env["RUSTFLAGS"] += " -C target-feature=" + (",".join(target_features))
658        target_linker = self.get_toml("linker", build_section)
659        if target_linker is not None:
660            env["RUSTFLAGS"] += " -C linker=" + target_linker
661        env["RUSTFLAGS"] += " -Wrust_2018_idioms -Wunused_lifetimes"
662        if self.get_toml("deny-warnings", "rust") != "false":
663            env["RUSTFLAGS"] += " -Dwarnings"
664
665        env["PATH"] = os.path.join(self.bin_root(), "bin") + \
666            os.pathsep + env["PATH"]
667        if not os.path.isfile(self.cargo()):
668            raise Exception("no cargo executable found at `{}`".format(
669                self.cargo()))
670        args = [self.cargo(), "build", "--manifest-path",
671                os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
672        for _ in range(1, self.verbose):
673            args.append("--verbose")
674        if self.use_locked_deps:
675            args.append("--locked")
676        if self.use_vendored_sources:
677            args.append("--frozen")
678        run(args, env=env, verbose=self.verbose)
679
680    def build_triple(self):
681        """Build triple as in LLVM"""
682        config = self.get_toml('build')
683        if config:
684            return config
685        return default_build_triple()
686
687    def check_submodule(self, module, slow_submodules):
688        if not slow_submodules:
689            checked_out = subprocess.Popen(["git", "rev-parse", "HEAD"],
690                                           cwd=os.path.join(self.rust_root, module),
691                                           stdout=subprocess.PIPE)
692            return checked_out
693        else:
694            return None
695
696    def update_submodule(self, module, checked_out, recorded_submodules):
697        module_path = os.path.join(self.rust_root, module)
698
699        if checked_out is not None:
700            default_encoding = sys.getdefaultencoding()
701            checked_out = checked_out.communicate()[0].decode(default_encoding).strip()
702            if recorded_submodules[module] == checked_out:
703                return
704
705        print("Updating submodule", module)
706
707        run(["git", "submodule", "-q", "sync", module],
708            cwd=self.rust_root, verbose=self.verbose)
709        try:
710            run(["git", "submodule", "update",
711                 "--init", "--recursive", "--progress", module],
712                cwd=self.rust_root, verbose=self.verbose, exception=True)
713        except RuntimeError:
714            # Some versions of git don't support --progress.
715            run(["git", "submodule", "update",
716                 "--init", "--recursive", module],
717                cwd=self.rust_root, verbose=self.verbose)
718        run(["git", "reset", "-q", "--hard"],
719            cwd=module_path, verbose=self.verbose)
720        run(["git", "clean", "-qdfx"],
721            cwd=module_path, verbose=self.verbose)
722
723    def update_submodules(self):
724        """Update submodules"""
725        if (not os.path.exists(os.path.join(self.rust_root, ".git"))) or \
726                self.get_toml('submodules') == "false":
727            return
728
729        # check the existence of 'git' command
730        try:
731            subprocess.check_output(['git', '--version'])
732        except (subprocess.CalledProcessError, OSError):
733            print("error: `git` is not found, please make sure it's installed and in the path.")
734            sys.exit(1)
735
736        slow_submodules = self.get_toml('fast-submodules') == "false"
737        start_time = time()
738        if slow_submodules:
739            print('Unconditionally updating all submodules')
740        else:
741            print('Updating only changed submodules')
742        default_encoding = sys.getdefaultencoding()
743        submodules = [s.split(' ', 1)[1] for s in subprocess.check_output(
744            ["git", "config", "--file",
745             os.path.join(self.rust_root, ".gitmodules"),
746             "--get-regexp", "path"]
747        ).decode(default_encoding).splitlines()]
748        filtered_submodules = []
749        submodules_names = []
750        for module in submodules:
751            if module.endswith("llvm-project"):
752                if self.get_toml('llvm-config') and self.get_toml('lld') != 'true':
753                    continue
754            check = self.check_submodule(module, slow_submodules)
755            filtered_submodules.append((module, check))
756            submodules_names.append(module)
757        recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names,
758                                    cwd=self.rust_root, stdout=subprocess.PIPE)
759        recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines()
760        recorded_submodules = {}
761        for data in recorded:
762            data = data.split()
763            recorded_submodules[data[3]] = data[2]
764        for module in filtered_submodules:
765            self.update_submodule(module[0], module[1], recorded_submodules)
766        print("Submodules updated in %.2f seconds" % (time() - start_time))
767
768    def set_normal_environment(self):
769        """Set download URL for normal environment"""
770        if 'RUSTUP_DIST_SERVER' in os.environ:
771            self._download_url = os.environ['RUSTUP_DIST_SERVER']
772        else:
773            self._download_url = 'https://static.rust-lang.org'
774
775    def set_dev_environment(self):
776        """Set download URL for development environment"""
777        if 'RUSTUP_DEV_DIST_SERVER' in os.environ:
778            self._download_url = os.environ['RUSTUP_DEV_DIST_SERVER']
779        else:
780            self._download_url = 'https://dev-static.rust-lang.org'
781
782    def check_vendored_status(self):
783        """Check that vendoring is configured properly"""
784        vendor_dir = os.path.join(self.rust_root, 'vendor')
785        if 'SUDO_USER' in os.environ and not self.use_vendored_sources:
786            if os.environ.get('USER') != os.environ['SUDO_USER']:
787                self.use_vendored_sources = True
788                print('info: looks like you are running this command under `sudo`')
789                print('      and so in order to preserve your $HOME this will now')
790                print('      use vendored sources by default.')
791                if not os.path.exists(vendor_dir):
792                    print('error: vendoring required, but vendor directory does not exist.')
793                    print('       Run `cargo vendor` without sudo to initialize the '
794                        'vendor directory.')
795                    raise Exception("{} not found".format(vendor_dir))
796
797        if self.use_vendored_sources:
798            if not os.path.exists('.cargo'):
799                os.makedirs('.cargo')
800            with output('.cargo/config') as cargo_config:
801                cargo_config.write(
802                    "[source.crates-io]\n"
803                    "replace-with = 'vendored-sources'\n"
804                    "registry = 'https://example.com'\n"
805                    "\n"
806                    "[source.vendored-sources]\n"
807                    "directory = '{}/vendor'\n"
808                .format(self.rust_root))
809        else:
810            if os.path.exists('.cargo'):
811                shutil.rmtree('.cargo')
812
813    def ensure_vendored(self):
814        """Ensure that the vendored sources are available if needed"""
815        vendor_dir = os.path.join(self.rust_root, 'vendor')
816        # Note that this does not handle updating the vendored dependencies if
817        # the rust git repository is updated. Normal development usually does
818        # not use vendoring, so hopefully this isn't too much of a problem.
819        if self.use_vendored_sources and not os.path.exists(vendor_dir):
820            run([self.cargo(), "vendor"],
821                verbose=self.verbose, cwd=self.rust_root)
822
823
824def bootstrap(help_triggered):
825    """Configure, fetch, build and run the initial bootstrap"""
826
827    # If the user is asking for help, let them know that the whole download-and-build
828    # process has to happen before anything is printed out.
829    if help_triggered:
830        print("info: Downloading and building bootstrap before processing --help")
831        print("      command. See src/bootstrap/README.md for help with common")
832        print("      commands.")
833
834    parser = argparse.ArgumentParser(description='Build rust')
835    parser.add_argument('--config')
836    parser.add_argument('--build')
837    parser.add_argument('--src')
838    parser.add_argument('--clean', action='store_true')
839    parser.add_argument('-v', '--verbose', action='count', default=0)
840
841    args = [a for a in sys.argv if a != '-h' and a != '--help']
842    args, _ = parser.parse_known_args(args)
843
844    # Configure initial bootstrap
845    build = RustBuild()
846    build.rust_root = args.src or os.path.abspath(os.path.join(__file__, '../../..'))
847    build.verbose = args.verbose
848    build.clean = args.clean
849
850    try:
851        with open(args.config or 'config.toml') as config:
852            build.config_toml = config.read()
853    except (OSError, IOError):
854        pass
855
856    config_verbose = build.get_toml('verbose', 'build')
857    if config_verbose is not None:
858        build.verbose = max(build.verbose, int(config_verbose))
859
860    build.use_vendored_sources = build.get_toml('vendor', 'build') == 'true'
861
862    build.use_locked_deps = build.get_toml('locked-deps', 'build') == 'true'
863
864    build.check_vendored_status()
865
866    data = stage0_data(build.rust_root)
867    build.date = data['date']
868    build.rustc_channel = data['rustc']
869    build.cargo_channel = data['cargo']
870
871    if 'dev' in data:
872        build.set_dev_environment()
873    else:
874        build.set_normal_environment()
875
876    build.update_submodules()
877
878    # Fetch/build the bootstrap
879    build.build = args.build or build.build_triple()
880    build.download_stage0()
881    sys.stdout.flush()
882    build.ensure_vendored()
883    build.build_bootstrap()
884    sys.stdout.flush()
885
886    # Run the bootstrap
887    args = [build.bootstrap_binary()]
888    args.extend(sys.argv[1:])
889    env = os.environ.copy()
890    env["BUILD"] = build.build
891    env["SRC"] = build.rust_root
892    env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
893    env["BOOTSTRAP_PYTHON"] = sys.executable
894    env["BUILD_DIR"] = build.build_dir
895    env["RUSTC_BOOTSTRAP"] = '1'
896    env["CARGO"] = build.cargo()
897    env["RUSTC"] = build.rustc()
898    run(args, env=env, verbose=build.verbose)
899
900
901def main():
902    """Entry point for the bootstrap process"""
903    start_time = time()
904
905    # x.py help <cmd> ...
906    if len(sys.argv) > 1 and sys.argv[1] == 'help':
907        sys.argv = [sys.argv[0], '-h'] + sys.argv[2:]
908
909    help_triggered = (
910        '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1)
911    try:
912        bootstrap(help_triggered)
913        if not help_triggered:
914            print("Build completed successfully in {}".format(
915                format_build_time(time() - start_time)))
916    except (SystemExit, KeyboardInterrupt) as error:
917        if hasattr(error, 'code') and isinstance(error.code, int):
918            exit_code = error.code
919        else:
920            exit_code = 1
921            print(error)
922        if not help_triggered:
923            print("Build completed unsuccessfully in {}".format(
924                format_build_time(time() - start_time)))
925        sys.exit(exit_code)
926
927
928if __name__ == '__main__':
929    main()