fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #103046 - JanBeh:PR_clarify_cmp_terminology, r=workingjubilee3bcad65fbf6

master
24.7 KiB586 linesraw
1#!/usr/bin/env python
2
3# ignore-tidy-linelength
4
5from __future__ import absolute_import, division, print_function
6import sys
7import os
8rust_dir = os.path.dirname(os.path.abspath(__file__))
9rust_dir = os.path.dirname(rust_dir)
10rust_dir = os.path.dirname(rust_dir)
11sys.path.append(os.path.join(rust_dir, "src", "bootstrap"))
12import bootstrap # noqa: E402
13
14
15class Option(object):
16    def __init__(self, name, rustbuild, desc, value):
17        self.name = name
18        self.rustbuild = rustbuild
19        self.desc = desc
20        self.value = value
21
22
23options = []
24
25
26def o(*args):
27    options.append(Option(*args, value=False))
28
29
30def v(*args):
31    options.append(Option(*args, value=True))
32
33
34o("debug", "rust.debug", "enables debugging environment; does not affect optimization of bootstrapped code")
35o("docs", "build.docs", "build standard library documentation")
36o("compiler-docs", "build.compiler-docs", "build compiler documentation")
37o("optimize-tests", "rust.optimize-tests", "build tests with optimizations")
38o("verbose-tests", "rust.verbose-tests", "enable verbose output when running tests")
39o("ccache", "llvm.ccache", "invoke gcc/clang via ccache to reuse object files between builds")
40o("sccache", None, "invoke gcc/clang via sccache to reuse object files between builds")
41o("local-rust", None, "use an installed rustc rather than downloading a snapshot")
42v("local-rust-root", None, "set prefix for local rust binary")
43o("local-rebuild", "build.local-rebuild", "assume local-rust matches the current version, for rebuilds; implies local-rust, and is implied if local-rust already matches the current version")
44o("llvm-static-stdcpp", "llvm.static-libstdcpp", "statically link to libstdc++ for LLVM")
45o("llvm-link-shared", "llvm.link-shared", "prefer shared linking to LLVM (llvm-config --link-shared)")
46o("rpath", "rust.rpath", "build rpaths into rustc itself")
47o("codegen-tests", "rust.codegen-tests", "run the tests/codegen tests")
48o("ninja", "llvm.ninja", "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)")
49o("locked-deps", "build.locked-deps", "force Cargo.lock to be up to date")
50o("vendor", "build.vendor", "enable usage of vendored Rust crates")
51o("sanitizers", "build.sanitizers", "build the sanitizer runtimes (asan, lsan, msan, tsan, hwasan)")
52o("dist-src", "rust.dist-src", "when building tarballs enables building a source tarball")
53o("cargo-native-static", "build.cargo-native-static", "static native libraries in cargo")
54o("profiler", "build.profiler", "build the profiler runtime")
55o("full-tools", None, "enable all tools")
56o("lld", "rust.lld", "build lld")
57o("clang", "llvm.clang", "build clang")
58o("missing-tools", "dist.missing-tools", "allow failures when building tools")
59o("use-libcxx", "llvm.use-libcxx", "build LLVM with libc++")
60o("control-flow-guard", "rust.control-flow-guard", "Enable Control Flow Guard")
61o("patch-binaries-for-nix", "build.patch-binaries-for-nix", "whether patch binaries for usage with Nix toolchains")
62
63v("llvm-cflags", "llvm.cflags", "build LLVM with these extra compiler flags")
64v("llvm-cxxflags", "llvm.cxxflags", "build LLVM with these extra compiler flags")
65v("llvm-ldflags", "llvm.ldflags", "build LLVM with these extra linker flags")
66
67v("llvm-libunwind", "rust.llvm-libunwind", "use LLVM libunwind")
68
69# Optimization and debugging options. These may be overridden by the release
70# channel, etc.
71o("optimize-llvm", "llvm.optimize", "build optimized LLVM")
72o("llvm-assertions", "llvm.assertions", "build LLVM with assertions")
73o("llvm-plugins", "llvm.plugins", "build LLVM with plugin interface")
74o("debug-assertions", "rust.debug-assertions", "build with debugging assertions")
75o("debug-assertions-std", "rust.debug-assertions-std", "build the standard library with debugging assertions")
76o("overflow-checks", "rust.overflow-checks", "build with overflow checks")
77o("overflow-checks-std", "rust.overflow-checks-std", "build the standard library with overflow checks")
78o("llvm-release-debuginfo", "llvm.release-debuginfo", "build LLVM with debugger metadata")
79v("debuginfo-level", "rust.debuginfo-level", "debuginfo level for Rust code")
80v("debuginfo-level-rustc", "rust.debuginfo-level-rustc", "debuginfo level for the compiler")
81v("debuginfo-level-std", "rust.debuginfo-level-std", "debuginfo level for the standard library")
82v("debuginfo-level-tools", "rust.debuginfo-level-tools", "debuginfo level for the tools")
83v("debuginfo-level-tests", "rust.debuginfo-level-tests", "debuginfo level for the test suites run with compiletest")
84v("save-toolstates", "rust.save-toolstates", "save build and test status of external tools into this file")
85
86v("prefix", "install.prefix", "set installation prefix")
87v("localstatedir", "install.localstatedir", "local state directory")
88v("datadir", "install.datadir", "install data")
89v("sysconfdir", "install.sysconfdir", "install system configuration files")
90v("infodir", "install.infodir", "install additional info")
91v("libdir", "install.libdir", "install libraries")
92v("mandir", "install.mandir", "install man pages in PATH")
93v("docdir", "install.docdir", "install documentation in PATH")
94v("bindir", "install.bindir", "install binaries")
95
96v("llvm-root", None, "set LLVM root")
97v("llvm-config", None, "set path to llvm-config")
98v("llvm-filecheck", None, "set path to LLVM's FileCheck utility")
99v("python", "build.python", "set path to python")
100v("android-cross-path", "target.arm-linux-androideabi.android-ndk",
101  "Android NDK standalone path (deprecated)")
102v("i686-linux-android-ndk", "target.i686-linux-android.android-ndk",
103  "i686-linux-android NDK standalone path")
104v("arm-linux-androideabi-ndk", "target.arm-linux-androideabi.android-ndk",
105  "arm-linux-androideabi NDK standalone path")
106v("armv7-linux-androideabi-ndk", "target.armv7-linux-androideabi.android-ndk",
107  "armv7-linux-androideabi NDK standalone path")
108v("thumbv7neon-linux-androideabi-ndk", "target.thumbv7neon-linux-androideabi.android-ndk",
109  "thumbv7neon-linux-androideabi NDK standalone path")
110v("aarch64-linux-android-ndk", "target.aarch64-linux-android.android-ndk",
111  "aarch64-linux-android NDK standalone path")
112v("x86_64-linux-android-ndk", "target.x86_64-linux-android.android-ndk",
113  "x86_64-linux-android NDK standalone path")
114v("musl-root", "target.x86_64-unknown-linux-musl.musl-root",
115  "MUSL root installation directory (deprecated)")
116v("musl-root-x86_64", "target.x86_64-unknown-linux-musl.musl-root",
117  "x86_64-unknown-linux-musl install directory")
118v("musl-root-i586", "target.i586-unknown-linux-musl.musl-root",
119  "i586-unknown-linux-musl install directory")
120v("musl-root-i686", "target.i686-unknown-linux-musl.musl-root",
121  "i686-unknown-linux-musl install directory")
122v("musl-root-arm", "target.arm-unknown-linux-musleabi.musl-root",
123  "arm-unknown-linux-musleabi install directory")
124v("musl-root-armhf", "target.arm-unknown-linux-musleabihf.musl-root",
125  "arm-unknown-linux-musleabihf install directory")
126v("musl-root-armv5te", "target.armv5te-unknown-linux-musleabi.musl-root",
127  "armv5te-unknown-linux-musleabi install directory")
128v("musl-root-armv7", "target.armv7-unknown-linux-musleabi.musl-root",
129  "armv7-unknown-linux-musleabi install directory")
130v("musl-root-armv7hf", "target.armv7-unknown-linux-musleabihf.musl-root",
131  "armv7-unknown-linux-musleabihf install directory")
132v("musl-root-aarch64", "target.aarch64-unknown-linux-musl.musl-root",
133  "aarch64-unknown-linux-musl install directory")
134v("musl-root-mips", "target.mips-unknown-linux-musl.musl-root",
135  "mips-unknown-linux-musl install directory")
136v("musl-root-mipsel", "target.mipsel-unknown-linux-musl.musl-root",
137  "mipsel-unknown-linux-musl install directory")
138v("musl-root-mips64", "target.mips64-unknown-linux-muslabi64.musl-root",
139  "mips64-unknown-linux-muslabi64 install directory")
140v("musl-root-mips64el", "target.mips64el-unknown-linux-muslabi64.musl-root",
141  "mips64el-unknown-linux-muslabi64 install directory")
142v("musl-root-riscv32gc", "target.riscv32gc-unknown-linux-musl.musl-root",
143  "riscv32gc-unknown-linux-musl install directory")
144v("musl-root-riscv64gc", "target.riscv64gc-unknown-linux-musl.musl-root",
145  "riscv64gc-unknown-linux-musl install directory")
146v("qemu-armhf-rootfs", "target.arm-unknown-linux-gnueabihf.qemu-rootfs",
147  "rootfs in qemu testing, you probably don't want to use this")
148v("qemu-aarch64-rootfs", "target.aarch64-unknown-linux-gnu.qemu-rootfs",
149  "rootfs in qemu testing, you probably don't want to use this")
150v("qemu-riscv64-rootfs", "target.riscv64gc-unknown-linux-gnu.qemu-rootfs",
151  "rootfs in qemu testing, you probably don't want to use this")
152v("experimental-targets", "llvm.experimental-targets",
153  "experimental LLVM targets to build")
154v("release-channel", "rust.channel", "the name of the release channel to build")
155v("release-description", "rust.description", "optional descriptive string for version output")
156v("dist-compression-formats", None, "List of compression formats to use")
157
158# Used on systems where "cc" is unavailable
159v("default-linker", "rust.default-linker", "the default linker")
160
161# Many of these are saved below during the "writing configuration" step
162# (others are conditionally saved).
163o("manage-submodules", "build.submodules", "let the build manage the git submodules")
164o("full-bootstrap", "build.full-bootstrap", "build three compilers instead of two (not recommended except for testing reproducible builds)")
165o("extended", "build.extended", "build an extended rust tool set")
166
167v("tools", None, "List of extended tools will be installed")
168v("codegen-backends", None, "List of codegen backends to build")
169v("build", "build.build", "GNUs ./configure syntax LLVM build triple")
170v("host", None, "List of GNUs ./configure syntax LLVM host triples")
171v("target", None, "List of GNUs ./configure syntax LLVM target triples")
172
173# Options specific to this configure script
174o("option-checking", None, "complain about unrecognized options in this configure script")
175o("verbose-configure", None, "don't truncate options when printing them in this configure script")
176v("set", None, "set arbitrary key/value pairs in TOML configuration")
177
178
179def p(msg):
180    print("configure: " + msg)
181
182
183def err(msg):
184    print("\nconfigure: ERROR: " + msg + "\n")
185    sys.exit(1)
186
187def is_value_list(key):
188    for option in options:
189        if option.name == key and option.desc.startswith('List of'):
190            return True
191    return False
192
193if '--help' in sys.argv or '-h' in sys.argv:
194    print('Usage: ./configure [options]')
195    print('')
196    print('Options')
197    for option in options:
198        if 'android' in option.name:
199            # no one needs to know about these obscure options
200            continue
201        if option.value:
202            print('\t{:30} {}'.format('--{}=VAL'.format(option.name), option.desc))
203        else:
204            print('\t{:30} {}'.format('--enable-{}'.format(option.name), option.desc))
205    print('')
206    print('This configure script is a thin configuration shim over the true')
207    print('configuration system, `config.toml`. You can explore the comments')
208    print('in `config.example.toml` next to this configure script to see')
209    print('more information about what each option is. Additionally you can')
210    print('pass `--set` as an argument to set arbitrary key/value pairs')
211    print('in the TOML configuration if desired')
212    print('')
213    print('Also note that all options which take `--enable` can similarly')
214    print('be passed with `--disable-foo` to forcibly disable the option')
215    sys.exit(0)
216
217VERBOSE = False
218
219# Parse all command line arguments into one of these three lists, handling
220# boolean and value-based options separately
221def parse_args(args):
222    unknown_args = []
223    need_value_args = []
224    known_args = {}
225
226    i = 0
227    while i < len(args):
228        arg = args[i]
229        i += 1
230        if not arg.startswith('--'):
231            unknown_args.append(arg)
232            continue
233
234        found = False
235        for option in options:
236            value = None
237            if option.value:
238                keyval = arg[2:].split('=', 1)
239                key = keyval[0]
240                if option.name != key:
241                    continue
242
243                if len(keyval) > 1:
244                    value = keyval[1]
245                elif i < len(args):
246                    value = args[i]
247                    i += 1
248                else:
249                    need_value_args.append(arg)
250                    continue
251            else:
252                if arg[2:] == 'enable-' + option.name:
253                    value = True
254                elif arg[2:] == 'disable-' + option.name:
255                    value = False
256                else:
257                    continue
258
259            found = True
260            if option.name not in known_args:
261                known_args[option.name] = []
262            known_args[option.name].append((option, value))
263            break
264
265        if not found:
266            unknown_args.append(arg)
267
268    # Note: here and a few other places, we use [-1] to apply the *last* value
269    # passed.  But if option-checking is enabled, then the known_args loop will
270    # also assert that options are only passed once.
271    option_checking = ('option-checking' not in known_args
272                    or known_args['option-checking'][-1][1])
273    if option_checking:
274        if len(unknown_args) > 0:
275            err("Option '" + unknown_args[0] + "' is not recognized")
276        if len(need_value_args) > 0:
277            err("Option '{0}' needs a value ({0}=val)".format(need_value_args[0]))
278
279    global VERBOSE
280    VERBOSE = 'verbose-configure' in known_args
281
282    config = {}
283
284    set('build.configure-args', args, config)
285    apply_args(known_args, option_checking, config)
286    return parse_example_config(known_args, config)
287
288
289def build(known_args):
290    if 'build' in known_args:
291        return known_args['build'][-1][1]
292    return bootstrap.default_build_triple(verbose=False)
293
294
295def set(key, value, config):
296    if isinstance(value, list):
297        # Remove empty values, which value.split(',') tends to generate.
298        value = [v for v in value if v]
299
300    s = "{:20} := {}".format(key, value)
301    if len(s) < 70 or VERBOSE:
302        p(s)
303    else:
304        p(s[:70] + " ...")
305
306    arr = config
307    parts = key.split('.')
308    for i, part in enumerate(parts):
309        if i == len(parts) - 1:
310            if is_value_list(part) and isinstance(value, str):
311                value = value.split(',')
312            arr[part] = value
313        else:
314            if part not in arr:
315                arr[part] = {}
316            arr = arr[part]
317
318
319def apply_args(known_args, option_checking, config):
320    for key in known_args:
321        # The `set` option is special and can be passed a bunch of times
322        if key == 'set':
323            for _option, value in known_args[key]:
324                keyval = value.split('=', 1)
325                if len(keyval) == 1 or keyval[1] == "true":
326                    value = True
327                elif keyval[1] == "false":
328                    value = False
329                else:
330                    value = keyval[1]
331                set(keyval[0], value, config)
332            continue
333
334        # Ensure each option is only passed once
335        arr = known_args[key]
336        if option_checking and len(arr) > 1:
337            err("Option '{}' provided more than once".format(key))
338        option, value = arr[-1]
339
340        # If we have a clear avenue to set our value in rustbuild, do so
341        if option.rustbuild is not None:
342            set(option.rustbuild, value, config)
343            continue
344
345        # Otherwise we're a "special" option and need some extra handling, so do
346        # that here.
347        build_triple = build(known_args)
348
349        if option.name == 'sccache':
350            set('llvm.ccache', 'sccache', config)
351        elif option.name == 'local-rust':
352            for path in os.environ['PATH'].split(os.pathsep):
353                if os.path.exists(path + '/rustc'):
354                    set('build.rustc', path + '/rustc', config)
355                    break
356            for path in os.environ['PATH'].split(os.pathsep):
357                if os.path.exists(path + '/cargo'):
358                    set('build.cargo', path + '/cargo', config)
359                    break
360        elif option.name == 'local-rust-root':
361            set('build.rustc', value + '/bin/rustc', config)
362            set('build.cargo', value + '/bin/cargo', config)
363        elif option.name == 'llvm-root':
364            set('target.{}.llvm-config'.format(build_triple), value + '/bin/llvm-config', config)
365        elif option.name == 'llvm-config':
366            set('target.{}.llvm-config'.format(build_triple), value, config)
367        elif option.name == 'llvm-filecheck':
368            set('target.{}.llvm-filecheck'.format(build_triple), value, config)
369        elif option.name == 'tools':
370            set('build.tools', value.split(','), config)
371        elif option.name == 'codegen-backends':
372            set('rust.codegen-backends', value.split(','), config)
373        elif option.name == 'host':
374            set('build.host', value.split(','), config)
375        elif option.name == 'target':
376            set('build.target', value.split(','), config)
377        elif option.name == 'full-tools':
378            set('rust.codegen-backends', ['llvm'], config)
379            set('rust.lld', True, config)
380            set('rust.llvm-tools', True, config)
381            set('build.extended', True, config)
382        elif option.name in ['option-checking', 'verbose-configure']:
383            # this was handled above
384            pass
385        elif option.name == 'dist-compression-formats':
386            set('dist.compression-formats', value.split(','), config)
387        else:
388            raise RuntimeError("unhandled option {}".format(option.name))
389
390# "Parse" the `config.example.toml` file into the various sections, and we'll
391# use this as a template of a `config.toml` to write out which preserves
392# all the various comments and whatnot.
393#
394# Note that the `target` section is handled separately as we'll duplicate it
395# per configured target, so there's a bit of special handling for that here.
396def parse_example_config(known_args, config):
397    sections = {}
398    cur_section = None
399    sections[None] = []
400    section_order = [None]
401    targets = {}
402    top_level_keys = []
403
404    with open(rust_dir + '/config.example.toml') as example_config:
405        example_lines = example_config.read().split("\n")
406    for line in example_lines:
407        if cur_section is None:
408            if line.count('=') == 1:
409                top_level_key = line.split('=')[0]
410                top_level_key = top_level_key.strip(' #')
411                top_level_keys.append(top_level_key)
412        if line.startswith('['):
413            cur_section = line[1:-1]
414            if cur_section.startswith('target'):
415                cur_section = 'target'
416            elif '.' in cur_section:
417                raise RuntimeError("don't know how to deal with section: {}".format(cur_section))
418            sections[cur_section] = [line]
419            section_order.append(cur_section)
420        else:
421            sections[cur_section].append(line)
422
423    # Fill out the `targets` array by giving all configured targets a copy of the
424    # `target` section we just loaded from the example config
425    configured_targets = [build(known_args)]
426    if 'build' in config:
427        if 'host' in config['build']:
428            configured_targets += config['build']['host']
429        if 'target' in config['build']:
430            configured_targets += config['build']['target']
431    if 'target' in config:
432        for target in config['target']:
433            configured_targets.append(target)
434    for target in configured_targets:
435        targets[target] = sections['target'][:]
436        # For `.` to be valid TOML, it needs to be quoted. But `bootstrap.py` doesn't use a proper TOML parser and fails to parse the target.
437        # Avoid using quotes unless it's necessary.
438        targets[target][0] = targets[target][0].replace("x86_64-unknown-linux-gnu", "'{}'".format(target) if "." in target else target)
439
440    if 'profile' not in config:
441        set('profile', 'dist', config)
442    configure_file(sections, top_level_keys, targets, config)
443    return section_order, sections, targets
444
445
446def is_number(value):
447    try:
448        float(value)
449        return True
450    except ValueError:
451        return False
452
453
454# Here we walk through the constructed configuration we have from the parsed
455# command line arguments. We then apply each piece of configuration by
456# basically just doing a `sed` to change the various configuration line to what
457# we've got configure.
458def to_toml(value):
459    if isinstance(value, bool):
460        if value:
461            return "true"
462        else:
463            return "false"
464    elif isinstance(value, list):
465        return '[' + ', '.join(map(to_toml, value)) + ']'
466    elif isinstance(value, str):
467        # Don't put quotes around numeric values
468        if is_number(value):
469            return value
470        else:
471            return "'" + value + "'"
472    elif isinstance(value, dict):
473        return "{" + ", ".join(map(lambda a: "{} = {}".format(to_toml(a[0]), to_toml(a[1])), value.items())) + "}"
474    else:
475        raise RuntimeError('no toml')
476
477
478def configure_section(lines, config):
479    for key in config:
480        value = config[key]
481        found = False
482        for i, line in enumerate(lines):
483            if not line.startswith('#' + key + ' = '):
484                continue
485            found = True
486            lines[i] = "{} = {}".format(key, to_toml(value))
487            break
488        if not found:
489            # These are used by rpm, but aren't accepted by x.py.
490            # Give a warning that they're ignored, but not a hard error.
491            if key in ["infodir", "localstatedir"]:
492                print("warning: {} will be ignored".format(key))
493            else:
494                raise RuntimeError("failed to find config line for {}".format(key))
495
496
497def configure_top_level_key(lines, top_level_key, value):
498    for i, line in enumerate(lines):
499        if line.startswith('#' + top_level_key + ' = ') or line.startswith(top_level_key + ' = '):
500            lines[i] = "{} = {}".format(top_level_key, to_toml(value))
501            return
502
503    raise RuntimeError("failed to find config line for {}".format(top_level_key))
504
505
506# Modify `sections` to reflect the parsed arguments and example configs.
507def configure_file(sections, top_level_keys, targets, config):
508    for section_key, section_config in config.items():
509        if section_key not in sections and section_key not in top_level_keys:
510            raise RuntimeError("config key {} not in sections or top_level_keys".format(section_key))
511        if section_key in top_level_keys:
512            configure_top_level_key(sections[None], section_key, section_config)
513
514        elif  section_key == 'target':
515            for target in section_config:
516                configure_section(targets[target], section_config[target])
517        else:
518            configure_section(sections[section_key], section_config)
519
520
521def write_uncommented(target, f):
522    block = []
523    is_comment = True
524
525    for line in target:
526        block.append(line)
527        if len(line) == 0:
528            if not is_comment:
529                for ln in block:
530                    f.write(ln + "\n")
531            block = []
532            is_comment = True
533            continue
534        is_comment = is_comment and line.startswith('#')
535    return f
536
537
538def write_config_toml(writer, section_order, targets, sections):
539    for section in section_order:
540        if section == 'target':
541            for target in targets:
542                writer = write_uncommented(targets[target], writer)
543        else:
544            writer = write_uncommented(sections[section], writer)
545
546def quit_if_file_exists(file):
547    if os.path.isfile(file):
548        msg = "Existing '{}' detected. Exiting".format(file)
549
550        # If the output object directory isn't empty, we can get these errors
551        host_objdir = os.environ.get("OBJDIR_ON_HOST")
552        if host_objdir is not None:
553            msg += "\nIs objdir '{}' clean?".format(host_objdir)
554
555        err(msg)
556
557if __name__ == "__main__":
558    # If 'config.toml' already exists, exit the script at this point
559    quit_if_file_exists('config.toml')
560
561    if "GITHUB_ACTIONS" in os.environ:
562        print("::group::Configure the build")
563    p("processing command line")
564    # Parse all known arguments into a configuration structure that reflects the
565    # TOML we're going to write out
566    p("")
567    section_order, sections, targets = parse_args(sys.argv[1:])
568
569    # Now that we've built up our `config.toml`, write it all out in the same
570    # order that we read it in.
571    p("")
572    p("writing `config.toml` in current directory")
573    with bootstrap.output('config.toml') as f:
574        write_config_toml(f, section_order, targets, sections)
575
576    with bootstrap.output('Makefile') as f:
577        contents = os.path.join(rust_dir, 'src', 'bootstrap', 'mk', 'Makefile.in')
578        contents = open(contents).read()
579        contents = contents.replace("$(CFG_SRC_DIR)", rust_dir + '/')
580        contents = contents.replace("$(CFG_PYTHON)", sys.executable)
581        f.write(contents)
582
583    p("")
584    p("run `python {}/x.py --help`".format(rust_dir))
585    if "GITHUB_ACTIONS" in os.environ:
586        print("::endgroup::")