fork/rules_rust

git clone https://git.julia.blue/fork/rules_rust

Daniel Wagner-HallExternal path dependencies work (#3025)7edda6f1

main
20.2 KiB570 linesraw
1"""Rules for vendoring Bazel targets into existing workspaces"""
2
3load("//crate_universe/private:generate_utils.bzl", "compile_config", "render_config")
4load("//crate_universe/private:splicing_utils.bzl", "kebab_case_keys", generate_splicing_config = "splicing_config")
5load("//crate_universe/private:urls.bzl", "CARGO_BAZEL_LABEL")
6load("//rust/platform:triple_mappings.bzl", "SUPPORTED_PLATFORM_TRIPLES")
7
8_UNIX_WRAPPER = """\
9#!/usr/bin/env bash
10
11set -euo pipefail
12
13export RUNTIME_PWD="$(pwd)"
14if [[ -z "${{BAZEL_REAL:-}}" ]]; then
15    BAZEL_REAL="$(which bazel || echo 'bazel')"
16fi
17
18_ENVIRON=()
19_ENVIRON+=(BAZEL_REAL="${{BAZEL_REAL}}")
20_ENVIRON+=(BUILD_WORKSPACE_DIRECTORY="${{BUILD_WORKSPACE_DIRECTORY}}")
21_ENVIRON+=(PATH="${{PATH}}")
22
23if [[ -n "${{CARGO_BAZEL_DEBUG:-}}" ]]; then
24    _ENVIRON+=(CARGO_BAZEL_DEBUG="${{CARGO_BAZEL_DEBUG}}")
25fi
26
27# The path needs to be preserved to prevent bazel from starting with different
28# startup options (requiring a restart of bazel).
29# If you provide an empty path, bazel starts itself with
30# --default_system_javabase set to the empty string, but if you provide a path,
31# it may set it to a value (eg. "/usr/local/buildtools/java/jdk11").
32exec env - \\
33${{_ENVIRON[@]}} \\
34{env} \\
35    "{bin}" \\
36    {args} \\
37    --nonhermetic-root-bazel-workspace-dir="${{BUILD_WORKSPACE_DIRECTORY}}" \\
38    "$@"
39"""
40
41_WINDOWS_WRAPPER = """\
42@ECHO OFF
43set RUNTIME_PWD=%CD%
44{env}
45
46{bin} {args} --nonhermetic-root-bazel-workspace-dir=%BUILD_WORKSPACE_DIRECTORY% %*
47exit %ERRORLEVEL%
48"""
49
50CARGO_BAZEL_GENERATOR_PATH = "CARGO_BAZEL_GENERATOR_PATH"
51
52def _default_render_config():
53    return json.decode(render_config())
54
55def _runfiles_path(file, is_windows):
56    if is_windows:
57        runtime_pwd_var = "%RUNTIME_PWD%"
58    else:
59        runtime_pwd_var = "${RUNTIME_PWD}"
60
61    return "{}/{}".format(runtime_pwd_var, file.short_path)
62
63def _is_windows(ctx):
64    toolchain = ctx.toolchains[Label("@rules_rust//rust:toolchain_type")]
65    return toolchain.target_os == "windows"
66
67def _get_output_package(ctx):
68    # Determine output directory
69    if ctx.attr.vendor_path.startswith("/"):
70        output = ctx.attr.vendor_path
71    else:
72        output = "{}/{}".format(
73            ctx.label.package,
74            ctx.attr.vendor_path,
75        )
76    return output.lstrip("/")
77
78def _write_data_file(ctx, name, data):
79    file = ctx.actions.declare_file("{}.{}".format(ctx.label.name, name))
80    ctx.actions.write(
81        output = file,
82        content = data,
83    )
84    return file
85
86def _prepare_manifest_path(target):
87    """Generate manifest paths that are resolvable by `cargo_bazel::SplicingManifest::resolve`
88
89    Args:
90        target (Target): A `crate_vendor.manifest` target
91
92    Returns:
93        str: A string representing the path to a manifest.
94    """
95    files = target[DefaultInfo].files.to_list()
96    if len(files) != 1:
97        fail("The manifest {} hand an unexpected number of files: {}".format(
98            target.label,
99            files,
100        ))
101
102    manifest = files[0]
103
104    if target.label.workspace_root.startswith("external"):
105        # The short path of an external file is expected to start with `../`
106        if not manifest.short_path.startswith("../"):
107            fail("Unexpected shortpath for {}: {}".format(
108                manifest,
109                manifest.short_path,
110            ))
111        return manifest.short_path.replace("../", "${output_base}/external/", 1)
112
113    return "${build_workspace_directory}/" + manifest.short_path
114
115def _write_splicing_manifest(ctx):
116    # Manifests are required to be single files
117    manifests = {_prepare_manifest_path(m): str(m.label) for m in ctx.attr.manifests}
118
119    manifest = _write_data_file(
120        ctx = ctx,
121        name = "cargo-bazel-splicing-manifest.json",
122        data = generate_splicing_manifest(
123            packages = ctx.attr.packages,
124            splicing_config = ctx.attr.splicing_config,
125            cargo_config = ctx.attr.cargo_config,
126            manifests = manifests,
127            manifest_to_path = _prepare_manifest_path,
128        ),
129    )
130
131    is_windows = _is_windows(ctx)
132
133    args = ["--splicing-manifest", _runfiles_path(manifest, is_windows)]
134    runfiles = [manifest] + ctx.files.manifests + ([ctx.file.cargo_config] if ctx.attr.cargo_config else [])
135    return args, runfiles
136
137def generate_splicing_manifest(packages, splicing_config, cargo_config, manifests, manifest_to_path):
138    # Deserialize information about direct packages
139    direct_packages_info = {
140        # Ensure the data is using kebab-case as that's what `cargo_toml::DependencyDetail` expects.
141        pkg: kebab_case_keys(dict(json.decode(data)))
142        for (pkg, data) in packages.items()
143    }
144
145    config = json.decode(splicing_config or generate_splicing_config())
146    splicing_manifest_content = {
147        "cargo_config": str(manifest_to_path(cargo_config)) if cargo_config else None,
148        "direct_packages": direct_packages_info,
149        "manifests": manifests,
150    }
151
152    return json.encode_indent(
153        dict(dict(config).items() + splicing_manifest_content.items()),
154        indent = " " * 4,
155    )
156
157def _write_config_file(ctx):
158    workspace_name = ctx.workspace_name
159    if workspace_name == "__main__" or ctx.workspace_name == "_main":
160        workspace_name = ""
161
162    config = _write_data_file(
163        ctx = ctx,
164        name = "cargo-bazel-config.json",
165        data = generate_config_file(
166            ctx,
167            mode = ctx.attr.mode,
168            annotations = ctx.attr.annotations,
169            generate_binaries = ctx.attr.generate_binaries,
170            generate_build_scripts = ctx.attr.generate_build_scripts,
171            generate_target_compatible_with = ctx.attr.generate_target_compatible_with,
172            supported_platform_triples = ctx.attr.supported_platform_triples,
173            repository_name = ctx.attr.repository_name,
174            output_pkg = _get_output_package(ctx),
175            workspace_name = workspace_name,
176            render_config = dict(json.decode(ctx.attr.render_config)) if ctx.attr.render_config else None,
177        ),
178    )
179
180    is_windows = _is_windows(ctx)
181    args = ["--config", _runfiles_path(config, is_windows)]
182    runfiles = [config] + ctx.files.manifests
183    return args, runfiles
184
185def generate_config_file(
186        ctx,
187        mode,
188        annotations,
189        generate_binaries,
190        generate_build_scripts,
191        generate_target_compatible_with,
192        supported_platform_triples,
193        repository_name,
194        output_pkg,
195        workspace_name,
196        render_config,
197        repository_ctx = None):
198    """Writes the rendering config to cargo-bazel-config.json.
199
200    Args:
201        ctx: The rule's context.
202        mode (str): The vendoring mode.
203        annotations: Any annotations provided.
204        generate_binaries (bool): Whether to generate binaries for the crates.
205        generate_build_scripts (bool): Whether to generate BUILD.bazel files.
206        generate_target_compatible_with (bool): DEPRECATED: Moved to `render_config`.
207        supported_platform_triples (str): The platform triples to support in
208            the generated BUILD.bazel files.
209        repository_name (str): The name of the repository to generate.
210        output_pkg: The path to the package containing the build files.
211        workspace_name (str): The name of the workspace.
212        render_config: The render config to use.
213        repository_ctx (repository_ctx, optional): A repository context object
214            used for enabling certain functionality.
215
216    Returns:
217        file: The cargo-bazel-config.json written.
218    """
219    default_render_config = _default_render_config()
220    if render_config == None:
221        render_config = default_render_config
222
223    if mode == "local":
224        build_file_base_template = "//{}/{{name}}-{{version}}:BUILD.bazel".format(output_pkg)
225        if workspace_name != "":
226            build_file_base_template = "@{}//{}/{{name}}-{{version}}:BUILD.bazel".format(workspace_name, output_pkg)
227        crate_label_template = "//{}/{{name}}-{{version}}:{{target}}".format(
228            output_pkg,
229        )
230    else:
231        build_file_base_template = "//{}:BUILD.{{name}}-{{version}}.bazel".format(output_pkg)
232        if workspace_name != "":
233            build_file_base_template = "@{}//{}:BUILD.{{name}}-{{version}}.bazel".format(workspace_name, output_pkg)
234        crate_label_template = render_config["crate_label_template"]
235
236    # If `workspace_name` is blank (such as when using modules), the `@{}//{}:{{file}}` template would generate
237    # a reference like `Label(@//<stuff>)`. This causes issues if the module doing the `crates_vendor`ing is not the root module.
238    # See: https://github.com/bazelbuild/rules_rust/issues/2661
239    crates_module_template_value = "//{}:{{file}}".format(output_pkg)
240    if workspace_name != "":
241        crates_module_template_value = "@{}//{}:{{file}}".format(
242            workspace_name,
243            output_pkg,
244        )
245
246    updates = {
247        "build_file_template": build_file_base_template,
248        "crate_label_template": crate_label_template,
249        "crates_module_template": crates_module_template_value,
250        "vendor_mode": mode,
251    }
252
253    # "crate_label_template" is explicitly supported above in non-local modes
254    excluded_from_key_check = ["crate_label_template"]
255
256    for key in updates:
257        if (render_config[key] != default_render_config[key]) and key not in excluded_from_key_check:
258            fail("The `crates_vendor.render_config` attribute does not support the `{}` parameter. Please update {} to remove this value.".format(
259                key,
260                ctx.label,
261            ))
262
263    render_config.update(updates)
264
265    # Allow users to override the regen command.
266    if "regen_command" not in render_config or not render_config["regen_command"]:
267        render_config.update({"regen_command": "bazel run {}".format(ctx.label)})
268
269    config_data = compile_config(
270        crate_annotations = annotations,
271        generate_binaries = generate_binaries,
272        generate_build_scripts = generate_build_scripts,
273        generate_target_compatible_with = generate_target_compatible_with,
274        cargo_config = None,
275        render_config = render_config,
276        supported_platform_triples = supported_platform_triples,
277        repository_name = repository_name or ctx.label.name,
278        repository_ctx = repository_ctx,
279    )
280
281    return json.encode_indent(
282        config_data,
283        indent = " " * 4,
284    )
285
286def _crates_vendor_impl(ctx):
287    toolchain = ctx.toolchains[Label("@rules_rust//rust:toolchain_type")]
288    is_windows = _is_windows(ctx)
289
290    environ = {
291        "CARGO": _runfiles_path(toolchain.cargo, is_windows),
292        "RUSTC": _runfiles_path(toolchain.rustc, is_windows),
293    }
294
295    args = ["vendor"]
296
297    cargo_bazel_runfiles = []
298
299    # Allow action envs to override the use of the cargo-bazel target.
300    if CARGO_BAZEL_GENERATOR_PATH in ctx.var:
301        bin_path = ctx.var[CARGO_BAZEL_GENERATOR_PATH]
302    elif ctx.executable.cargo_bazel:
303        bin_path = _runfiles_path(ctx.executable.cargo_bazel, is_windows)
304        cargo_bazel_runfiles.append(ctx.executable.cargo_bazel)
305    else:
306        fail("{} is missing either the `cargo_bazel` attribute or the '{}' action env".format(
307            ctx.label,
308            CARGO_BAZEL_GENERATOR_PATH,
309        ))
310
311    # Generate config file
312    config_args, config_runfiles = _write_config_file(ctx)
313    args.extend(config_args)
314    cargo_bazel_runfiles.extend(config_runfiles)
315
316    # Generate splicing manifest
317    splicing_manifest_args, splicing_manifest_runfiles = _write_splicing_manifest(ctx)
318    args.extend(splicing_manifest_args)
319    cargo_bazel_runfiles.extend(splicing_manifest_runfiles)
320
321    # Add an optional `Cargo.lock` file.
322    if ctx.attr.cargo_lockfile:
323        args.extend([
324            "--cargo-lockfile",
325            _runfiles_path(ctx.file.cargo_lockfile, is_windows),
326        ])
327        cargo_bazel_runfiles.extend([ctx.file.cargo_lockfile])
328
329    # Optionally include buildifier
330    if ctx.attr.buildifier:
331        args.extend(["--buildifier", _runfiles_path(ctx.executable.buildifier, is_windows)])
332        cargo_bazel_runfiles.append(ctx.executable.buildifier)
333
334    # Optionally include an explicit `bazel` path
335    if ctx.attr.bazel:
336        args.extend(["--bazel", _runfiles_path(ctx.executable.bazel, is_windows)])
337        cargo_bazel_runfiles.append(ctx.executable.bazel)
338
339    # Determine platform specific settings
340    if is_windows:
341        extension = ".bat"
342        template = _WINDOWS_WRAPPER
343        env_template = "\nset {}={}"
344    else:
345        extension = ".sh"
346        template = _UNIX_WRAPPER
347        env_template = "{}={}"
348
349    # Write the wrapper script
350    runner = ctx.actions.declare_file(ctx.label.name + extension)
351    ctx.actions.write(
352        output = runner,
353        content = template.format(
354            env = " ".join([env_template.format(key, val) for key, val in environ.items()]),
355            bin = bin_path,
356            args = " ".join(args),
357        ),
358        is_executable = True,
359    )
360
361    return DefaultInfo(
362        files = depset([runner]),
363        runfiles = ctx.runfiles(
364            files = cargo_bazel_runfiles,
365            transitive_files = toolchain.all_files,
366        ),
367        executable = runner,
368    )
369
370CRATES_VENDOR_ATTRS = {
371    "annotations": attr.string_list_dict(
372        doc = "Extra settings to apply to crates. See [crate.annotation](#crateannotation).",
373    ),
374    "bazel": attr.label(
375        doc = "The path to a bazel binary used to locate the output_base for the current workspace.",
376        cfg = "exec",
377        executable = True,
378        allow_files = True,
379    ),
380    "buildifier": attr.label(
381        doc = "The path to a [buildifier](https://github.com/bazelbuild/buildtools/blob/5.0.1/buildifier/README.md) binary used to format generated BUILD files.",
382        cfg = "exec",
383        executable = True,
384        allow_files = True,
385        default = Label("//crate_universe/private/vendor:buildifier"),
386    ),
387    "cargo_bazel": attr.label(
388        doc = (
389            "The cargo-bazel binary to use for vendoring. If this attribute is not set, then a " +
390            "`{}` action env will be used.".format(CARGO_BAZEL_GENERATOR_PATH)
391        ),
392        cfg = "exec",
393        executable = True,
394        allow_files = True,
395        default = CARGO_BAZEL_LABEL,
396    ),
397    "cargo_config": attr.label(
398        doc = "A [Cargo configuration](https://doc.rust-lang.org/cargo/reference/config.html) file.",
399        allow_single_file = True,
400    ),
401    "cargo_lockfile": attr.label(
402        doc = "The path to an existing `Cargo.lock` file",
403        allow_single_file = True,
404    ),
405    "generate_binaries": attr.bool(
406        doc = (
407            "Whether to generate `rust_binary` targets for all the binary crates in every package. " +
408            "By default only the `rust_library` targets are generated."
409        ),
410        default = False,
411    ),
412    "generate_build_scripts": attr.bool(
413        doc = (
414            "Whether or not to generate " +
415            "[cargo build scripts](https://doc.rust-lang.org/cargo/reference/build-scripts.html) by default."
416        ),
417        default = True,
418    ),
419    "generate_target_compatible_with": attr.bool(
420        doc = "DEPRECATED: Moved to `render_config`.",
421        default = True,
422    ),
423    "manifests": attr.label_list(
424        doc = "A list of Cargo manifests (`Cargo.toml` files).",
425        allow_files = ["Cargo.toml"],
426    ),
427    "mode": attr.string(
428        doc = (
429            "Flags determining how crates should be vendored. `local` is where crate source and BUILD files are " +
430            "written to the repository. `remote` is where only BUILD files are written and repository rules " +
431            "used to fetch source code."
432        ),
433        values = [
434            "local",
435            "remote",
436        ],
437        default = "remote",
438    ),
439    "packages": attr.string_dict(
440        doc = "A set of crates (packages) specifications to depend on. See [crate.spec](#crate.spec).",
441    ),
442    "render_config": attr.string(
443        doc = (
444            "The configuration flags to use for rendering. Use `//crate_universe:defs.bzl\\%render_config` to " +
445            "generate the value for this field. If unset, the defaults defined there will be used."
446        ),
447    ),
448    "repository_name": attr.string(
449        doc = "The name of the repository to generate for `remote` vendor modes. If unset, the label name will be used",
450    ),
451    "splicing_config": attr.string(
452        doc = (
453            "The configuration flags to use for splicing Cargo maniests. Use `//crate_universe:defs.bzl\\%rsplicing_config` to " +
454            "generate the value for this field. If unset, the defaults defined there will be used."
455        ),
456    ),
457    "supported_platform_triples": attr.string_list(
458        doc = "A set of all platform triples to consider when generating dependencies.",
459        default = SUPPORTED_PLATFORM_TRIPLES,
460    ),
461    "vendor_path": attr.string(
462        doc = "The path to a directory to write files into. Absolute paths will be treated as relative to the workspace root",
463        default = "crates",
464    ),
465}
466
467crates_vendor = rule(
468    implementation = _crates_vendor_impl,
469    doc = """\
470A rule for defining Rust dependencies (crates) and writing targets for them to the current workspace.
471This rule is useful for users whose workspaces are expected to be consumed in other workspaces as the
472rendered `BUILD` files reduce the number of workspace dependencies, allowing for easier loads. This rule
473handles all the same [workflows](#workflows) `crate_universe` rules do.
474
475Example:
476
477Given the following workspace structure:
478
479```text
480[workspace]/
481    WORKSPACE
482    BUILD
483    Cargo.toml
484    3rdparty/
485        BUILD
486    src/
487        main.rs
488```
489
490The following is something that'd be found in `3rdparty/BUILD`:
491
492```python
493load("@rules_rust//crate_universe:defs.bzl", "crates_vendor", "crate")
494
495crates_vendor(
496    name = "crates_vendor",
497    annotations = {
498        "rand": [crate.annotation(
499            default_features = False,
500            features = ["small_rng"],
501        )],
502    },
503    cargo_lockfile = "//:Cargo.Bazel.lock",
504    manifests = ["//:Cargo.toml"],
505    mode = "remote",
506    vendor_path = "crates",
507    tags = ["manual"],
508)
509```
510
511The above creates a target that can be run to write `BUILD` files into the `3rdparty`
512directory next to where the target is defined. To run it, simply call:
513
514```shell
515bazel run //3rdparty:crates_vendor
516```
517
518<a id="#crates_vendor_repinning_updating_dependencies"></a>
519
520### Repinning / Updating Dependencies
521
522Repinning dependencies is controlled by both the `CARGO_BAZEL_REPIN` environment variable or the `--repin`
523flag to the `crates_vendor` binary. To update dependencies, simply add the flag ro your `bazel run` invocation.
524
525```shell
526bazel run //3rdparty:crates_vendor -- --repin
527```
528
529Under the hood, `--repin` will trigger a [cargo update](https://doc.rust-lang.org/cargo/commands/cargo-update.html)
530call against the generated workspace. The following table describes how to control particular values passed to the
531`cargo update` command.
532
533| Value | Cargo command |
534| --- | --- |
535| Any of [`true`, `1`, `yes`, `on`, `workspace`] | `cargo update --workspace` |
536| Any of [`full`, `eager`, `all`] | `cargo update` |
537| `package_name` | `cargo upgrade --package package_name` |
538| `package_name@1.2.3` | `cargo upgrade --package package_name --precise 1.2.3` |
539
540""",
541    attrs = CRATES_VENDOR_ATTRS,
542    executable = True,
543    toolchains = ["@rules_rust//rust:toolchain_type"],
544)
545
546def _crates_vendor_remote_repository_impl(repository_ctx):
547    build_file = repository_ctx.path(repository_ctx.attr.build_file)
548    defs_module = repository_ctx.path(repository_ctx.attr.defs_module)
549
550    repository_ctx.file("BUILD.bazel", repository_ctx.read(build_file))
551    repository_ctx.file("defs.bzl", repository_ctx.read(defs_module))
552    repository_ctx.file("crates.bzl", "")
553    repository_ctx.file("WORKSPACE.bazel", """workspace(name = "{}")""".format(
554        repository_ctx.name,
555    ))
556
557crates_vendor_remote_repository = repository_rule(
558    doc = "Creates a repository paired with `crates_vendor` targets using the `remote` vendor mode.",
559    implementation = _crates_vendor_remote_repository_impl,
560    attrs = {
561        "build_file": attr.label(
562            doc = "The BUILD file to use for the root package",
563            mandatory = True,
564        ),
565        "defs_module": attr.label(
566            doc = "The `defs.bzl` file to use in the repository",
567            mandatory = True,
568        ),
569    },
570)