fork/rules_rust

git clone https://git.julia.blue/fork/rules_rust

Julia RyanAdd patching examplea4c87621

cleanup
21.1 KiB592 linesraw
1"""Rules for vendoring Bazel targets into existing workspaces"""
2
3load("//crate_universe/private:generate_utils.bzl", "compile_config", "render_config")
4load("//crate_universe/private:splicing_utils.bzl", "kebab_case_keys", generate_splicing_config = "splicing_config")
5load("//crate_universe/private:urls.bzl", "CARGO_BAZEL_LABEL")
6load("//rust/platform:triple_mappings.bzl", "SUPPORTED_PLATFORM_TRIPLES")
7
8_UNIX_WRAPPER = """\
9#!/usr/bin/env bash
10
11set -euo pipefail
12
13export RUNTIME_PWD="$(pwd)"
14if [[ -z "${{BAZEL_REAL:-}}" ]]; then
15    BAZEL_REAL="$(which bazel || echo 'bazel')"
16fi
17
18_ENVIRON=()
19_ENVIRON+=(BAZEL_REAL="${{BAZEL_REAL}}")
20_ENVIRON+=(BUILD_WORKSPACE_DIRECTORY="${{BUILD_WORKSPACE_DIRECTORY}}")
21_ENVIRON+=(PATH="${{PATH}}")
22
23if [[ -n "${{CARGO_BAZEL_DEBUG:-}}" ]]; then
24    _ENVIRON+=(CARGO_BAZEL_DEBUG="${{CARGO_BAZEL_DEBUG}}")
25fi
26
27# The path needs to be preserved to prevent bazel from starting with different
28# startup options (requiring a restart of bazel).
29# If you provide an empty path, bazel starts itself with
30# --default_system_javabase set to the empty string, but if you provide a path,
31# it may set it to a value (eg. "/usr/local/buildtools/java/jdk11").
32exec env - \\
33${{_ENVIRON[@]}} \\
34{env} \\
35    "{bin}" \\
36    {args} \\
37    "$@"
38"""
39
40_WINDOWS_WRAPPER = """\
41@ECHO OFF
42set RUNTIME_PWD=%CD%
43{env}
44
45{bin} {args} %*
46exit %ERRORLEVEL%
47"""
48
49CARGO_BAZEL_GENERATOR_PATH = "CARGO_BAZEL_GENERATOR_PATH"
50
51def _default_render_config():
52    return json.decode(render_config())
53
54def _runfiles_path(file, is_windows):
55    if is_windows:
56        runtime_pwd_var = "%RUNTIME_PWD%"
57    else:
58        runtime_pwd_var = "${RUNTIME_PWD}"
59
60    return "{}/{}".format(runtime_pwd_var, file.short_path)
61
62def _is_windows(ctx):
63    toolchain = ctx.toolchains[Label("@rules_rust//rust:toolchain_type")]
64    return toolchain.target_os == "windows"
65
66def _get_output_package(ctx):
67    # Determine output directory
68    if ctx.attr.vendor_path.startswith("/"):
69        output = ctx.attr.vendor_path
70    else:
71        output = "{}/{}".format(
72            ctx.label.package,
73            ctx.attr.vendor_path,
74        )
75    return output.lstrip("/")
76
77def _write_data_file(ctx, name, data):
78    file = ctx.actions.declare_file("{}.{}".format(ctx.label.name, name))
79    ctx.actions.write(
80        output = file,
81        content = data,
82    )
83    return file
84
85def _prepare_manifest_path(target):
86    """Generate manifest paths that are resolvable by `cargo_bazel::SplicingManifest::resolve`
87
88    Args:
89        target (Target): A `crate_vendor.manifest` target
90
91    Returns:
92        str: A string representing the path to a manifest.
93    """
94    files = target[DefaultInfo].files.to_list()
95    if len(files) != 1:
96        fail("The manifest {} hand an unexpected number of files: {}".format(
97            target.label,
98            files,
99        ))
100
101    manifest = files[0]
102
103    if target.label.workspace_root.startswith("external"):
104        # The short path of an external file is expected to start with `../`
105        if not manifest.short_path.startswith("../"):
106            fail("Unexpected shortpath for {}: {}".format(
107                manifest,
108                manifest.short_path,
109            ))
110        return manifest.short_path.replace("../", "${output_base}/external/", 1)
111
112    return "${build_workspace_directory}/" + manifest.short_path
113
114def _write_splicing_manifest(ctx):
115    # Manifests are required to be single files
116    manifests = {_prepare_manifest_path(m): str(m.label) for m in ctx.attr.manifests}
117
118    manifest = _write_data_file(
119        ctx = ctx,
120        name = "cargo-bazel-splicing-manifest.json",
121        data = generate_splicing_manifest(
122            packages = ctx.attr.packages,
123            splicing_config = ctx.attr.splicing_config,
124            cargo_config = ctx.attr.cargo_config,
125            manifests = manifests,
126            manifest_to_path = _prepare_manifest_path,
127        ),
128    )
129
130    is_windows = _is_windows(ctx)
131
132    args = ["--splicing-manifest", _runfiles_path(manifest, is_windows)]
133    runfiles = [manifest] + ctx.files.manifests + ([ctx.file.cargo_config] if ctx.attr.cargo_config else [])
134    return args, runfiles
135
136def generate_splicing_manifest(packages, splicing_config, cargo_config, manifests, manifest_to_path):
137    # Deserialize information about direct packages
138    direct_packages_info = {
139        # Ensure the data is using kebab-case as that's what `cargo_toml::DependencyDetail` expects.
140        pkg: kebab_case_keys(dict(json.decode(data)))
141        for (pkg, data) in packages.items()
142    }
143
144    config = json.decode(splicing_config or generate_splicing_config())
145    splicing_manifest_content = {
146        "cargo_config": str(manifest_to_path(cargo_config)) if cargo_config else None,
147        "direct_packages": direct_packages_info,
148        "manifests": manifests,
149    }
150
151    return json.encode_indent(
152        dict(dict(config).items() + splicing_manifest_content.items()),
153        indent = " " * 4,
154    )
155
156def _write_config_file(ctx):
157    workspace_name = ctx.workspace_name
158    if workspace_name == "__main__" or ctx.workspace_name == "_main":
159        workspace_name = ""
160
161    config = _write_data_file(
162        ctx = ctx,
163        name = "cargo-bazel-config.json",
164        data = generate_config_file(
165            ctx,
166            mode = ctx.attr.mode,
167            annotations = ctx.attr.annotations,
168            generate_binaries = ctx.attr.generate_binaries,
169            generate_build_scripts = ctx.attr.generate_build_scripts,
170            generate_target_compatible_with = ctx.attr.generate_target_compatible_with,
171            supported_platform_triples = ctx.attr.supported_platform_triples,
172            repository_name = ctx.attr.repository_name,
173            output_pkg = _get_output_package(ctx),
174            workspace_name = workspace_name,
175            render_config = dict(json.decode(ctx.attr.render_config)) if ctx.attr.render_config else None,
176        ),
177    )
178
179    is_windows = _is_windows(ctx)
180    args = ["--config", _runfiles_path(config, is_windows)]
181    runfiles = [config] + ctx.files.manifests
182    return args, runfiles
183
184def generate_config_file(
185        ctx,
186        mode,
187        annotations,
188        generate_binaries,
189        generate_build_scripts,
190        generate_target_compatible_with,
191        supported_platform_triples,
192        repository_name,
193        output_pkg,
194        workspace_name,
195        render_config,
196        repository_ctx = None):
197    """Writes the rendering config to cargo-bazel-config.json.
198
199    Args:
200        ctx: The rule's context.
201        mode (str): The vendoring mode.
202        annotations: Any annotations provided.
203        generate_binaries (bool): Whether to generate binaries for the crates.
204        generate_build_scripts (bool): Whether to generate BUILD.bazel files.
205        generate_target_compatible_with (bool): DEPRECATED: Moved to `render_config`.
206        supported_platform_triples (str): The platform triples to support in
207            the generated BUILD.bazel files.
208        repository_name (str): The name of the repository to generate.
209        output_pkg: The path to the package containing the build files.
210        workspace_name (str): The name of the workspace.
211        render_config: The render config to use.
212        repository_ctx (repository_ctx, optional): A repository context object
213            used for enabling certain functionality.
214
215    Returns:
216        file: The cargo-bazel-config.json written.
217    """
218    default_render_config = _default_render_config()
219    if render_config == None:
220        render_config = default_render_config
221
222    if mode == "local":
223        build_file_base_template = "//{}/{{name}}-{{version}}:BUILD.bazel".format(output_pkg)
224        if workspace_name != "":
225            build_file_base_template = "@{}//{}/{{name}}-{{version}}:BUILD.bazel".format(workspace_name, output_pkg)
226        crate_label_template = "//{}/{{name}}-{{version}}:{{target}}".format(
227            output_pkg,
228        )
229    else:
230        build_file_base_template = "//{}:BUILD.{{name}}-{{version}}.bazel".format(output_pkg)
231        if workspace_name != "":
232            build_file_base_template = "@{}//{}:BUILD.{{name}}-{{version}}.bazel".format(workspace_name, output_pkg)
233        crate_label_template = render_config["crate_label_template"]
234
235    # If `workspace_name` is blank (such as when using modules), the `@{}//{}:{{file}}` template would generate
236    # a reference like `Label(@//<stuff>)`. This causes issues if the module doing the `crates_vendor`ing is not the root module.
237    # See: https://github.com/bazelbuild/rules_rust/issues/2661
238    crates_module_template_value = "//{}:{{file}}".format(output_pkg)
239    if workspace_name != "":
240        crates_module_template_value = "@{}//{}:{{file}}".format(
241            workspace_name,
242            output_pkg,
243        )
244
245    updates = {
246        "build_file_template": build_file_base_template,
247        "crate_label_template": crate_label_template,
248        "crates_module_template": crates_module_template_value,
249        "vendor_mode": mode,
250    }
251
252    # "crate_label_template" is explicitly supported above in non-local modes
253    excluded_from_key_check = ["crate_label_template"]
254
255    for key in updates:
256        if (render_config[key] != default_render_config[key]) and key not in excluded_from_key_check:
257            fail("The `crates_vendor.render_config` attribute does not support the `{}` parameter. Please update {} to remove this value.".format(
258                key,
259                ctx.label,
260            ))
261
262    render_config.update(updates)
263
264    # Allow users to override the regen command.
265    if "regen_command" not in render_config or not render_config["regen_command"]:
266        render_config.update({"regen_command": "bazel run {}".format(ctx.label)})
267
268    config_data = compile_config(
269        crate_annotations = annotations,
270        generate_binaries = generate_binaries,
271        generate_build_scripts = generate_build_scripts,
272        generate_target_compatible_with = generate_target_compatible_with,
273        cargo_config = None,
274        render_config = render_config,
275        supported_platform_triples = supported_platform_triples,
276        repository_name = repository_name or ctx.label.name,
277        repository_ctx = repository_ctx,
278    )
279
280    return json.encode_indent(
281        config_data,
282        indent = " " * 4,
283    )
284
285def _crates_vendor_impl(ctx):
286    toolchain = ctx.toolchains[Label("@rules_rust//rust:toolchain_type")]
287    is_windows = _is_windows(ctx)
288
289    environ = {
290        "CARGO": _runfiles_path(toolchain.cargo, is_windows),
291        "RUSTC": _runfiles_path(toolchain.rustc, is_windows),
292    }
293
294    args = ["vendor"]
295
296    cargo_bazel_runfiles = []
297
298    # Allow action envs to override the use of the cargo-bazel target.
299    if CARGO_BAZEL_GENERATOR_PATH in ctx.var:
300        bin_path = ctx.var[CARGO_BAZEL_GENERATOR_PATH]
301    elif ctx.executable.cargo_bazel:
302        bin_path = _runfiles_path(ctx.executable.cargo_bazel, is_windows)
303        cargo_bazel_runfiles.append(ctx.executable.cargo_bazel)
304    else:
305        fail("{} is missing either the `cargo_bazel` attribute or the '{}' action env".format(
306            ctx.label,
307            CARGO_BAZEL_GENERATOR_PATH,
308        ))
309
310    # Generate config file
311    config_args, config_runfiles = _write_config_file(ctx)
312    args.extend(config_args)
313    cargo_bazel_runfiles.extend(config_runfiles)
314
315    # Generate splicing manifest
316    splicing_manifest_args, splicing_manifest_runfiles = _write_splicing_manifest(ctx)
317    args.extend(splicing_manifest_args)
318    cargo_bazel_runfiles.extend(splicing_manifest_runfiles)
319
320    # Add an optional `Cargo.lock` file.
321    if ctx.attr.cargo_lockfile:
322        args.extend([
323            "--cargo-lockfile",
324            _runfiles_path(ctx.file.cargo_lockfile, is_windows),
325        ])
326        cargo_bazel_runfiles.extend([ctx.file.cargo_lockfile])
327
328    # Optionally include buildifier
329    if ctx.attr.buildifier:
330        args.extend(["--buildifier", _runfiles_path(ctx.executable.buildifier, is_windows)])
331        cargo_bazel_runfiles.append(ctx.executable.buildifier)
332
333    # Optionally include an explicit `bazel` path
334    if ctx.attr.bazel:
335        args.extend(["--bazel", _runfiles_path(ctx.executable.bazel, is_windows)])
336        cargo_bazel_runfiles.append(ctx.executable.bazel)
337
338    # Determine platform specific settings
339    if is_windows:
340        extension = ".bat"
341        template = _WINDOWS_WRAPPER
342        env_template = "\nset {}={}"
343    else:
344        extension = ".sh"
345        template = _UNIX_WRAPPER
346        env_template = "{}={}"
347
348    # Write the wrapper script
349    runner = ctx.actions.declare_file(ctx.label.name + extension)
350    ctx.actions.write(
351        output = runner,
352        content = template.format(
353            env = " ".join([env_template.format(key, val) for key, val in environ.items()]),
354            bin = bin_path,
355            args = " ".join(args),
356        ),
357        is_executable = True,
358    )
359
360    return DefaultInfo(
361        files = depset([runner]),
362        runfiles = ctx.runfiles(
363            files = cargo_bazel_runfiles,
364            transitive_files = toolchain.all_files,
365        ),
366        executable = runner,
367    )
368
369CRATES_VENDOR_ATTRS = {
370    "annotations": attr.string_list_dict(
371        doc = "Extra settings to apply to crates. See [crate.annotation](#crateannotation).",
372    ),
373    "bazel": attr.label(
374        doc = "The path to a bazel binary used to locate the output_base for the current workspace.",
375        cfg = "exec",
376        executable = True,
377        allow_files = True,
378    ),
379    "buildifier": attr.label(
380        doc = "The path to a [buildifier](https://github.com/bazelbuild/buildtools/blob/5.0.1/buildifier/README.md) binary used to format generated BUILD files.",
381        cfg = "exec",
382        executable = True,
383        allow_files = True,
384        default = Label("//crate_universe/private/vendor:buildifier"),
385    ),
386    "cargo_bazel": attr.label(
387        doc = (
388            "The cargo-bazel binary to use for vendoring. If this attribute is not set, then a " +
389            "`{}` action env will be used.".format(CARGO_BAZEL_GENERATOR_PATH)
390        ),
391        cfg = "exec",
392        executable = True,
393        allow_files = True,
394        default = CARGO_BAZEL_LABEL,
395    ),
396    "cargo_config": attr.label(
397        doc = "A [Cargo configuration](https://doc.rust-lang.org/cargo/reference/config.html) file.",
398        allow_single_file = True,
399    ),
400    "cargo_lockfile": attr.label(
401        doc = "The path to an existing `Cargo.lock` file",
402        allow_single_file = True,
403    ),
404    "generate_binaries": attr.bool(
405        doc = (
406            "Whether to generate `rust_binary` targets for all the binary crates in every package. " +
407            "By default only the `rust_library` targets are generated."
408        ),
409        default = False,
410    ),
411    "generate_build_scripts": attr.bool(
412        doc = (
413            "Whether or not to generate " +
414            "[cargo build scripts](https://doc.rust-lang.org/cargo/reference/build-scripts.html) by default."
415        ),
416        default = True,
417    ),
418    "generate_target_compatible_with": attr.bool(
419        doc = "DEPRECATED: Moved to `render_config`.",
420        default = True,
421    ),
422    "manifests": attr.label_list(
423        doc = "A list of Cargo manifests (`Cargo.toml` files).",
424        allow_files = ["Cargo.toml"],
425    ),
426    "mode": attr.string(
427        doc = (
428            "Flags determining how crates should be vendored. `local` is where crate source and BUILD files are " +
429            "written to the repository. `remote` is where only BUILD files are written and repository rules " +
430            "used to fetch source code."
431        ),
432        values = [
433            "local",
434            "remote",
435        ],
436        default = "remote",
437    ),
438    "packages": attr.string_dict(
439        doc = "A set of crates (packages) specifications to depend on. See [crate.spec](#crate.spec).",
440    ),
441    "render_config": attr.string(
442        doc = (
443            "The configuration flags to use for rendering. Use `//crate_universe:defs.bzl\\%render_config` to " +
444            "generate the value for this field. If unset, the defaults defined there will be used."
445        ),
446    ),
447    "repository_name": attr.string(
448        doc = "The name of the repository to generate for `remote` vendor modes. If unset, the label name will be used",
449    ),
450    "splicing_config": attr.string(
451        doc = (
452            "The configuration flags to use for splicing Cargo maniests. Use `//crate_universe:defs.bzl\\%rsplicing_config` to " +
453            "generate the value for this field. If unset, the defaults defined there will be used."
454        ),
455    ),
456    "supported_platform_triples": attr.string_list(
457        doc = "A set of all platform triples to consider when generating dependencies.",
458        default = SUPPORTED_PLATFORM_TRIPLES,
459    ),
460    "vendor_path": attr.string(
461        doc = "The path to a directory to write files into. Absolute paths will be treated as relative to the workspace root",
462        default = "crates",
463    ),
464}
465
466crates_vendor = rule(
467    implementation = _crates_vendor_impl,
468    doc = """\
469A rule for defining Rust dependencies (crates) and writing targets for them to the current workspace.
470This rule is useful for users whose workspaces are expected to be consumed in other workspaces as the
471rendered `BUILD` files reduce the number of workspace dependencies, allowing for easier loads. This rule
472handles all the same [workflows](#workflows) `crate_universe` rules do.
473
474Example:
475
476Given the following workspace structure:
477
478```text
479[workspace]/
480    WORKSPACE
481    BUILD
482    Cargo.toml
483    3rdparty/
484        BUILD
485    src/
486        main.rs
487```
488
489The following is something that'd be found in `3rdparty/BUILD`:
490
491```python
492load("@rules_rust//crate_universe:defs.bzl", "crates_vendor", "crate")
493
494crates_vendor(
495    name = "crates_vendor",
496    annotations = {
497        "rand": [crate.annotation(
498            default_features = False,
499            features = ["small_rng"],
500        )],
501    },
502    cargo_lockfile = "//:Cargo.Bazel.lock",
503    manifests = ["//:Cargo.toml"],
504    mode = "remote",
505    vendor_path = "crates",
506    tags = ["manual"],
507)
508```
509
510The above creates a target that can be run to write `BUILD` files into the `3rdparty`
511directory next to where the target is defined. To run it, simply call:
512
513```shell
514bazel run //3rdparty:crates_vendor
515```
516
517### Patching
518
519If you need to replace crates in your third_party dependency tree with forks, the `patch` section can be used in your cargo manifest:
520
521```toml
522[patch.crates-io]
523fuse3 = { path = "forks/fuse3" }
524```
525
526A build file for the patched crate will still be generated under the vendor path
527(such as `crates/fuse3-0.6.1/BUILD.bazel`), but `cargo vendor` will NOT download the
528crate to that location, and the generated BUILD file will point to your copy. For this
529to work, you need to define the following filegroups in your patch crate's BUILD.bazel:
530`:srcs`, `:crate_root`, `:compile_data`, and (if the crate has a build script)
531`:build_script_crate_root`. See the
532[patching example](https://github.com/bazelbuild/rules_rust/tree/main/examples/crate_universe/vendor_local_patching)
533for an example of what those BUILD files look like.
534
535This feature can also be used to avoid vendoring source code for crates that you never
536actually use for the platforms you're targeting. In that case you don't need an extra
537BUILD file, just crate a empty/stub Cargo.toml and `lib.rs`, and list the path to that
538in your `patch` section.
539
540<a id="#crates_vendor_repinning_updating_dependencies"></a>
541
542### Repinning / Updating Dependencies
543
544Repinning dependencies is controlled by both the `CARGO_BAZEL_REPIN` environment variable or the `--repin`
545flag to the `crates_vendor` binary. To update dependencies, simply add the flag to your `bazel run` invocation.
546
547```shell
548bazel run //3rdparty:crates_vendor -- --repin
549```
550
551Under the hood, `--repin` will trigger a [cargo update](https://doc.rust-lang.org/cargo/commands/cargo-update.html)
552call against the generated workspace. The following table describes how to control particular values passed to the
553`cargo update` command.
554
555| Value | Cargo command |
556| --- | --- |
557| Any of [`true`, `1`, `yes`, `on`, `workspace`] | `cargo update --workspace` |
558| Any of [`full`, `eager`, `all`] | `cargo update` |
559| `package_name` | `cargo upgrade --package package_name` |
560| `package_name@1.2.3` | `cargo upgrade --package package_name --precise 1.2.3` |
561
562""",
563    attrs = CRATES_VENDOR_ATTRS,
564    executable = True,
565    toolchains = ["@rules_rust//rust:toolchain_type"],
566)
567
568def _crates_vendor_remote_repository_impl(repository_ctx):
569    build_file = repository_ctx.path(repository_ctx.attr.build_file)
570    defs_module = repository_ctx.path(repository_ctx.attr.defs_module)
571
572    repository_ctx.file("BUILD.bazel", repository_ctx.read(build_file))
573    repository_ctx.file("defs.bzl", repository_ctx.read(defs_module))
574    repository_ctx.file("crates.bzl", "")
575    repository_ctx.file("WORKSPACE.bazel", """workspace(name = "{}")""".format(
576        repository_ctx.name,
577    ))
578
579crates_vendor_remote_repository = repository_rule(
580    doc = "Creates a repository paired with `crates_vendor` targets using the `remote` vendor mode.",
581    implementation = _crates_vendor_remote_repository_impl,
582    attrs = {
583        "build_file": attr.label(
584            doc = "The BUILD file to use for the root package",
585            mandatory = True,
586        ),
587        "defs_module": attr.label(
588            doc = "The `defs.bzl` file to use in the repository",
589            mandatory = True,
590        ),
591    },
592)