fork/rules_rust

git clone https://git.julia.blue/fork/rules_rust

Daniel Wagner-HallExternal path dependencies work (#3025)7edda6f1

main
14.9 KiB342 linesraw
1"""`crates_repository` rule implementation"""
2
3load("//crate_universe/private:common_utils.bzl", "get_rust_tools")
4load(
5    "//crate_universe/private:generate_utils.bzl",
6    "CRATES_REPOSITORY_ENVIRON",
7    "determine_repin",
8    "execute_generator",
9    "generate_config",
10    "get_generator",
11    "get_lockfiles",
12)
13load(
14    "//crate_universe/private:splicing_utils.bzl",
15    "create_splicing_manifest",
16    "splice_workspace_manifest",
17)
18load("//crate_universe/private:urls.bzl", "CARGO_BAZEL_SHA256S", "CARGO_BAZEL_URLS")
19load("//rust:defs.bzl", "rust_common")
20load("//rust/platform:triple.bzl", "get_host_triple")
21load("//rust/platform:triple_mappings.bzl", "SUPPORTED_PLATFORM_TRIPLES")
22
23def _crates_repository_impl(repository_ctx):
24    # Determine the current host's platform triple
25    host_triple = get_host_triple(repository_ctx)
26
27    # Locate the generator to use
28    generator, generator_sha256 = get_generator(repository_ctx, host_triple.str)
29
30    # Generate a config file for all settings
31    config_path = generate_config(repository_ctx)
32
33    # Locate the lockfiles
34    lockfiles = get_lockfiles(repository_ctx)
35
36    # Locate Rust tools (cargo, rustc)
37    tools = get_rust_tools(repository_ctx, host_triple)
38    cargo_path = repository_ctx.path(tools.cargo)
39    rustc_path = repository_ctx.path(tools.rustc)
40
41    # Create a manifest of all dependency inputs
42    splicing_manifest = create_splicing_manifest(repository_ctx)
43
44    # Determine whether or not to repin depednencies
45    repin = determine_repin(
46        repository_ctx = repository_ctx,
47        generator = generator,
48        lockfile_path = lockfiles.bazel,
49        config = config_path,
50        splicing_manifest = splicing_manifest,
51        cargo = cargo_path,
52        rustc = rustc_path,
53        repin_instructions = repository_ctx.attr.repin_instructions,
54    )
55
56    # If re-pinning is enabled, gather additional inputs for the generator
57    kwargs = dict()
58    if repin:
59        # Generate a top level Cargo workspace and manifest for use in generation
60        metadata_path = splice_workspace_manifest(
61            repository_ctx = repository_ctx,
62            generator = generator,
63            cargo_lockfile = lockfiles.cargo,
64            splicing_manifest = splicing_manifest,
65            config_path = config_path,
66            cargo = cargo_path,
67            rustc = rustc_path,
68        )
69
70        kwargs.update({
71            "metadata": metadata_path,
72        })
73
74    paths_to_track_file = repository_ctx.path("paths-to-track")
75    warnings_output_file = repository_ctx.path("warnings-output-file")
76
77    # Run the generator
78    execute_generator(
79        repository_ctx = repository_ctx,
80        generator = generator,
81        config = config_path,
82        splicing_manifest = splicing_manifest,
83        lockfile_path = lockfiles.bazel,
84        cargo_lockfile_path = lockfiles.cargo,
85        repository_dir = repository_ctx.path("."),
86        cargo = cargo_path,
87        rustc = rustc_path,
88        paths_to_track_file = paths_to_track_file,
89        warnings_output_file = warnings_output_file,
90        # sysroot = tools.sysroot,
91        **kwargs
92    )
93
94    paths_to_track = json.decode(repository_ctx.read(paths_to_track_file))
95    for path in paths_to_track:
96        # This read triggers watching the file at this path and invalidates the repository_rule which will get re-run.
97        # Ideally we'd use repository_ctx.watch, but it doesn't support files outside of the workspace, and we need to support that.
98        repository_ctx.read(path)
99
100    warnings_output_file = json.decode(repository_ctx.read(warnings_output_file))
101    for warning in warnings_output_file:
102        # buildifier: disable=print
103        print("WARN: {}".format(warning))
104
105    # Determine the set of reproducible values
106    attrs = {attr: getattr(repository_ctx.attr, attr) for attr in dir(repository_ctx.attr)}
107    exclude = ["to_json", "to_proto"]
108    for attr in exclude:
109        attrs.pop(attr, None)
110
111    # Note that this is only scoped to the current host platform. Users should
112    # ensure they provide all the values necessary for the host environments
113    # they support
114    if generator_sha256:
115        attrs.update({"generator_sha256s": generator_sha256})
116
117    # Inform users that the repository rule can be made deterministic if they
118    # add a label to a lockfile path specifically for Bazel.
119    if not lockfiles.bazel:
120        attrs.update({"lockfile": repository_ctx.attr.cargo_lockfile.relative("cargo-bazel-lock.json")})
121
122    return attrs
123
124crates_repository = repository_rule(
125    doc = """\
126A rule for defining and downloading Rust dependencies (crates). This rule
127handles all the same [workflows](#workflows) `crate_universe` rules do.
128
129Environment Variables:
130
131| variable | usage |
132| --- | --- |
133| `CARGO_BAZEL_GENERATOR_SHA256` | The sha256 checksum of the file located at `CARGO_BAZEL_GENERATOR_URL` |
134| `CARGO_BAZEL_GENERATOR_URL` | The URL of a cargo-bazel binary. This variable takes precedence over attributes and can use `file://` for local paths |
135| `CARGO_BAZEL_ISOLATED` | An authorative flag as to whether or not the `CARGO_HOME` environment variable should be isolated from the host configuration |
136| `CARGO_BAZEL_REPIN` | An indicator that the dependencies represented by the rule should be regenerated. `REPIN` may also be used. See [Repinning / Updating Dependencies](#repinning--updating-dependencies) for more details. |
137| `CARGO_BAZEL_REPIN_ONLY` | A comma-delimited allowlist for rules to execute repinning. Can be useful if multiple instances of the repository rule are used in a Bazel workspace, but repinning should be limited to one of them. |
138
139Example:
140
141Given the following workspace structure:
142
143```text
144[workspace]/
145    WORKSPACE.bazel
146    BUILD.bazel
147    Cargo.toml
148    Cargo.Bazel.lock
149    src/
150        main.rs
151```
152
153The following is something that'd be found in the `WORKSPACE` file:
154
155```python
156load("@rules_rust//crate_universe:defs.bzl", "crates_repository", "crate")
157
158crates_repository(
159    name = "crate_index",
160    annotations = {
161        "rand": [crate.annotation(
162            default_features = False,
163            features = ["small_rng"],
164        )],
165    },
166    cargo_lockfile = "//:Cargo.Bazel.lock",
167    lockfile = "//:cargo-bazel-lock.json",
168    manifests = ["//:Cargo.toml"],
169    # Should match the version represented by the currently registered `rust_toolchain`.
170    rust_version = "1.60.0",
171)
172```
173
174The above will create an external repository which contains aliases and macros for accessing
175Rust targets found in the dependency graph defined by the given manifests.
176
177**NOTE**: The `cargo_lockfile` and `lockfile` must be manually created. The rule unfortunately does not yet create
178it on its own. When initially setting up this rule, an empty file should be created and then
179populated by repinning dependencies.
180
181### Repinning / Updating Dependencies
182
183Dependency syncing and updating is done in the repository rule which means it's done during the
184analysis phase of builds. As mentioned in the environments variable table above, the `CARGO_BAZEL_REPIN`
185(or `REPIN`) environment variables can be used to force the rule to update dependencies and potentially
186render a new lockfile. Given an instance of this repository rule named `crate_index`, the easiest way to
187repin dependencies is to run:
188
189```shell
190CARGO_BAZEL_REPIN=1 bazel sync --only=crate_index
191```
192
193This will result in all dependencies being updated for a project. The `CARGO_BAZEL_REPIN` environment variable
194can also be used to customize how dependencies are updated. The following table shows translations from environment
195variable values to the equivilant [cargo update](https://doc.rust-lang.org/cargo/commands/cargo-update.html) command
196that is called behind the scenes to update dependencies.
197
198| Value | Cargo command |
199| --- | --- |
200| Any of [`true`, `1`, `yes`, `on`, `workspace`] | `cargo update --workspace` |
201| Any of [`full`, `eager`, `all`] | `cargo update` |
202| `package_name` | `cargo upgrade --package package_name` |
203| `package_name@1.2.3` | `cargo upgrade --package package_name@1.2.3` |
204| `package_name@1.2.3=4.5.6` | `cargo upgrade --package package_name@1.2.3 --precise=4.5.6` |
205
206If the `crates_repository` is used multiple times in the same Bazel workspace (e.g. for multiple independent
207Rust workspaces), it may additionally be useful to use the `CARGO_BAZEL_REPIN_ONLY` environment variable, which
208limits execution of the repinning to one or multiple instances of the `crates_repository` rule via a comma-delimited
209allowlist:
210
211```shell
212CARGO_BAZEL_REPIN=1 CARGO_BAZEL_REPIN_ONLY=crate_index bazel sync --only=crate_index
213```
214
215""",
216    implementation = _crates_repository_impl,
217    attrs = {
218        "annotations": attr.string_list_dict(
219            doc = "Extra settings to apply to crates. See [crate.annotation](#crateannotation).",
220        ),
221        "cargo_config": attr.label(
222            doc = "A [Cargo configuration](https://doc.rust-lang.org/cargo/reference/config.html) file",
223        ),
224        "cargo_lockfile": attr.label(
225            doc = (
226                "The path used to store the `crates_repository` specific " +
227                "[Cargo.lock](https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html) file. " +
228                "In the case that your `crates_repository` corresponds directly with an existing " +
229                "`Cargo.toml` file which has a paired `Cargo.lock` file, that `Cargo.lock` file " +
230                "should be used here, which will keep the versions used by cargo and bazel in sync."
231            ),
232            mandatory = True,
233        ),
234        "compressed_windows_toolchain_names": attr.bool(
235            doc = "Wether or not the toolchain names of windows toolchains are expected to be in a `compressed` format.",
236            default = True,
237        ),
238        "generate_binaries": attr.bool(
239            doc = (
240                "Whether to generate `rust_binary` targets for all the binary crates in every package. " +
241                "By default only the `rust_library` targets are generated."
242            ),
243            default = False,
244        ),
245        "generate_build_scripts": attr.bool(
246            doc = (
247                "Whether or not to generate " +
248                "[cargo build scripts](https://doc.rust-lang.org/cargo/reference/build-scripts.html) by default."
249            ),
250            default = True,
251        ),
252        "generate_target_compatible_with": attr.bool(
253            doc = "DEPRECATED: Moved to `render_config`.",
254            default = True,
255        ),
256        "generator": attr.string(
257            doc = (
258                "The absolute label of a generator. Eg. `@cargo_bazel_bootstrap//:cargo-bazel`. " +
259                "This is typically used when bootstrapping"
260            ),
261        ),
262        "generator_sha256s": attr.string_dict(
263            doc = "Dictionary of `host_triple` -> `sha256` for a `cargo-bazel` binary.",
264            default = CARGO_BAZEL_SHA256S,
265        ),
266        "generator_urls": attr.string_dict(
267            doc = (
268                "URL template from which to download the `cargo-bazel` binary. `{host_triple}` and will be " +
269                "filled in according to the host platform."
270            ),
271            default = CARGO_BAZEL_URLS,
272        ),
273        "isolated": attr.bool(
274            doc = (
275                "If true, `CARGO_HOME` will be overwritten to a directory within the generated repository in " +
276                "order to prevent other uses of Cargo from impacting having any effect on the generated targets " +
277                "produced by this rule. For users who either have multiple `crate_repository` definitions in a " +
278                "WORKSPACE or rapidly re-pin dependencies, setting this to false may improve build times. This " +
279                "variable is also controled by `CARGO_BAZEL_ISOLATED` environment variable."
280            ),
281            default = True,
282        ),
283        "lockfile": attr.label(
284            doc = (
285                "The path to a file to use for reproducible renderings. " +
286                "If set, this file must exist within the workspace (but can be empty) before this rule will work."
287            ),
288        ),
289        "manifests": attr.label_list(
290            doc = "A list of Cargo manifests (`Cargo.toml` files).",
291        ),
292        "packages": attr.string_dict(
293            doc = "A set of crates (packages) specifications to depend on. See [crate.spec](#crate.spec).",
294        ),
295        "quiet": attr.bool(
296            doc = "If stdout and stderr should not be printed to the terminal.",
297            default = True,
298        ),
299        "render_config": attr.string(
300            doc = (
301                "The configuration flags to use for rendering. Use `//crate_universe:defs.bzl\\%render_config` to " +
302                "generate the value for this field. If unset, the defaults defined there will be used."
303            ),
304        ),
305        "repin_instructions": attr.string(
306            doc = "Instructions to re-pin the repository if required. Many people have wrapper scripts for keeping dependencies up to date, and would like to point users to that instead of the default.",
307        ),
308        "rust_toolchain_cargo_template": attr.string(
309            doc = (
310                "The template to use for finding the host `cargo` binary. `{version}` (eg. '1.53.0'), " +
311                "`{triple}` (eg. 'x86_64-unknown-linux-gnu'), `{arch}` (eg. 'aarch64'), `{vendor}` (eg. 'unknown'), " +
312                "`{system}` (eg. 'darwin'), `{cfg}` (eg. 'exec'), `{channel}` (eg. 'stable'), and `{tool}` (eg. " +
313                "'rustc.exe') will be replaced in the string if present."
314            ),
315            default = "@rust_{system}_{arch}__{triple}__{channel}_tools//:bin/{tool}",
316        ),
317        "rust_toolchain_rustc_template": attr.string(
318            doc = (
319                "The template to use for finding the host `rustc` binary. `{version}` (eg. '1.53.0'), " +
320                "`{triple}` (eg. 'x86_64-unknown-linux-gnu'), `{arch}` (eg. 'aarch64'), `{vendor}` (eg. 'unknown'), " +
321                "`{system}` (eg. 'darwin'), `{cfg}` (eg. 'exec'), `{channel}` (eg. 'stable'), and `{tool}` (eg. " +
322                "'cargo.exe') will be replaced in the string if present."
323            ),
324            default = "@rust_{system}_{arch}__{triple}__{channel}_tools//:bin/{tool}",
325        ),
326        "rust_version": attr.string(
327            doc = "The version of Rust the currently registered toolchain is using. Eg. `1.56.0`, or `nightly/2021-09-08`",
328            default = rust_common.default_version,
329        ),
330        "splicing_config": attr.string(
331            doc = (
332                "The configuration flags to use for splicing Cargo maniests. Use `//crate_universe:defs.bzl\\%rsplicing_config` to " +
333                "generate the value for this field. If unset, the defaults defined there will be used."
334            ),
335        ),
336        "supported_platform_triples": attr.string_list(
337            doc = "A set of all platform triples to consider when generating dependencies.",
338            default = SUPPORTED_PLATFORM_TRIPLES,
339        ),
340    },
341    environ = CRATES_REPOSITORY_ENVIRON,
342)