fork/rules_rust

git clone https://git.julia.blue/fork/rules_rust

UebelAndreFixed crate_universe lockfile checks for crates_repository rule (#1468)82a437cc

0.8.1
16.5 KiB453 linesraw
1"""Utilities directly related to the `generate` step of `cargo-bazel`."""
2
3load(":common_utils.bzl", "CARGO_BAZEL_ISOLATED", "REPIN_ENV_VARS", "cargo_environ", "execute")
4
5CARGO_BAZEL_GENERATOR_SHA256 = "CARGO_BAZEL_GENERATOR_SHA256"
6CARGO_BAZEL_GENERATOR_URL = "CARGO_BAZEL_GENERATOR_URL"
7
8GENERATOR_ENV_VARS = [
9    CARGO_BAZEL_GENERATOR_URL,
10    CARGO_BAZEL_GENERATOR_SHA256,
11]
12
13CRATES_REPOSITORY_ENVIRON = GENERATOR_ENV_VARS + REPIN_ENV_VARS + [
14    CARGO_BAZEL_ISOLATED,
15]
16
17def get_generator(repository_ctx, host_triple):
18    """Query network resources to locate a `cargo-bazel` binary
19
20    Args:
21        repository_ctx (repository_ctx): The rule's context object.
22        host_triple (string): A string representing the host triple
23
24    Returns:
25        tuple(path, dict): The path to a `cargo-bazel` binary and the host sha256 pairing.
26            The pairing (dict) may be `None` if there is no need to update the attribute
27    """
28    use_environ = False
29    for var in GENERATOR_ENV_VARS:
30        if var in repository_ctx.os.environ:
31            use_environ = True
32
33    output = repository_ctx.path("cargo-bazel.exe" if "win" in repository_ctx.os.name else "cargo-bazel")
34
35    # The `generator` attribute is the next highest priority behind
36    # environment variables. We check those first before deciding to
37    # use an explicitly provided variable.
38    if not use_environ and repository_ctx.attr.generator:
39        generator = repository_ctx.path(Label(repository_ctx.attr.generator))
40
41        # Resolve a few levels of symlinks to ensure we're accessing the direct binary
42        for _ in range(1, 100):
43            real_generator = generator.realpath
44            if real_generator == generator:
45                break
46            generator = real_generator
47        return generator, None
48
49    # The environment variable will take precedence if set
50    if use_environ:
51        generator_sha256 = repository_ctx.os.environ.get(CARGO_BAZEL_GENERATOR_SHA256)
52        generator_url = repository_ctx.os.environ.get(CARGO_BAZEL_GENERATOR_URL)
53    else:
54        generator_sha256 = repository_ctx.attr.generator_sha256s.get(host_triple)
55        generator_url = repository_ctx.attr.generator_urls.get(host_triple)
56
57    if not generator_url:
58        fail((
59            "No generator URL was found either in the `CARGO_BAZEL_GENERATOR_URL` " +
60            "environment variable or for the `{}` triple in the `generator_urls` attribute"
61        ).format(host_triple))
62
63    # Download the file into place
64    if generator_sha256:
65        repository_ctx.download(
66            output = output,
67            url = generator_url,
68            sha256 = generator_sha256,
69            executable = True,
70        )
71        return output, None
72
73    result = repository_ctx.download(
74        output = output,
75        url = generator_url,
76        executable = True,
77    )
78
79    return output, {host_triple: result.sha256}
80
81def render_config(
82        build_file_template = "//:BUILD.{name}-{version}.bazel",
83        crate_label_template = "@{repository}__{name}-{version}//:{target}",
84        crate_repository_template = "{repository}__{name}-{version}",
85        crates_module_template = "//:{file}",
86        default_package_name = None,
87        platforms_template = "@rules_rust//rust/platform:{triple}",
88        regen_command = None,
89        vendor_mode = None):
90    """Various settings used to configure rendered outputs
91
92    The template parameters each support a select number of format keys. A description of each key
93    can be found below where the supported keys for each template can be found in the parameter docs
94
95    | key | definition |
96    | --- | --- |
97    | `name` | The name of the crate. Eg `tokio` |
98    | `repository` | The rendered repository name for the crate. Directly relates to `crate_repository_template`. |
99    | `triple` | A platform triple. Eg `x86_64-unknown-linux-gnu` |
100    | `version` | The crate version. Eg `1.2.3` |
101    | `target` | The library or binary target of the crate |
102    | `file` | The basename of a file |
103
104    Args:
105        build_file_template (str, optional): The base template to use for BUILD file names. The available format keys
106            are [`{name}`, {version}`].
107        crate_label_template (str, optional): The base template to use for crate labels. The available format keys
108            are [`{repository}`, `{name}`, `{version}`, `{target}`].
109        crate_repository_template (str, optional): The base template to use for Crate label repository names. The
110            available format keys are [`{repository}`, `{name}`, `{version}`].
111        crates_module_template (str, optional): The pattern to use for the `defs.bzl` and `BUILD.bazel`
112            file names used for the crates module. The available format keys are [`{file}`].
113        default_package_name (str, optional): The default package name to use in the rendered macros. This affects the
114            auto package detection of things like `all_crate_deps`.
115        platforms_template (str, optional): The base template to use for platform names.
116            See [platforms documentation](https://docs.bazel.build/versions/main/platforms.html). The available format
117            keys are [`{triple}`].
118        regen_command (str, optional): An optional command to demonstrate how generated files should be regenerated.
119        vendor_mode (str, optional): An optional configuration for rendirng content to be rendered into repositories.
120
121    Returns:
122        string: A json encoded struct to match the Rust `config::RenderConfig` struct
123    """
124    return json.encode(struct(
125        build_file_template = build_file_template,
126        crate_label_template = crate_label_template,
127        crate_repository_template = crate_repository_template,
128        crates_module_template = crates_module_template,
129        default_package_name = default_package_name,
130        platforms_template = platforms_template,
131        regen_command = regen_command,
132        vendor_mode = vendor_mode,
133    ))
134
135def _crate_id(name, version):
136    """Creates a `cargo_bazel::config::CrateId`.
137
138    Args:
139        name (str): The name of the crate
140        version (str): The crate's version
141
142    Returns:
143        str: A serialized representation of a CrateId
144    """
145    return "{} {}".format(name, version)
146
147def collect_crate_annotations(annotations, repository_name):
148    """Deserialize and sanitize crate annotations.
149
150    Args:
151        annotations (dict): A mapping of crate names to lists of serialized annotations
152        repository_name (str): The name of the repository that owns the annotations
153
154    Returns:
155        dict: A mapping of `cargo_bazel::config::CrateId` to sets of annotations
156    """
157    annotations = {name: [json.decode(a) for a in annotation] for name, annotation in annotations.items()}
158    crate_annotations = {}
159    for name, annotation in annotations.items():
160        for (version, data) in annotation:
161            if name == "*" and version != "*":
162                fail(
163                    "Wildcard crate names must have wildcard crate versions. " +
164                    "Please update the `annotations` attribute of the {} crates_repository".format(
165                        repository_name,
166                    ),
167                )
168            id = _crate_id(name, version)
169            if id in crate_annotations:
170                fail("Found duplicate entries for {}".format(id))
171
172            crate_annotations.update({id: data})
173    return crate_annotations
174
175def _read_cargo_config(repository_ctx):
176    if repository_ctx.attr.cargo_config:
177        config = repository_ctx.path(repository_ctx.attr.cargo_config)
178        return repository_ctx.read(config)
179    return None
180
181def _update_render_config(config, repository_name):
182    """Add the repository name to the render config
183
184    Args:
185        config (dict): A `render_config` struct
186        repository_name (str): The name of the repository that owns the config
187
188    Returns:
189        struct: An updated `render_config`.
190    """
191
192    # Add the repository name as it's very relevant to rendering.
193    config.update({"repository_name": repository_name})
194
195    return struct(**config)
196
197def _get_render_config(repository_ctx):
198    if repository_ctx.attr.render_config:
199        config = dict(json.decode(repository_ctx.attr.render_config))
200    else:
201        config = dict(json.decode(render_config()))
202
203    if not config.get("regen_command"):
204        config["regen_command"] = "bazel sync --only={}".format(
205            repository_ctx.name,
206        )
207
208    return config
209
210def compile_config(crate_annotations, generate_build_scripts, cargo_config, render_config, supported_platform_triples, repository_name, repository_ctx = None):
211    """Create a config file for generating crate targets
212
213    [cargo_config]: https://doc.rust-lang.org/cargo/reference/config.html
214
215    Args:
216        crate_annotations (dict): Extra settings to apply to crates. See
217            `crates_repository.annotations` or `crates_vendor.annotations`.
218        generate_build_scripts (bool): Whether or not to globally disable build scripts.
219        cargo_config (str): The optional contents of a [Cargo config][cargo_config].
220        render_config (dict): The deserialized dict of the `render_config` function.
221        supported_platform_triples (list): A list of platform triples
222        repository_name (str): The name of the repository being generated
223        repository_ctx (repository_ctx, optional): A repository context object used for enabling
224            certain functionality.
225
226    Returns:
227        struct: A struct matching a `cargo_bazel::config::Config`.
228    """
229    annotations = collect_crate_annotations(crate_annotations, repository_name)
230
231    # Load additive build files if any have been provided.
232    unexpected = []
233    for name, data in annotations.items():
234        f = data.pop("additive_build_file", None)
235        if f and not repository_ctx:
236            unexpected.append(name)
237            f = None
238        content = [x for x in [
239            data.pop("additive_build_file_content", None),
240            repository_ctx.read(Label(f)) if f else None,
241        ] if x]
242        if content:
243            data.update({"additive_build_file_content": "\n".join(content)})
244
245    if unexpected:
246        fail("The following annotations use `additive_build_file` which is not supported for {}: {}".format(repository_name, unexpected))
247
248    config = struct(
249        generate_build_scripts = generate_build_scripts,
250        annotations = annotations,
251        cargo_config = cargo_config,
252        rendering = _update_render_config(
253            config = render_config,
254            repository_name = repository_name,
255        ),
256        supported_platform_triples = supported_platform_triples,
257    )
258
259    return config
260
261def generate_config(repository_ctx):
262    """Generate a config file from various attributes passed to the rule.
263
264    Args:
265        repository_ctx (repository_ctx): The rule's context object.
266
267    Returns:
268        struct: A struct containing the path to a config and it's contents
269    """
270
271    config = compile_config(
272        crate_annotations = repository_ctx.attr.annotations,
273        generate_build_scripts = repository_ctx.attr.generate_build_scripts,
274        cargo_config = _read_cargo_config(repository_ctx),
275        render_config = _get_render_config(repository_ctx),
276        supported_platform_triples = repository_ctx.attr.supported_platform_triples,
277        repository_name = repository_ctx.name,
278        repository_ctx = repository_ctx,
279    )
280
281    config_path = repository_ctx.path("cargo-bazel.json")
282    repository_ctx.file(
283        config_path,
284        json.encode_indent(config, indent = " " * 4),
285    )
286
287    return config_path
288
289def get_lockfiles(repository_ctx):
290    """_summary_
291
292    Args:
293        repository_ctx (repository_ctx): The rule's context object.
294
295    Returns:
296        struct: _description_
297    """
298    return struct(
299        cargo = repository_ctx.path(repository_ctx.attr.cargo_lockfile),
300        bazel = repository_ctx.path(repository_ctx.attr.lockfile) if repository_ctx.attr.lockfile else None,
301    )
302
303def determine_repin(repository_ctx, generator, lockfile_path, config, splicing_manifest, cargo, rustc):
304    """Use the `cargo-bazel` binary to determine whether or not dpeendencies need to be re-pinned
305
306    Args:
307        repository_ctx (repository_ctx): The rule's context object.
308        generator (path): The path to a `cargo-bazel` binary.
309        config (path): The path to a `cargo-bazel` config file. See `generate_config`.
310        splicing_manifest (path): The path to a `cargo-bazel` splicing manifest. See `create_splicing_manifest`
311        lockfile_path (path): The path to a "lock" file for reproducible outputs.
312        cargo (path): The path to a Cargo binary.
313        rustc (path): The path to a Rustc binary.
314
315    Returns:
316        bool: True if dependencies need to be re-pinned
317    """
318
319    # If a repin environment variable is set, always repin
320    for var in REPIN_ENV_VARS:
321        if var in repository_ctx.os.environ and repository_ctx.os.environ[var].lower() not in ["false", "no", "0", "off"]:
322            return True
323
324    # If a deterministic lockfile was not added then always repin
325    if not lockfile_path:
326        return True
327
328    # Run the binary to check if a repin is needed
329    args = [
330        generator,
331        "query",
332        "--lockfile",
333        lockfile_path,
334        "--config",
335        config,
336        "--splicing-manifest",
337        splicing_manifest,
338        "--cargo",
339        cargo,
340        "--rustc",
341        rustc,
342    ]
343
344    env = {
345        "CARGO": str(cargo),
346        "RUSTC": str(rustc),
347        "RUST_BACKTRACE": "full",
348    }
349
350    # Add any Cargo environment variables to the `cargo-bazel` execution
351    env.update(cargo_environ(repository_ctx))
352
353    result = execute(
354        repository_ctx = repository_ctx,
355        args = args,
356        env = env,
357    )
358
359    # If it was determined repinning should occur but there was no
360    # flag indicating repinning was requested, an error is raised
361    # since repinning should be an explicit action
362    if result.stdout.strip().lower() == "repin":
363        # buildifier: disable=print
364        print(result.stderr)
365        fail((
366            "The current `lockfile` is out of date for '{}'. Please re-run " +
367            "bazel using `CARGO_BAZEL_REPIN=true` if this is expected " +
368            "and the lockfile should be updated."
369        ).format(repository_ctx.name))
370
371    return False
372
373def execute_generator(
374        repository_ctx,
375        lockfile_path,
376        cargo_lockfile_path,
377        generator,
378        config,
379        splicing_manifest,
380        repository_dir,
381        cargo,
382        rustc,
383        metadata = None):
384    """Execute the `cargo-bazel` binary to produce `BUILD` and `.bzl` files.
385
386    Args:
387        repository_ctx (repository_ctx): The rule's context object.
388        lockfile_path (path): The path to a "lock" file (file used for reproducible renderings).
389        cargo_lockfile_path (path): The path to a "Cargo.lock" file within the root workspace.
390        generator (path): The path to a `cargo-bazel` binary.
391        config (path): The path to a `cargo-bazel` config file.
392        splicing_manifest (path): The path to a `cargo-bazel` splicing manifest. See `create_splicing_manifest`
393        repository_dir (path): The output path for the Bazel module and BUILD files.
394        cargo (path): The path of a Cargo binary.
395        rustc (path): The path of a Rustc binary.
396        metadata (path, optional): The path to a Cargo metadata json file. If this is set, it indicates to
397            the generator that repinning is required. This file must be adjacent to a `Cargo.toml` and
398            `Cargo.lock` file.
399
400    Returns:
401        struct: The results of `repository_ctx.execute`.
402    """
403    repository_ctx.report_progress("Generating crate BUILD files.")
404
405    args = [
406        generator,
407        "generate",
408        "--cargo-lockfile",
409        cargo_lockfile_path,
410        "--config",
411        config,
412        "--splicing-manifest",
413        splicing_manifest,
414        "--repository-dir",
415        repository_dir,
416        "--cargo",
417        cargo,
418        "--rustc",
419        rustc,
420    ]
421
422    if lockfile_path:
423        args.extend([
424            "--lockfile",
425            lockfile_path,
426        ])
427
428    env = {
429        "RUST_BACKTRACE": "full",
430    }
431
432    # Some components are not required unless re-pinning is enabled
433    if metadata:
434        args.extend([
435            "--repin",
436            "--metadata",
437            metadata,
438        ])
439        env.update({
440            "CARGO": str(cargo),
441            "RUSTC": str(rustc),
442        })
443
444    # Add any Cargo environment variables to the `cargo-bazel` execution
445    env.update(cargo_environ(repository_ctx))
446
447    result = execute(
448        repository_ctx = repository_ctx,
449        args = args,
450        env = env,
451    )
452
453    return result