fork/rules_rust

git clone https://git.julia.blue/fork/rules_rust

Maximilian GoisserAdd `CARGO_BAZEL_REPIN_ONLY` repinning allowlist (#1798)f651cd18

rust-analyzer
17.2 KiB473 linesraw
1"""Utilities directly related to the `generate` step of `cargo-bazel`."""
2
3load(":common_utils.bzl", "CARGO_BAZEL_ISOLATED", "REPIN_ALLOWLIST_ENV_VAR", "REPIN_ENV_VARS", "cargo_environ", "execute")
4
5CARGO_BAZEL_GENERATOR_SHA256 = "CARGO_BAZEL_GENERATOR_SHA256"
6CARGO_BAZEL_GENERATOR_URL = "CARGO_BAZEL_GENERATOR_URL"
7
8GENERATOR_ENV_VARS = [
9    CARGO_BAZEL_GENERATOR_URL,
10    CARGO_BAZEL_GENERATOR_SHA256,
11]
12
13CRATES_REPOSITORY_ENVIRON = GENERATOR_ENV_VARS + REPIN_ENV_VARS + [
14    REPIN_ALLOWLIST_ENV_VAR,
15] + [
16    CARGO_BAZEL_ISOLATED,
17]
18
19def get_generator(repository_ctx, host_triple):
20    """Query network resources to locate a `cargo-bazel` binary
21
22    Args:
23        repository_ctx (repository_ctx): The rule's context object.
24        host_triple (string): A string representing the host triple
25
26    Returns:
27        tuple(path, dict): The path to a `cargo-bazel` binary and the host sha256 pairing.
28            The pairing (dict) may be `None` if there is no need to update the attribute
29    """
30    use_environ = False
31    for var in GENERATOR_ENV_VARS:
32        if var in repository_ctx.os.environ:
33            use_environ = True
34
35    output = repository_ctx.path("cargo-bazel.exe" if "win" in repository_ctx.os.name else "cargo-bazel")
36
37    # The `generator` attribute is the next highest priority behind
38    # environment variables. We check those first before deciding to
39    # use an explicitly provided variable.
40    if not use_environ and repository_ctx.attr.generator:
41        generator = repository_ctx.path(Label(repository_ctx.attr.generator))
42
43        # Resolve a few levels of symlinks to ensure we're accessing the direct binary
44        for _ in range(1, 100):
45            real_generator = generator.realpath
46            if real_generator == generator:
47                break
48            generator = real_generator
49        return generator, None
50
51    # The environment variable will take precedence if set
52    if use_environ:
53        generator_sha256 = repository_ctx.os.environ.get(CARGO_BAZEL_GENERATOR_SHA256)
54        generator_url = repository_ctx.os.environ.get(CARGO_BAZEL_GENERATOR_URL)
55    else:
56        generator_sha256 = repository_ctx.attr.generator_sha256s.get(host_triple)
57        generator_url = repository_ctx.attr.generator_urls.get(host_triple)
58
59    if not generator_url:
60        fail((
61            "No generator URL was found either in the `CARGO_BAZEL_GENERATOR_URL` " +
62            "environment variable or for the `{}` triple in the `generator_urls` attribute"
63        ).format(host_triple))
64
65    # Download the file into place
66    if generator_sha256:
67        repository_ctx.download(
68            output = output,
69            url = generator_url,
70            sha256 = generator_sha256,
71            executable = True,
72        )
73        return output, None
74
75    result = repository_ctx.download(
76        output = output,
77        url = generator_url,
78        executable = True,
79    )
80
81    return output, {host_triple: result.sha256}
82
83def render_config(
84        build_file_template = "//:BUILD.{name}-{version}.bazel",
85        crate_label_template = "@{repository}__{name}-{version}//:{target}",
86        crate_repository_template = "{repository}__{name}-{version}",
87        crates_module_template = "//:{file}",
88        default_package_name = None,
89        platforms_template = "@rules_rust//rust/platform:{triple}",
90        regen_command = None,
91        vendor_mode = None):
92    """Various settings used to configure rendered outputs
93
94    The template parameters each support a select number of format keys. A description of each key
95    can be found below where the supported keys for each template can be found in the parameter docs
96
97    | key | definition |
98    | --- | --- |
99    | `name` | The name of the crate. Eg `tokio` |
100    | `repository` | The rendered repository name for the crate. Directly relates to `crate_repository_template`. |
101    | `triple` | A platform triple. Eg `x86_64-unknown-linux-gnu` |
102    | `version` | The crate version. Eg `1.2.3` |
103    | `target` | The library or binary target of the crate |
104    | `file` | The basename of a file |
105
106    Args:
107        build_file_template (str, optional): The base template to use for BUILD file names. The available format keys
108            are [`{name}`, {version}`].
109        crate_label_template (str, optional): The base template to use for crate labels. The available format keys
110            are [`{repository}`, `{name}`, `{version}`, `{target}`].
111        crate_repository_template (str, optional): The base template to use for Crate label repository names. The
112            available format keys are [`{repository}`, `{name}`, `{version}`].
113        crates_module_template (str, optional): The pattern to use for the `defs.bzl` and `BUILD.bazel`
114            file names used for the crates module. The available format keys are [`{file}`].
115        default_package_name (str, optional): The default package name to use in the rendered macros. This affects the
116            auto package detection of things like `all_crate_deps`.
117        platforms_template (str, optional): The base template to use for platform names.
118            See [platforms documentation](https://docs.bazel.build/versions/main/platforms.html). The available format
119            keys are [`{triple}`].
120        regen_command (str, optional): An optional command to demonstrate how generated files should be regenerated.
121        vendor_mode (str, optional): An optional configuration for rendirng content to be rendered into repositories.
122
123    Returns:
124        string: A json encoded struct to match the Rust `config::RenderConfig` struct
125    """
126    return json.encode(struct(
127        build_file_template = build_file_template,
128        crate_label_template = crate_label_template,
129        crate_repository_template = crate_repository_template,
130        crates_module_template = crates_module_template,
131        default_package_name = default_package_name,
132        platforms_template = platforms_template,
133        regen_command = regen_command,
134        vendor_mode = vendor_mode,
135    ))
136
137def _crate_id(name, version):
138    """Creates a `cargo_bazel::config::CrateId`.
139
140    Args:
141        name (str): The name of the crate
142        version (str): The crate's version
143
144    Returns:
145        str: A serialized representation of a CrateId
146    """
147    return "{} {}".format(name, version)
148
149def collect_crate_annotations(annotations, repository_name):
150    """Deserialize and sanitize crate annotations.
151
152    Args:
153        annotations (dict): A mapping of crate names to lists of serialized annotations
154        repository_name (str): The name of the repository that owns the annotations
155
156    Returns:
157        dict: A mapping of `cargo_bazel::config::CrateId` to sets of annotations
158    """
159    annotations = {name: [json.decode(a) for a in annotation] for name, annotation in annotations.items()}
160    crate_annotations = {}
161    for name, annotation in annotations.items():
162        for (version, data) in annotation:
163            if name == "*" and version != "*":
164                fail(
165                    "Wildcard crate names must have wildcard crate versions. " +
166                    "Please update the `annotations` attribute of the {} crates_repository".format(
167                        repository_name,
168                    ),
169                )
170            id = _crate_id(name, version)
171            if id in crate_annotations:
172                fail("Found duplicate entries for {}".format(id))
173
174            crate_annotations.update({id: data})
175    return crate_annotations
176
177def _read_cargo_config(repository_ctx):
178    if repository_ctx.attr.cargo_config:
179        config = repository_ctx.path(repository_ctx.attr.cargo_config)
180        return repository_ctx.read(config)
181    return None
182
183def _update_render_config(config, repository_name):
184    """Add the repository name to the render config
185
186    Args:
187        config (dict): A `render_config` struct
188        repository_name (str): The name of the repository that owns the config
189
190    Returns:
191        struct: An updated `render_config`.
192    """
193
194    # Add the repository name as it's very relevant to rendering.
195    config.update({"repository_name": repository_name})
196
197    return struct(**config)
198
199def _get_render_config(repository_ctx):
200    if repository_ctx.attr.render_config:
201        config = dict(json.decode(repository_ctx.attr.render_config))
202    else:
203        config = dict(json.decode(render_config()))
204
205    if not config.get("regen_command"):
206        config["regen_command"] = "bazel sync --only={}".format(
207            repository_ctx.name,
208        )
209
210    return config
211
212def compile_config(
213        crate_annotations,
214        generate_binaries,
215        generate_build_scripts,
216        cargo_config,
217        render_config,
218        supported_platform_triples,
219        repository_name,
220        repository_ctx = None):
221    """Create a config file for generating crate targets
222
223    [cargo_config]: https://doc.rust-lang.org/cargo/reference/config.html
224
225    Args:
226        crate_annotations (dict): Extra settings to apply to crates. See
227            `crates_repository.annotations` or `crates_vendor.annotations`.
228        generate_binaries (bool): Whether to generate `rust_binary` targets for all bins.
229        generate_build_scripts (bool): Whether or not to globally disable build scripts.
230        cargo_config (str): The optional contents of a [Cargo config][cargo_config].
231        render_config (dict): The deserialized dict of the `render_config` function.
232        supported_platform_triples (list): A list of platform triples
233        repository_name (str): The name of the repository being generated
234        repository_ctx (repository_ctx, optional): A repository context object used for enabling
235            certain functionality.
236
237    Returns:
238        struct: A struct matching a `cargo_bazel::config::Config`.
239    """
240    annotations = collect_crate_annotations(crate_annotations, repository_name)
241
242    # Load additive build files if any have been provided.
243    unexpected = []
244    for name, data in annotations.items():
245        f = data.pop("additive_build_file", None)
246        if f and not repository_ctx:
247            unexpected.append(name)
248            f = None
249        content = [x for x in [
250            data.pop("additive_build_file_content", None),
251            repository_ctx.read(Label(f)) if f else None,
252        ] if x]
253        if content:
254            data.update({"additive_build_file_content": "\n".join(content)})
255
256    if unexpected:
257        fail("The following annotations use `additive_build_file` which is not supported for {}: {}".format(repository_name, unexpected))
258
259    config = struct(
260        generate_binaries = generate_binaries,
261        generate_build_scripts = generate_build_scripts,
262        annotations = annotations,
263        cargo_config = cargo_config,
264        rendering = _update_render_config(
265            config = render_config,
266            repository_name = repository_name,
267        ),
268        supported_platform_triples = supported_platform_triples,
269    )
270
271    return config
272
273def generate_config(repository_ctx):
274    """Generate a config file from various attributes passed to the rule.
275
276    Args:
277        repository_ctx (repository_ctx): The rule's context object.
278
279    Returns:
280        struct: A struct containing the path to a config and it's contents
281    """
282
283    config = compile_config(
284        crate_annotations = repository_ctx.attr.annotations,
285        generate_binaries = repository_ctx.attr.generate_binaries,
286        generate_build_scripts = repository_ctx.attr.generate_build_scripts,
287        cargo_config = _read_cargo_config(repository_ctx),
288        render_config = _get_render_config(repository_ctx),
289        supported_platform_triples = repository_ctx.attr.supported_platform_triples,
290        repository_name = repository_ctx.name,
291        repository_ctx = repository_ctx,
292    )
293
294    config_path = repository_ctx.path("cargo-bazel.json")
295    repository_ctx.file(
296        config_path,
297        json.encode_indent(config, indent = " " * 4),
298    )
299
300    return config_path
301
302def get_lockfiles(repository_ctx):
303    """_summary_
304
305    Args:
306        repository_ctx (repository_ctx): The rule's context object.
307
308    Returns:
309        struct: _description_
310    """
311    return struct(
312        cargo = repository_ctx.path(repository_ctx.attr.cargo_lockfile),
313        bazel = repository_ctx.path(repository_ctx.attr.lockfile) if repository_ctx.attr.lockfile else None,
314    )
315
316def determine_repin(repository_ctx, generator, lockfile_path, config, splicing_manifest, cargo, rustc):
317    """Use the `cargo-bazel` binary to determine whether or not dpeendencies need to be re-pinned
318
319    Args:
320        repository_ctx (repository_ctx): The rule's context object.
321        generator (path): The path to a `cargo-bazel` binary.
322        config (path): The path to a `cargo-bazel` config file. See `generate_config`.
323        splicing_manifest (path): The path to a `cargo-bazel` splicing manifest. See `create_splicing_manifest`
324        lockfile_path (path): The path to a "lock" file for reproducible outputs.
325        cargo (path): The path to a Cargo binary.
326        rustc (path): The path to a Rustc binary.
327
328    Returns:
329        bool: True if dependencies need to be re-pinned
330    """
331
332    # If a repin environment variable is set, always repin
333    for var in REPIN_ENV_VARS:
334        if var in repository_ctx.os.environ and repository_ctx.os.environ[var].lower() not in ["false", "no", "0", "off"]:
335            # If a repin allowlist is present only force repin if name is in list
336            if REPIN_ALLOWLIST_ENV_VAR in repository_ctx.os.environ:
337                indices_to_repin = repository_ctx.os.environ[REPIN_ALLOWLIST_ENV_VAR].split(",")
338                if repository_ctx.name in indices_to_repin:
339                    return True
340            else:
341                return True
342
343    # If a deterministic lockfile was not added then always repin
344    if not lockfile_path:
345        return True
346
347    # Run the binary to check if a repin is needed
348    args = [
349        generator,
350        "query",
351        "--lockfile",
352        lockfile_path,
353        "--config",
354        config,
355        "--splicing-manifest",
356        splicing_manifest,
357        "--cargo",
358        cargo,
359        "--rustc",
360        rustc,
361    ]
362
363    env = {
364        "CARGO": str(cargo),
365        "RUSTC": str(rustc),
366        "RUST_BACKTRACE": "full",
367    }
368
369    # Add any Cargo environment variables to the `cargo-bazel` execution
370    env.update(cargo_environ(repository_ctx))
371
372    result = execute(
373        repository_ctx = repository_ctx,
374        args = args,
375        env = env,
376    )
377
378    # If it was determined repinning should occur but there was no
379    # flag indicating repinning was requested, an error is raised
380    # since repinning should be an explicit action
381    if result.stdout.strip().lower() == "repin":
382        fail(("\n".join([
383            result.stderr,
384            (
385                "The current `lockfile` is out of date for '{}'. Please re-run " +
386                "bazel using `CARGO_BAZEL_REPIN=true` if this is expected " +
387                "and the lockfile should be updated."
388            ).format(repository_ctx.name),
389        ])))
390
391    return False
392
393def execute_generator(
394        repository_ctx,
395        lockfile_path,
396        cargo_lockfile_path,
397        generator,
398        config,
399        splicing_manifest,
400        repository_dir,
401        cargo,
402        rustc,
403        metadata = None):
404    """Execute the `cargo-bazel` binary to produce `BUILD` and `.bzl` files.
405
406    Args:
407        repository_ctx (repository_ctx): The rule's context object.
408        lockfile_path (path): The path to a "lock" file (file used for reproducible renderings).
409        cargo_lockfile_path (path): The path to a "Cargo.lock" file within the root workspace.
410        generator (path): The path to a `cargo-bazel` binary.
411        config (path): The path to a `cargo-bazel` config file.
412        splicing_manifest (path): The path to a `cargo-bazel` splicing manifest. See `create_splicing_manifest`
413        repository_dir (path): The output path for the Bazel module and BUILD files.
414        cargo (path): The path of a Cargo binary.
415        rustc (path): The path of a Rustc binary.
416        metadata (path, optional): The path to a Cargo metadata json file. If this is set, it indicates to
417            the generator that repinning is required. This file must be adjacent to a `Cargo.toml` and
418            `Cargo.lock` file.
419
420    Returns:
421        struct: The results of `repository_ctx.execute`.
422    """
423    repository_ctx.report_progress("Generating crate BUILD files.")
424
425    args = [
426        generator,
427        "generate",
428        "--cargo-lockfile",
429        cargo_lockfile_path,
430        "--config",
431        config,
432        "--splicing-manifest",
433        splicing_manifest,
434        "--repository-dir",
435        repository_dir,
436        "--cargo",
437        cargo,
438        "--rustc",
439        rustc,
440    ]
441
442    if lockfile_path:
443        args.extend([
444            "--lockfile",
445            lockfile_path,
446        ])
447
448    env = {
449        "RUST_BACKTRACE": "full",
450    }
451
452    # Some components are not required unless re-pinning is enabled
453    if metadata:
454        args.extend([
455            "--repin",
456            "--metadata",
457            metadata,
458        ])
459        env.update({
460            "CARGO": str(cargo),
461            "RUSTC": str(rustc),
462        })
463
464    # Add any Cargo environment variables to the `cargo-bazel` execution
465    env.update(cargo_environ(repository_ctx))
466
467    result = execute(
468        repository_ctx = repository_ctx,
469        args = args,
470        env = env,
471    )
472
473    return result