fork/rules_rust

git clone https://git.julia.blue/fork/rules_rust

UebelAndrecrate_universe: Added tests for serialized config files. (#1291)48bb32f8

0.5.0
16.4 KiB457 linesraw
1"""Utilities directly related to the `generate` step of `cargo-bazel`."""
2
3load(":common_utils.bzl", "CARGO_BAZEL_ISOLATED", "cargo_environ", "execute")
4
5CARGO_BAZEL_GENERATOR_SHA256 = "CARGO_BAZEL_GENERATOR_SHA256"
6CARGO_BAZEL_GENERATOR_URL = "CARGO_BAZEL_GENERATOR_URL"
7CARGO_BAZEL_REPIN = "CARGO_BAZEL_REPIN"
8REPIN = "REPIN"
9
10GENERATOR_ENV_VARS = [
11    CARGO_BAZEL_GENERATOR_URL,
12    CARGO_BAZEL_GENERATOR_SHA256,
13]
14
15REPIN_ENV_VARS = [
16    REPIN,
17    CARGO_BAZEL_REPIN,
18]
19
20CRATES_REPOSITORY_ENVIRON = GENERATOR_ENV_VARS + REPIN_ENV_VARS + [
21    CARGO_BAZEL_ISOLATED,
22]
23
24def get_generator(repository_ctx, host_triple):
25    """Query network resources to locate a `cargo-bazel` binary
26
27    Args:
28        repository_ctx (repository_ctx): The rule's context object.
29        host_triple (string): A string representing the host triple
30
31    Returns:
32        tuple(path, dict): The path to a `cargo-bazel` binary and the host sha256 pairing.
33            The pairing (dict) may be `None` if there is no need to update the attribute
34    """
35    use_environ = False
36    for var in GENERATOR_ENV_VARS:
37        if var in repository_ctx.os.environ:
38            use_environ = True
39
40    output = repository_ctx.path("cargo-bazel.exe" if "win" in repository_ctx.os.name else "cargo-bazel")
41
42    # The `generator` attribute is the next highest priority behind
43    # environment variables. We check those first before deciding to
44    # use an explicitly provided variable.
45    if not use_environ and repository_ctx.attr.generator:
46        generator = repository_ctx.path(Label(repository_ctx.attr.generator))
47
48        # Resolve a few levels of symlinks to ensure we're accessing the direct binary
49        for _ in range(1, 100):
50            real_generator = generator.realpath
51            if real_generator == generator:
52                break
53            generator = real_generator
54        return generator, None
55
56    # The environment variable will take precedence if set
57    if use_environ:
58        generator_sha256 = repository_ctx.os.environ.get(CARGO_BAZEL_GENERATOR_SHA256)
59        generator_url = repository_ctx.os.environ.get(CARGO_BAZEL_GENERATOR_URL)
60    else:
61        generator_sha256 = repository_ctx.attr.generator_sha256s.get(host_triple)
62        generator_url = repository_ctx.attr.generator_urls.get(host_triple)
63
64    if not generator_url:
65        fail((
66            "No generator URL was found either in the `CARGO_BAZEL_GENERATOR_URL` " +
67            "environment variable or for the `{}` triple in the `generator_urls` attribute"
68        ).format(host_triple))
69
70    # Download the file into place
71    if generator_sha256:
72        repository_ctx.download(
73            output = output,
74            url = generator_url,
75            sha256 = generator_sha256,
76            executable = True,
77        )
78        return output, None
79
80    result = repository_ctx.download(
81        output = output,
82        url = generator_url,
83        executable = True,
84    )
85
86    return output, {host_triple: result.sha256}
87
88def render_config(
89        build_file_template = "//:BUILD.{name}-{version}.bazel",
90        crate_label_template = "@{repository}__{name}-{version}//:{target}",
91        crate_repository_template = "{repository}__{name}-{version}",
92        crates_module_template = "//:{file}",
93        default_package_name = None,
94        platforms_template = "@rules_rust//rust/platform:{triple}",
95        vendor_mode = None):
96    """Various settings used to configure rendered outputs
97
98    The template parameters each support a select number of format keys. A description of each key
99    can be found below where the supported keys for each template can be found in the parameter docs
100
101    | key | definition |
102    | --- | --- |
103    | `name` | The name of the crate. Eg `tokio` |
104    | `repository` | The rendered repository name for the crate. Directly relates to `crate_repository_template`. |
105    | `triple` | A platform triple. Eg `x86_64-unknown-linux-gnu` |
106    | `version` | The crate version. Eg `1.2.3` |
107    | `target` | The library or binary target of the crate |
108    | `file` | The basename of a file |
109
110    Args:
111        build_file_template (str, optional): The base template to use for BUILD file names. The available format keys
112            are [`{name}`, {version}`].
113        crate_label_template (str, optional): The base template to use for crate labels. The available format keys
114            are [`{repository}`, `{name}`, `{version}`, `{target}`].
115        crate_repository_template (str, optional): The base template to use for Crate label repository names. The
116            available format keys are [`{repository}`, `{name}`, `{version}`].
117        crates_module_template (str, optional): The pattern to use for the `defs.bzl` and `BUILD.bazel`
118            file names used for the crates module. The available format keys are [`{file}`].
119        default_package_name (str, optional): The default package name to use in the rendered macros. This affects the
120            auto package detection of things like `all_crate_deps`.
121        platforms_template (str, optional): The base template to use for platform names.
122            See [platforms documentation](https://docs.bazel.build/versions/main/platforms.html). The available format
123            keys are [`{triple}`].
124        vendor_mode (str, optional): An optional configuration for rendirng content to be rendered into repositories.
125
126    Returns:
127        string: A json encoded struct to match the Rust `config::RenderConfig` struct
128    """
129    return json.encode(struct(
130        build_file_template = build_file_template,
131        crate_label_template = crate_label_template,
132        crate_repository_template = crate_repository_template,
133        crates_module_template = crates_module_template,
134        default_package_name = default_package_name,
135        platforms_template = platforms_template,
136        vendor_mode = vendor_mode,
137    ))
138
139def _crate_id(name, version):
140    """Creates a `cargo_bazel::config::CrateId`.
141
142    Args:
143        name (str): The name of the crate
144        version (str): The crate's version
145
146    Returns:
147        str: A serialized representation of a CrateId
148    """
149    return "{} {}".format(name, version)
150
151def collect_crate_annotations(annotations, repository_name):
152    """Deserialize and sanitize crate annotations.
153
154    Args:
155        annotations (dict): A mapping of crate names to lists of serialized annotations
156        repository_name (str): The name of the repository that owns the annotations
157
158    Returns:
159        dict: A mapping of `cargo_bazel::config::CrateId` to sets of annotations
160    """
161    annotations = {name: [json.decode(a) for a in annotation] for name, annotation in annotations.items()}
162    crate_annotations = {}
163    for name, annotation in annotations.items():
164        for (version, data) in annotation:
165            if name == "*" and version != "*":
166                fail(
167                    "Wildcard crate names must have wildcard crate versions. " +
168                    "Please update the `annotations` attribute of the {} crates_repository".format(
169                        repository_name,
170                    ),
171                )
172            id = _crate_id(name, version)
173            if id in crate_annotations:
174                fail("Found duplicate entries for {}".format(id))
175
176            crate_annotations.update({id: data})
177    return crate_annotations
178
179def _read_cargo_config(repository_ctx):
180    if repository_ctx.attr.cargo_config:
181        config = repository_ctx.path(repository_ctx.attr.cargo_config)
182        return repository_ctx.read(config)
183    return None
184
185def _update_render_config(config, repository_name):
186    """Add the repository name to the render config
187
188    Args:
189        config (dict): A `render_config` struct
190        repository_name (str): The name of the repository that owns the config
191
192    Returns:
193        struct: An updated `render_config`.
194    """
195
196    # Add the repository name as it's very relevant to rendering.
197    config.update({"repository_name": repository_name})
198
199    return struct(**config)
200
201def _get_render_config(repository_ctx):
202    if repository_ctx.attr.render_config:
203        config = dict(json.decode(repository_ctx.attr.render_config))
204    else:
205        config = dict(json.decode(render_config()))
206
207    return config
208
209def compile_config(crate_annotations, generate_build_scripts, cargo_config, render_config, supported_platform_triples, repository_name, repository_ctx = None):
210    """Create a config file for generating crate targets
211
212    [cargo_config]: https://doc.rust-lang.org/cargo/reference/config.html
213
214    Args:
215        crate_annotations (dict): Extra settings to apply to crates. See
216            `crates_repository.annotations` or `crates_vendor.annotations`.
217        generate_build_scripts (bool): Whether or not to globally disable build scripts.
218        cargo_config (str): The optional contents of a [Cargo config][cargo_config].
219        render_config (dict): The deserialized dict of the `render_config` function.
220        supported_platform_triples (list): A list of platform triples
221        repository_name (str): The name of the repository being generated
222        repository_ctx (repository_ctx, optional): A repository context object used for enabling
223            certain functionality.
224
225    Returns:
226        struct: A struct matching a `cargo_bazel::config::Config`.
227    """
228    annotations = collect_crate_annotations(crate_annotations, repository_name)
229
230    # Load additive build files if any have been provided.
231    unexpected = []
232    for name, data in annotations.items():
233        f = data.pop("additive_build_file", None)
234        if f and not repository_ctx:
235            unexpected.append(name)
236            f = None
237        content = [x for x in [
238            data.pop("additive_build_file_content", None),
239            repository_ctx.read(Label(f)) if f else None,
240        ] if x]
241        if content:
242            data.update({"additive_build_file_content": "\n".join(content)})
243
244    if unexpected:
245        fail("The following annotations use `additive_build_file` which is not supported for {}: {}".format(repository_name, unexpected))
246
247    config = struct(
248        generate_build_scripts = generate_build_scripts,
249        annotations = annotations,
250        cargo_config = cargo_config,
251        rendering = _update_render_config(
252            config = render_config,
253            repository_name = repository_name,
254        ),
255        supported_platform_triples = supported_platform_triples,
256    )
257
258    return config
259
260def generate_config(repository_ctx):
261    """Generate a config file from various attributes passed to the rule.
262
263    Args:
264        repository_ctx (repository_ctx): The rule's context object.
265
266    Returns:
267        struct: A struct containing the path to a config and it's contents
268    """
269
270    config = compile_config(
271        crate_annotations = repository_ctx.attr.annotations,
272        generate_build_scripts = repository_ctx.attr.generate_build_scripts,
273        cargo_config = _read_cargo_config(repository_ctx),
274        render_config = _get_render_config(repository_ctx),
275        supported_platform_triples = repository_ctx.attr.supported_platform_triples,
276        repository_name = repository_ctx.name,
277        repository_ctx = repository_ctx,
278    )
279
280    config_path = repository_ctx.path("cargo-bazel.json")
281    repository_ctx.file(
282        config_path,
283        json.encode_indent(config, indent = " " * 4),
284    )
285
286    return config_path
287
288def get_lockfile(repository_ctx):
289    """Locate the lockfile and identify the it's type (Cargo or Bazel).
290
291    Args:
292        repository_ctx (repository_ctx): The rule's context object.
293
294    Returns:
295        struct: The path to the lockfile as well as it's type
296    """
297    if repository_ctx.attr.lockfile_kind == "auto":
298        if str(repository_ctx.attr.lockfile).endswith("Cargo.lock"):
299            kind = "cargo"
300        else:
301            kind = "bazel"
302    else:
303        kind = repository_ctx.attr.lockfile_kind
304
305    return struct(
306        path = repository_ctx.path(repository_ctx.attr.lockfile),
307        kind = kind,
308    )
309
310def determine_repin(repository_ctx, generator, lockfile_path, lockfile_kind, config, splicing_manifest, cargo, rustc):
311    """Use the `cargo-bazel` binary to determine whether or not dpeendencies need to be re-pinned
312
313    Args:
314        repository_ctx (repository_ctx): The rule's context object.
315        generator (path): The path to a `cargo-bazel` binary.
316        config (path): The path to a `cargo-bazel` config file. See `generate_config`.
317        splicing_manifest (path): The path to a `cargo-bazel` splicing manifest. See `create_splicing_manifest`
318        lockfile_path (path): The path to a "lock" file for reproducible outputs.
319        lockfile_kind (str): The type of lock file represented by `lockfile_path`
320        cargo (path): The path to a Cargo binary.
321        rustc (path): The path to a Rustc binary.
322
323    Returns:
324        bool: True if dependencies need to be re-pinned
325    """
326
327    # If a repin environment variable is set, always repin
328    for var in REPIN_ENV_VARS:
329        if repository_ctx.os.environ.get(var, "").lower() in ["true", "yes", "1", "on"]:
330            return True
331
332    # Cargo lockfiles should always be repinned.
333    if lockfile_kind == "cargo":
334        return True
335
336    # Run the binary to check if a repin is needed
337    args = [
338        generator,
339        "query",
340        "--lockfile",
341        lockfile_path,
342        "--config",
343        config,
344        "--splicing-manifest",
345        splicing_manifest,
346        "--cargo",
347        cargo,
348        "--rustc",
349        rustc,
350    ]
351
352    env = {
353        "CARGO": str(cargo),
354        "RUSTC": str(rustc),
355        "RUST_BACKTRACE": "full",
356    }
357
358    # Add any Cargo environment variables to the `cargo-bazel` execution
359    env.update(cargo_environ(repository_ctx))
360
361    result = execute(
362        repository_ctx = repository_ctx,
363        args = args,
364        env = env,
365    )
366
367    # If it was determined repinning should occur but there was no
368    # flag indicating repinning was requested, an error is raised
369    # since repinning should be an explicit action
370    if result.stdout.strip().lower() == "repin":
371        # buildifier: disable=print
372        print(result.stderr)
373        fail((
374            "The current `lockfile` is out of date for '{}'. Please re-run " +
375            "bazel using `CARGO_BAZEL_REPIN=true` if this is expected " +
376            "and the lockfile should be updated."
377        ).format(repository_ctx.name))
378
379    return False
380
381def execute_generator(
382        repository_ctx,
383        lockfile_path,
384        lockfile_kind,
385        generator,
386        config,
387        splicing_manifest,
388        repository_dir,
389        cargo,
390        rustc,
391        repin = False,
392        metadata = None):
393    """Execute the `cargo-bazel` binary to produce `BUILD` and `.bzl` files.
394
395    Args:
396        repository_ctx (repository_ctx): The rule's context object.
397        lockfile_path (path): The path to a "lock" file (file used for reproducible renderings).
398        lockfile_kind (str): The type of lockfile given (Cargo or Bazel).
399        generator (path): The path to a `cargo-bazel` binary.
400        config (path): The path to a `cargo-bazel` config file.
401        splicing_manifest (path): The path to a `cargo-bazel` splicing manifest. See `create_splicing_manifest`
402        repository_dir (path): The output path for the Bazel module and BUILD files.
403        cargo (path): The path of a Cargo binary.
404        rustc (path): The path of a Rustc binary.
405        repin (bool, optional): Whether or not to repin dependencies
406        metadata (path, optional): The path to a Cargo metadata json file.
407
408    Returns:
409        struct: The results of `repository_ctx.execute`.
410    """
411    repository_ctx.report_progress("Generating crate BUILD files.")
412
413    args = [
414        generator,
415        "generate",
416        "--lockfile",
417        lockfile_path,
418        "--lockfile-kind",
419        lockfile_kind,
420        "--config",
421        config,
422        "--splicing-manifest",
423        splicing_manifest,
424        "--repository-dir",
425        repository_dir,
426        "--cargo",
427        cargo,
428        "--rustc",
429        rustc,
430    ]
431
432    env = {
433        "RUST_BACKTRACE": "full",
434    }
435
436    # Some components are not required unless re-pinning is enabled
437    if repin:
438        args.extend([
439            "--repin",
440            "--metadata",
441            metadata,
442        ])
443        env.update({
444            "CARGO": str(cargo),
445            "RUSTC": str(rustc),
446        })
447
448    # Add any Cargo environment variables to the `cargo-bazel` execution
449    env.update(cargo_environ(repository_ctx))
450
451    result = execute(
452        repository_ctx = repository_ctx,
453        args = args,
454        env = env,
455    )
456
457    return result