fork/rules_rust

git clone https://git.julia.blue/fork/rules_rust

Stu SchwartzAdd wasm64 support. (#2866)beebf3b0

cleanup
12.5 KiB376 linesraw
1#!/usr/bin/env python3.11
2"""A script for generating the `//rust:known_shas.bzl` file."""
3
4import json
5import logging
6import os
7import shutil
8import subprocess
9import sys
10import tempfile
11import tomllib
12from pathlib import Path
13from typing import Any, Dict, Sequence
14
15KNOWN_SHAS_TEMPLATE = """\
16\"\"\"A module containing a mapping of Rust tools to checksums
17
18This is a generated file -- see //util/fetch_shas
19\"\"\"
20
21FILE_KEY_TO_SHA = {}
22"""
23
24
25def download_manifest_data(
26    stable_versions: Sequence[str], nightly_versions: Sequence[str], output_dir: Path
27) -> Dict[str, Sequence[Dict[str, Any]]]:
28    """Download and deserialize `channel-rust-*.toml` files for the requested versions
29
30    Args:
31        stable_versions: A list of Rust versions. E.g. `1.81.0`
32        nightly_versions: A list of nightly iso dates. E.g. `2024-09-05`
33        output_dir: The location where the intermediate files should be written.
34
35    Returns:
36        A mapping of channel name (`stable`, `nightly`) to deserialized data.
37    """
38
39    output_dir.mkdir(exist_ok=True, parents=True)
40    curl_config = output_dir / "curl_config.txt"
41    curl_config_lines = [
42        "--fail",
43        "--parallel",
44        "--silent",
45        "--create-dirs",
46    ]
47
48    stable_files = {}
49    for version in stable_versions:
50        output = output_dir / f"channel-rust-{version}.toml"
51        curl_config_lines.extend(
52            [
53                f"--output {output}",
54                f"--url https://static.rust-lang.org/dist/channel-rust-{version}.toml",
55            ]
56        )
57        stable_files[version] = output
58    nightly_files = {}
59    for version in nightly_versions:
60        output = output_dir / version / "channel-rust-nightly.toml"
61        curl_config_lines.extend(
62            [
63                f"--output {output}",
64                f"--url https://static.rust-lang.org/dist/{version}/channel-rust-nightly.toml",
65            ]
66        )
67        nightly_files[version] = output
68
69    curl_config.write_text("\n".join(curl_config_lines), encoding="utf-8")
70
71    logging.info("Downloading data...")
72    subprocess.run(
73        [
74            "curl",
75            "--config",
76            curl_config,
77        ],
78        check=True,
79    )
80    logging.info("Done.")
81
82    logging.info(
83        "Deserializing %s tomls...",
84        len(list(stable_files.keys()) + list(nightly_files.keys())),
85    )
86    for collection in (stable_files, nightly_files):
87        for version, file in collection.items():
88            data = file.read_text(encoding="utf-8")
89            assert data
90            try:
91                collection[version] = tomllib.loads(data)
92            except:
93                logging.error("Failed to parse toml: %s\n%s", file, data)
94                raise
95
96    logging.info("Done.")
97    return {
98        "stable": stable_files,
99        "nightly": nightly_files,
100    }
101
102
103def download_direct_sha256s(
104    artifacts: Sequence[str], output_dir: Path
105) -> Dict[str, str]:
106    """_summary_
107
108    This function is mostly here for backward compatibility. There are artifacts
109    referenced by the `channel-rust-*.toml` files which are marked as `available: false`
110    and probably intented to not be downloaded. But for now this is ignored and instead
111    a collection of artifacts whose hash data could not be found is explicitly checked
112    by trying to download the `.sha256` files directly. A 404 indicates the artifact
113    genuinely does not exist and anything else we find is extra data we can retain
114    in `known_shas.bzl`.
115
116    Args:
117        artifacts: A list of paths within `https://static.rust-lang.org/dist` to download.
118        output_dir: The location where the intermediate files should be written.
119
120    Returns:
121        A mapping of `artifacts` entries to their sha256 value.
122    """
123    output_dir.mkdir(exist_ok=True, parents=True)
124    status_config = output_dir / "status" / "curl_config.txt"
125    sha256_config = output_dir / "sha256" / "curl_config.txt"
126    common_config_lines = [
127        "--parallel",
128        "--create-dirs",
129    ]
130    status_config_lines = common_config_lines + [
131        "--silent",
132        "--head",
133    ]
134
135    statuses = {}
136    for url_path in artifacts:
137        output = status_config.parent / f"{url_path}.sha256.status"
138        status_config_lines.extend(
139            [
140                f"--output {output}",
141                f"--url https://static.rust-lang.org/dist/{url_path}.sha256",
142            ]
143        )
144        statuses[url_path] = output
145
146    status_config.parent.mkdir(exist_ok=True, parents=True)
147    status_config.write_text("\n".join(status_config_lines), encoding="utf-8")
148
149    logging.info("Checking for %s missing artifacts...", len(statuses))
150    result = subprocess.run(
151        [
152            "curl",
153            "--config",
154            status_config,
155        ],
156        check=True,
157    )
158    logging.info("Done.")
159
160    checksums = {}
161    missing = []
162    for url_path, status_file in statuses.items():
163        if not status_file.exists():
164            missing.append(f"https://static.rust-lang.org/dist/{url_path}.sha256")
165            continue
166
167        if status_file.read_text(encoding="utf-8").startswith("HTTP/2 404"):
168            continue
169
170        checksums[url_path] = sha256_config.parent / f"{url_path}.sha256"
171
172    if missing:
173        logging.warning(
174            "Status not found for %s artifacts:\n%s",
175            len(missing),
176            json.dumps(sorted(missing), indent=2),
177        )
178
179    logging.info("Downloading %s missing artifacts...", len(checksums))
180    sha256_config_lines = common_config_lines + [
181        "--verbose",
182    ]
183    for url_path, output in checksums.items():
184        sha256_config_lines.extend(
185            [
186                f"--output {output}",
187                f"--url https://static.rust-lang.org/dist/{url_path}.sha256",
188            ]
189        )
190
191    sha256_config.parent.mkdir(exist_ok=True, parents=True)
192    sha256_config.write_text("\n".join(sha256_config_lines), encoding="utf-8")
193    result = subprocess.run(
194        [
195            "curl",
196            "--config",
197            sha256_config,
198        ],
199        encoding="utf-8",
200        check=False,
201        stderr=subprocess.STDOUT,
202        stdout=subprocess.PIPE,
203    )
204
205    if result.returncode:
206        print(result.stdout, file=sys.stderr)
207        sys.exit(result.returncode)
208
209    logging.info("Done.")
210
211    return {
212        name: file.read_text(encoding="utf-8").split(" ")[0].strip()
213        for name, file in checksums.items()
214    }
215
216
217def load_data(file: Path) -> Sequence[str]:
218    """Load a `fetch_shas_*.txt` file
219
220    Args:
221        file: The file to load
222
223    Returns:
224        A list of the file's contents.
225    """
226    data = []
227    for line in file.read_text(encoding="utf-8").splitlines():
228        text = line.strip()
229        if not text:
230            continue
231        data.append(text)
232
233    return sorted(set(data))
234
235
236def main() -> None:
237    """The  main entrypoint."""
238    if "BUILD_WORKSPACE_DIRECTORY" in os.environ:
239        workspace_dir = Path(os.environ["BUILD_WORKSPACE_DIRECTORY"])
240    else:
241        workspace_dir = Path(__file__).parent.parent.parent
242
243    logging.basicConfig(
244        level=(
245            logging.INFO
246            if "RULES_RUST_FETCH_SHAS_DEBUG" not in os.environ
247            else logging.DEBUG
248        ),
249        format="%(asctime)s - %(levelname)s - %(message)s",
250        datefmt="%Y-%m-%d %H:%M:%S",
251    )
252    logging.info("Fetching known sha256 data...")
253
254    tools = load_data(workspace_dir / "util/fetch_shas/fetch_shas_TOOLS.txt")
255    host_tools = load_data(workspace_dir / "util/fetch_shas/fetch_shas_HOST_TOOLS.txt")
256    targets = load_data(workspace_dir / "util/fetch_shas/fetch_shas_TARGETS.txt")
257    stable_versions = load_data(
258        workspace_dir / "util/fetch_shas/fetch_shas_VERSIONS.txt"
259    )
260    nightly_iso_dates = load_data(
261        workspace_dir / "util/fetch_shas/fetch_shas_NIGHTLY_ISO_DATES.txt"
262    )
263
264    # Allow this directory to be optionally cleaned up.
265    tmp_dir = Path(tempfile.mkdtemp(prefix="rules-rust-fetch-shas-"))
266    logging.debug("Temp dir: %s", tmp_dir)
267    try:
268        manifest_data = download_manifest_data(
269            stable_versions=stable_versions,
270            nightly_versions=nightly_iso_dates,
271            output_dir=tmp_dir,
272        )
273
274        file_key_to_sha = {}
275
276        artifacts = []
277
278        logging.info("Parsing artifacts...")
279        for channel, versioned_info in manifest_data.items():
280            for version, info in versioned_info.items():
281                if channel == "stable":
282                    tool_tpl = f"{{pkg}}-{version}-{{target}}.{{ext}}"
283                    host_tool_tpl = f"{{pkg}}-{version}.{{ext}}"
284                else:
285                    tool_tpl = f"{version}/{{pkg}}-{channel}-{{target}}.{{ext}}"
286                    host_tool_tpl = f"{version}/{{pkg}}-{channel}.{{ext}}"
287
288                # Artifacts are commonly referred to with an internal (to the Rust org) name.
289                # In order to correctly evaluate whether or not a particular `channel-rust-*.toml`
290                # has an artifact, a mapping is made from what the internal Rust names are to
291                # what they're advertised as externally (e.g. what users would use when trying
292                # to download files directly).
293                renames = {
294                    "clippy-preview": "clippy",
295                    "llvm-tools-preview": "llvm-tools",
296                    "rustfmt-preview": "rustfmt",
297                }
298                if "renames" in info:
299                    for pkg, rename in info["renames"].items():
300                        if "to" in rename:
301                            renames[rename["to"]] = pkg
302
303                logging.debug("Renames (%s %s): %s", channel, version, renames)
304                for pkg, pkg_data in info["pkg"].items():
305                    pkg_name = renames.get(pkg, pkg)
306                    if pkg_name in tools:
307                        tool_template = tool_tpl
308                    elif pkg_name in host_tools:
309                        tool_template = host_tool_tpl
310                    else:
311                        logging.debug("Skipping %s %s %s", channel, version, pkg_name)
312                        continue
313
314                    drain_targets = list(targets)
315                    for target, target_data in pkg_data["target"].items():
316                        # One variant of the template has an extra format string field.
317                        # Replace is used to account for that.
318                        template = tool_template.replace("{target}", target)
319
320                        if target not in drain_targets:
321                            continue
322
323                        drain_targets.remove(target)
324
325                        if "hash" in target_data:
326                            file_key_to_sha[
327                                template.format(pkg=pkg_name, ext="tar.gz")
328                            ] = target_data["hash"]
329                        if "xz_hash" in target_data:
330                            file_key_to_sha[
331                                template.format(pkg=pkg_name, ext="tar.xz")
332                            ] = target_data["xz_hash"]
333
334                    # If an artifact is not advertised to be available for a particular
335                    # target then we track this and see if the sha256 data can be
336                    # downloaded directly.
337                    for target in drain_targets:
338                        # One variant of the template has an extra format string field.
339                        # Replace is used to account for that.
340                        template = tool_template.replace("{target}", target)
341
342                        # See if we can download the file directly.
343                        artifacts.extend(
344                            [
345                                template.format(pkg=pkg_name, ext="tar.gz"),
346                                template.format(pkg=pkg_name, ext="tar.xz"),
347                            ]
348                        )
349
350        logging.info("Done. Identified %s artifacts.", len(file_key_to_sha))
351
352        # Do a brute force check to find additional sha256 values.
353        file_key_to_sha.update(
354            download_direct_sha256s(
355                artifacts=sorted(set(artifacts)),
356                output_dir=tmp_dir / "retries",
357            )
358        )
359
360    finally:
361        if not "RULES_RUST_FETCH_SHAS_DEBUG" in os.environ:
362            shutil.rmtree(tmp_dir)
363
364    known_shas_file = workspace_dir / "rust/known_shas.bzl"
365    known_shas_file.write_text(
366        KNOWN_SHAS_TEMPLATE.format(
367            json.dumps(file_key_to_sha, sort_keys=True, indent=4).replace(
368                '"\n}', '",\n}'
369            )
370        )
371    )
372    logging.info("Done. Wrote %s", known_shas_file.relative_to(workspace_dir))
373
374
375if __name__ == "__main__":
376    main()