fork/rules_rust
git clone https://git.julia.blue/fork/rules_rust
beebf3b0
main
1#!/usr/bin/env python3.11 2"""A script for generating the `//rust:known_shas.bzl` file.""" 3 4import json 5import logging 6import os 7import shutil 8import subprocess 9import sys 10import tempfile 11import tomllib 12from pathlib import Path 13from typing import Any ,Dict ,Sequence 14 15KNOWN_SHAS_TEMPLATE = """\ 16\"\"\"A module containing a mapping of Rust tools to checksums 17 18This is a generated file -- see //util/fetch_shas 19\"\"\" 20 21FILE_KEY_TO_SHA = {} 22""" 23 24 25def download_manifest_data ( 26stable_versions :Sequence [str ],nightly_versions :Sequence [str ],output_dir :Path 27)-> Dict [str ,Sequence [Dict [str ,Any ]]]: 28"""Download and deserialize `channel-rust-*.toml` files for the requested versions 29 30Args: 31stable_versions: A list of Rust versions. E.g. `1.81.0` 32nightly_versions: A list of nightly iso dates. E.g. `2024-09-05` 33output_dir: The location where the intermediate files should be written. 34 35Returns: 36A mapping of channel name (`stable`, `nightly`) to deserialized data. 37""" 38 39output_dir .mkdir (exist_ok = True ,parents = True ) 40curl_config = output_dir / "curl_config.txt" 41curl_config_lines = [ 42"--fail" , 43"--parallel" , 44"--silent" , 45"--create-dirs" , 46 ] 47 48stable_files = {} 49for version in stable_versions : 50output = output_dir / f"channel-rust- { version } .toml" 51curl_config_lines .extend ( 52 [ 53f"--output { output } " , 54f"--url https://static.rust-lang.org/dist/channel-rust- { version } .toml" , 55 ] 56 ) 57stable_files [version ]= output 58nightly_files = {} 59for version in nightly_versions : 60output = output_dir / version / "channel-rust-nightly.toml" 61curl_config_lines .extend ( 62 [ 63f"--output { output } " , 64f"--url https://static.rust-lang.org/dist/ { version } /channel-rust-nightly.toml" , 65 ] 66 ) 67nightly_files [version ]= output 68 69curl_config .write_text ("\n" .join (curl_config_lines ),encoding = "utf-8" ) 70 71logging .info ("Downloading data..." ) 72subprocess .run ( 73 [ 74"curl" , 75"--config" , 76curl_config , 77 ], 78check = True , 79 ) 80logging .info ("Done." ) 81 82logging .info ( 83"Deserializing %s tomls..." , 84len (list (stable_files .keys ())+ list (nightly_files .keys ())), 85 ) 86for collection in (stable_files ,nightly_files ): 87for version ,file in collection .items (): 88data = file .read_text (encoding = "utf-8" ) 89assert data 90try : 91collection [version ]= tomllib .loads (data ) 92except : 93logging .error ("Failed to parse toml: %s\n%s" ,file ,data ) 94raise 95 96logging .info ("Done." ) 97return { 98"stable" :stable_files , 99"nightly" :nightly_files , 100 } 101 102 103def download_direct_sha256s ( 104artifacts :Sequence [str ],output_dir :Path 105)-> Dict [str ,str ]: 106"""_summary_ 107 108This function is mostly here for backward compatibility. There are artifacts 109referenced by the `channel-rust-*.toml` files which are marked as `available: false` 110and probably intented to not be downloaded. But for now this is ignored and instead 111a collection of artifacts whose hash data could not be found is explicitly checked 112by trying to download the `.sha256` files directly. A 404 indicates the artifact 113genuinely does not exist and anything else we find is extra data we can retain 114in `known_shas.bzl`. 115 116Args: 117artifacts: A list of paths within `https://static.rust-lang.org/dist` to download. 118output_dir: The location where the intermediate files should be written. 119 120Returns: 121A mapping of `artifacts` entries to their sha256 value. 122""" 123output_dir .mkdir (exist_ok = True ,parents = True ) 124status_config = output_dir / "status" / "curl_config.txt" 125sha256_config = output_dir / "sha256" / "curl_config.txt" 126common_config_lines = [ 127"--parallel" , 128"--create-dirs" , 129 ] 130status_config_lines = common_config_lines + [ 131"--silent" , 132"--head" , 133 ] 134 135statuses = {} 136for url_path in artifacts : 137output = status_config .parent / f" { url_path } .sha256.status" 138status_config_lines .extend ( 139 [ 140f"--output { output } " , 141f"--url https://static.rust-lang.org/dist/ { url_path } .sha256" , 142 ] 143 ) 144statuses [url_path ]= output 145 146status_config .parent .mkdir (exist_ok = True ,parents = True ) 147status_config .write_text ("\n" .join (status_config_lines ),encoding = "utf-8" ) 148 149logging .info ("Checking for %s missing artifacts..." ,len (statuses )) 150result = subprocess .run ( 151 [ 152"curl" , 153"--config" , 154status_config , 155 ], 156check = True , 157 ) 158logging .info ("Done." ) 159 160checksums = {} 161missing = [] 162for url_path ,status_file in statuses .items (): 163if not status_file .exists (): 164missing .append (f"https://static.rust-lang.org/dist/ { url_path } .sha256" ) 165continue 166 167if status_file .read_text (encoding = "utf-8" ).startswith ("HTTP/2 404" ): 168continue 169 170checksums [url_path ]= sha256_config .parent / f" { url_path } .sha256" 171 172if missing : 173logging .warning ( 174"Status not found for %s artifacts:\n%s" , 175len (missing ), 176json .dumps (sorted (missing ),indent = 2 ), 177 ) 178 179logging .info ("Downloading %s missing artifacts..." ,len (checksums )) 180sha256_config_lines = common_config_lines + [ 181"--verbose" , 182 ] 183for url_path ,output in checksums .items (): 184sha256_config_lines .extend ( 185 [ 186f"--output { output } " , 187f"--url https://static.rust-lang.org/dist/ { url_path } .sha256" , 188 ] 189 ) 190 191sha256_config .parent .mkdir (exist_ok = True ,parents = True ) 192sha256_config .write_text ("\n" .join (sha256_config_lines ),encoding = "utf-8" ) 193result = subprocess .run ( 194 [ 195"curl" , 196"--config" , 197sha256_config , 198 ], 199encoding = "utf-8" , 200check = False , 201stderr = subprocess .STDOUT , 202stdout = subprocess .PIPE , 203 ) 204 205if result .returncode : 206result .stdout ,file = sys .stderr ) 207sys .exit (result .returncode ) 208 209logging .info ("Done." ) 210 211return { 212name :file .read_text (encoding = "utf-8" ).split (" " )[0 ].strip () 213for name ,file in checksums .items () 214 } 215 216 217def load_data (file :Path )-> Sequence [str ]: 218"""Load a `fetch_shas_*.txt` file 219 220Args: 221file: The file to load 222 223Returns: 224A list of the file's contents. 225""" 226data = [] 227for line in file .read_text (encoding = "utf-8" ).splitlines (): 228text = line .strip () 229if not text : 230continue 231data .append (text ) 232 233return sorted (set (data )) 234 235 236def main ()-> None : 237"""The main entrypoint.""" 238if "BUILD_WORKSPACE_DIRECTORY" in os .environ : 239workspace_dir = Path (os .environ ["BUILD_WORKSPACE_DIRECTORY" ]) 240else : 241workspace_dir = Path (__file__ ).parent .parent .parent 242 243logging .basicConfig ( 244level = ( 245logging .INFO 246if "RULES_RUST_FETCH_SHAS_DEBUG" not in os .environ 247else logging .DEBUG 248 ), 249format = "%(asctime)s - %(levelname)s - %(message)s" , 250datefmt = "%Y-%m-%d %H:%M:%S" , 251 ) 252logging .info ("Fetching known sha256 data..." ) 253 254tools = load_data (workspace_dir / "util/fetch_shas/fetch_shas_TOOLS.txt" ) 255host_tools = load_data (workspace_dir / "util/fetch_shas/fetch_shas_HOST_TOOLS.txt" ) 256targets = load_data (workspace_dir / "util/fetch_shas/fetch_shas_TARGETS.txt" ) 257stable_versions = load_data ( 258workspace_dir / "util/fetch_shas/fetch_shas_VERSIONS.txt" 259 ) 260nightly_iso_dates = load_data ( 261workspace_dir / "util/fetch_shas/fetch_shas_NIGHTLY_ISO_DATES.txt" 262 ) 263 264# Allow this directory to be optionally cleaned up. 265tmp_dir = Path (tempfile .mkdtemp (prefix = "rules-rust-fetch-shas-" )) 266logging .debug ("Temp dir: %s" ,tmp_dir ) 267try : 268manifest_data = download_manifest_data ( 269stable_versions = stable_versions , 270nightly_versions = nightly_iso_dates , 271output_dir = tmp_dir , 272 ) 273 274file_key_to_sha = {} 275 276artifacts = [] 277 278logging .info ("Parsing artifacts..." ) 279for channel ,versioned_info in manifest_data .items (): 280for version ,info in versioned_info .items (): 281if channel == "stable" : 282tool_tpl = f"{{pkg}}- { version } -{{target}}.{{ext}}" 283host_tool_tpl = f"{{pkg}}- { version } .{{ext}}" 284else : 285tool_tpl = f" { version } /{{pkg}}- { channel } -{{target}}.{{ext}}" 286host_tool_tpl = f" { version } /{{pkg}}- { channel } .{{ext}}" 287 288# Artifacts are commonly referred to with an internal (to the Rust org) name. 289# In order to correctly evaluate whether or not a particular `channel-rust-*.toml` 290# has an artifact, a mapping is made from what the internal Rust names are to 291# what they're advertised as externally (e.g. what users would use when trying 292# to download files directly). 293renames = { 294"clippy-preview" :"clippy" , 295"llvm-tools-preview" :"llvm-tools" , 296"rustfmt-preview" :"rustfmt" , 297 } 298if "renames" in info : 299for pkg ,rename in info ["renames" ].items (): 300if "to" in rename : 301renames [rename ["to" ]]= pkg 302 303logging .debug ("Renames (%s %s): %s" ,channel ,version ,renames ) 304for pkg ,pkg_data in info ["pkg" ].items (): 305pkg_name = renames .get (pkg ,pkg ) 306if pkg_name in tools : 307tool_template = tool_tpl 308elif pkg_name in host_tools : 309tool_template = host_tool_tpl 310else : 311logging .debug ("Skipping %s %s %s" ,channel ,version ,pkg_name ) 312continue 313 314drain_targets = list (targets ) 315for target ,target_data in pkg_data ["target" ].items (): 316# One variant of the template has an extra format string field. 317# Replace is used to account for that. 318template = tool_template .replace ("{target}" ,target ) 319 320if target not in drain_targets : 321continue 322 323drain_targets .remove (target ) 324 325if "hash" in target_data : 326file_key_to_sha [ 327template .format (pkg = pkg_name ,ext = "tar.gz" ) 328 ]= target_data ["hash" ] 329if "xz_hash" in target_data : 330file_key_to_sha [ 331template .format (pkg = pkg_name ,ext = "tar.xz" ) 332 ]= target_data ["xz_hash" ] 333 334# If an artifact is not advertised to be available for a particular 335# target then we track this and see if the sha256 data can be 336# downloaded directly. 337for target in drain_targets : 338# One variant of the template has an extra format string field. 339# Replace is used to account for that. 340template = tool_template .replace ("{target}" ,target ) 341 342# See if we can download the file directly. 343artifacts .extend ( 344 [ 345template .format (pkg = pkg_name ,ext = "tar.gz" ), 346template .format (pkg = pkg_name ,ext = "tar.xz" ), 347 ] 348 ) 349 350logging .info ("Done. Identified %s artifacts." ,len (file_key_to_sha )) 351 352# Do a brute force check to find additional sha256 values. 353file_key_to_sha .update ( 354download_direct_sha256s ( 355artifacts = sorted (set (artifacts )), 356output_dir = tmp_dir / "retries" , 357 ) 358 ) 359 360finally : 361if not "RULES_RUST_FETCH_SHAS_DEBUG" in os .environ : 362shutil .rmtree (tmp_dir ) 363 364known_shas_file = workspace_dir / "rust/known_shas.bzl" 365known_shas_file .write_text ( 366KNOWN_SHAS_TEMPLATE .format ( 367json .dumps (file_key_to_sha ,sort_keys = True ,indent = 4 ).replace ( 368'"\n}' ,'",\n}' 369 ) 370 ) 371 ) 372logging .info ("Done. Wrote %s" ,known_shas_file .relative_to (workspace_dir )) 373 374 375if __name__ == "__main__" : 376main ()