fork/rust
git clone https://git.julia.blue/fork/rust
4560ea788cb
1.39.0
1#!/usr/bin/env python3 2# Simpler reimplementation of Android's sdkmanager 3# Extra features of this implementation are pinning and mirroring 4 5# These URLs are the Google repositories containing the list of available 6# packages and their versions. The list has been generated by listing the URLs 7# fetched while executing `tools/bin/sdkmanager --list` 8BASE_REPOSITORY = "https://dl.google.com/android/repository/" 9REPOSITORIES = [ 10 "sys-img/android/sys-img2-1.xml", 11 "sys-img/android-wear/sys-img2-1.xml", 12 "sys-img/android-wear-cn/sys-img2-1.xml", 13 "sys-img/android-tv/sys-img2-1.xml", 14 "sys-img/google_apis/sys-img2-1.xml", 15 "sys-img/google_apis_playstore/sys-img2-1.xml", 16 "addon2-1.xml", 17 "glass/addon2-1.xml", 18 "extras/intel/addon2-1.xml", 19 "repository2-1.xml", 20] 21 22# Available hosts: linux, macosx and windows 23HOST_OS = "linux" 24 25# Mirroring options 26MIRROR_BUCKET = "rust-lang-ci-mirrors" 27MIRROR_BUCKET_REGION = "us-west-1" 28MIRROR_BASE_DIR = "rustc/android/" 29 30import argparse 31import hashlib 32import os 33import subprocess 34import sys 35import tempfile 36import urllib.request 37import xml.etree.ElementTree as ET 38 39class Package: 40 def __init__(self, path, url, sha1, deps=None): 41 if deps is None: 42 deps = [] 43 self.path = path.strip() 44 self.url = url.strip() 45 self.sha1 = sha1.strip() 46 self.deps = deps 47 48 def download(self, base_url): 49 _, file = tempfile.mkstemp() 50 url = base_url + self.url 51 subprocess.run(["curl", "-o", file, url], check=True) 52 # Ensure there are no hash mismatches 53 with open(file, "rb") as f: 54 sha1 = hashlib.sha1(f.read()).hexdigest() 55 if sha1 != self.sha1: 56 raise RuntimeError( 57 "hash mismatch for package " + self.path + ": " + 58 sha1 + " vs " + self.sha1 + " (known good)" 59 ) 60 return file 61 62 def __repr__(self): 63 return "<Package "+self.path+" at "+self.url+" (sha1="+self.sha1+")" 64 65def fetch_url(url): 66 page = urllib.request.urlopen(url) 67 return page.read() 68 69def fetch_repository(base, repo_url): 70 packages = {} 71 root = ET.fromstring(fetch_url(base + repo_url)) 72 for package in root: 73 if package.tag != "remotePackage": 74 continue 75 path = package.attrib["path"] 76 77 for archive in package.find("archives"): 78 host_os = archive.find("host-os") 79 if host_os is not None and host_os.text != HOST_OS: 80 continue 81 complete = archive.find("complete") 82 url = os.path.join(os.path.dirname(repo_url), complete.find("url").text) 83 sha1 = complete.find("checksum").text 84 85 deps = [] 86 dependencies = package.find("dependencies") 87 if dependencies is not None: 88 for dep in dependencies: 89 deps.append(dep.attrib["path"]) 90 91 packages[path] = Package(path, url, sha1, deps) 92 break 93 94 return packages 95 96def fetch_repositories(): 97 packages = {} 98 for repo in REPOSITORIES: 99 packages.update(fetch_repository(BASE_REPOSITORY, repo)) 100 return packages 101 102class Lockfile: 103 def __init__(self, path): 104 self.path = path 105 self.packages = {} 106 if os.path.exists(path): 107 with open(path) as f: 108 for line in f: 109 path, url, sha1 = line.split(" ") 110 self.packages[path] = Package(path, url, sha1) 111 112 def add(self, packages, name, *, update=True): 113 if name not in packages: 114 raise NameError("package not found: " + name) 115 if not update and name in self.packages: 116 return 117 self.packages[name] = packages[name] 118 for dep in packages[name].deps: 119 self.add(packages, dep, update=False) 120 121 def save(self): 122 packages = list(sorted(self.packages.values(), key=lambda p: p.path)) 123 with open(self.path, "w") as f: 124 for package in packages: 125 f.write(package.path + " " + package.url + " " + package.sha1 + "\n") 126 127def cli_add_to_lockfile(args): 128 lockfile = Lockfile(args.lockfile) 129 packages = fetch_repositories() 130 for package in args.packages: 131 lockfile.add(packages, package) 132 lockfile.save() 133 134def cli_update_mirror(args): 135 lockfile = Lockfile(args.lockfile) 136 for package in lockfile.packages.values(): 137 path = package.download(BASE_REPOSITORY) 138 subprocess.run([ 139 "aws", "s3", "mv", path, 140 "s3://" + MIRROR_BUCKET + "/" + MIRROR_BASE_DIR + package.url, 141 "--profile=" + args.awscli_profile, 142 ], check=True) 143 144def cli_install(args): 145 lockfile = Lockfile(args.lockfile) 146 for package in lockfile.packages.values(): 147 # Download the file from the mirror into a temp file 148 url = "https://" + MIRROR_BUCKET + ".s3-" + MIRROR_BUCKET_REGION + \ 149 ".amazonaws.com/" + MIRROR_BASE_DIR 150 downloaded = package.download(url) 151 # Extract the file in a temporary directory 152 extract_dir = tempfile.mkdtemp() 153 subprocess.run([ 154 "unzip", "-q", downloaded, "-d", extract_dir, 155 ], check=True) 156 # Figure out the prefix used in the zip 157 subdirs = [d for d in os.listdir(extract_dir) if not d.startswith(".")] 158 if len(subdirs) != 1: 159 raise RuntimeError("extracted directory contains more than one dir") 160 # Move the extracted files in the proper directory 161 dest = os.path.join(args.dest, package.path.replace(";", "/")) 162 os.makedirs("/".join(dest.split("/")[:-1]), exist_ok=True) 163 os.rename(os.path.join(extract_dir, subdirs[0]), dest) 164 os.unlink(downloaded) 165 166def cli(): 167 parser = argparse.ArgumentParser() 168 subparsers = parser.add_subparsers() 169 170 add_to_lockfile = subparsers.add_parser("add-to-lockfile") 171 add_to_lockfile.add_argument("lockfile") 172 add_to_lockfile.add_argument("packages", nargs="+") 173 add_to_lockfile.set_defaults(func=cli_add_to_lockfile) 174 175 update_mirror = subparsers.add_parser("update-mirror") 176 update_mirror.add_argument("lockfile") 177 update_mirror.add_argument("--awscli-profile", default="default") 178 update_mirror.set_defaults(func=cli_update_mirror) 179 180 install = subparsers.add_parser("install") 181 install.add_argument("lockfile") 182 install.add_argument("dest") 183 install.set_defaults(func=cli_install) 184 185 args = parser.parse_args() 186 if not hasattr(args, "func"): 187 print("error: a subcommand is required (see --help)") 188 exit(1) 189 args.func(args) 190 191if __name__ == "__main__": 192 cli()