fork/rust
git clone https://git.julia.blue/fork/rust
3bcad65fbf6
master
1#!/usr/bin/env python3 2# Simpler reimplementation of Android's sdkmanager 3# Extra features of this implementation are pinning and mirroring 4 5import argparse 6import hashlib 7import os 8import subprocess 9import tempfile 10import urllib.request 11import xml.etree.ElementTree as ET 12 13# These URLs are the Google repositories containing the list of available 14# packages and their versions. The list has been generated by listing the URLs 15# fetched while executing `tools/bin/sdkmanager --list` 16BASE_REPOSITORY = "https://dl.google.com/android/repository/" 17REPOSITORIES = [ 18 "sys-img/android/sys-img2-1.xml", 19 "sys-img/android-wear/sys-img2-1.xml", 20 "sys-img/android-wear-cn/sys-img2-1.xml", 21 "sys-img/android-tv/sys-img2-1.xml", 22 "sys-img/google_apis/sys-img2-1.xml", 23 "sys-img/google_apis_playstore/sys-img2-1.xml", 24 "addon2-1.xml", 25 "glass/addon2-1.xml", 26 "extras/intel/addon2-1.xml", 27 "repository2-1.xml", 28] 29 30# Available hosts: linux, macosx and windows 31HOST_OS = "linux" 32 33# Mirroring options 34MIRROR_BUCKET = "rust-lang-ci-mirrors" 35MIRROR_BUCKET_REGION = "us-west-1" 36MIRROR_BASE_DIR = "rustc/android/" 37 38class Package: 39 def __init__(self, path, url, sha1, deps=None): 40 if deps is None: 41 deps = [] 42 self.path = path.strip() 43 self.url = url.strip() 44 self.sha1 = sha1.strip() 45 self.deps = deps 46 47 def download(self, base_url): 48 _, file = tempfile.mkstemp() 49 url = base_url + self.url 50 subprocess.run(["curl", "-o", file, url], check=True) 51 # Ensure there are no hash mismatches 52 with open(file, "rb") as f: 53 sha1 = hashlib.sha1(f.read()).hexdigest() 54 if sha1 != self.sha1: 55 raise RuntimeError( 56 "hash mismatch for package " + self.path + ": " + 57 sha1 + " vs " + self.sha1 + " (known good)" 58 ) 59 return file 60 61 def __repr__(self): 62 return "<Package "+self.path+" at "+self.url+" (sha1="+self.sha1+")" 63 64def fetch_url(url): 65 page = urllib.request.urlopen(url) 66 return page.read() 67 68def fetch_repository(base, repo_url): 69 packages = {} 70 root = ET.fromstring(fetch_url(base + repo_url)) 71 for package in root: 72 if package.tag != "remotePackage": 73 continue 74 path = package.attrib["path"] 75 76 for archive in package.find("archives"): 77 host_os = archive.find("host-os") 78 if host_os is not None and host_os.text != HOST_OS: 79 continue 80 complete = archive.find("complete") 81 url = os.path.join(os.path.dirname(repo_url), complete.find("url").text) 82 sha1 = complete.find("checksum").text 83 84 deps = [] 85 dependencies = package.find("dependencies") 86 if dependencies is not None: 87 for dep in dependencies: 88 deps.append(dep.attrib["path"]) 89 90 packages[path] = Package(path, url, sha1, deps) 91 break 92 93 return packages 94 95def fetch_repositories(): 96 packages = {} 97 for repo in REPOSITORIES: 98 packages.update(fetch_repository(BASE_REPOSITORY, repo)) 99 return packages 100 101class Lockfile: 102 def __init__(self, path): 103 self.path = path 104 self.packages = {} 105 if os.path.exists(path): 106 with open(path) as f: 107 for line in f: 108 path, url, sha1 = line.split(" ") 109 self.packages[path] = Package(path, url, sha1) 110 111 def add(self, packages, name, *, update=True): 112 if name not in packages: 113 raise NameError("package not found: " + name) 114 if not update and name in self.packages: 115 return 116 self.packages[name] = packages[name] 117 for dep in packages[name].deps: 118 self.add(packages, dep, update=False) 119 120 def save(self): 121 packages = list(sorted(self.packages.values(), key=lambda p: p.path)) 122 with open(self.path, "w") as f: 123 for package in packages: 124 f.write(package.path + " " + package.url + " " + package.sha1 + "\n") 125 126def cli_add_to_lockfile(args): 127 lockfile = Lockfile(args.lockfile) 128 packages = fetch_repositories() 129 for package in args.packages: 130 lockfile.add(packages, package) 131 lockfile.save() 132 133def cli_update_mirror(args): 134 lockfile = Lockfile(args.lockfile) 135 for package in lockfile.packages.values(): 136 path = package.download(BASE_REPOSITORY) 137 subprocess.run([ 138 "aws", "s3", "mv", path, 139 "s3://" + MIRROR_BUCKET + "/" + MIRROR_BASE_DIR + package.url, 140 "--profile=" + args.awscli_profile, 141 ], check=True) 142 143def cli_install(args): 144 lockfile = Lockfile(args.lockfile) 145 for package in lockfile.packages.values(): 146 # Download the file from the mirror into a temp file 147 url = "https://" + MIRROR_BUCKET + ".s3-" + MIRROR_BUCKET_REGION + \ 148 ".amazonaws.com/" + MIRROR_BASE_DIR 149 downloaded = package.download(url) 150 # Extract the file in a temporary directory 151 extract_dir = tempfile.mkdtemp() 152 subprocess.run([ 153 "unzip", "-q", downloaded, "-d", extract_dir, 154 ], check=True) 155 # Figure out the prefix used in the zip 156 subdirs = [d for d in os.listdir(extract_dir) if not d.startswith(".")] 157 if len(subdirs) != 1: 158 raise RuntimeError("extracted directory contains more than one dir") 159 # Move the extracted files in the proper directory 160 dest = os.path.join(args.dest, package.path.replace(";", "/")) 161 os.makedirs("/".join(dest.split("/")[:-1]), exist_ok=True) 162 os.rename(os.path.join(extract_dir, subdirs[0]), dest) 163 os.unlink(downloaded) 164 165def cli(): 166 parser = argparse.ArgumentParser() 167 subparsers = parser.add_subparsers() 168 169 add_to_lockfile = subparsers.add_parser("add-to-lockfile") 170 add_to_lockfile.add_argument("lockfile") 171 add_to_lockfile.add_argument("packages", nargs="+") 172 add_to_lockfile.set_defaults(func=cli_add_to_lockfile) 173 174 update_mirror = subparsers.add_parser("update-mirror") 175 update_mirror.add_argument("lockfile") 176 update_mirror.add_argument("--awscli-profile", default="default") 177 update_mirror.set_defaults(func=cli_update_mirror) 178 179 install = subparsers.add_parser("install") 180 install.add_argument("lockfile") 181 install.add_argument("dest") 182 install.set_defaults(func=cli_install) 183 184 args = parser.parse_args() 185 if not hasattr(args, "func"): 186 print("error: a subcommand is required (see --help)") 187 exit(1) 188 args.func(args) 189 190if __name__ == "__main__": 191 cli()