fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #103046 - JanBeh:PR_clarify_cmp_terminology, r=workingjubilee3bcad65fbf6

master
6.5 KiB191 linesraw
1#!/usr/bin/env python3
2# Simpler reimplementation of Android's sdkmanager
3# Extra features of this implementation are pinning and mirroring
4
5import argparse
6import hashlib
7import os
8import subprocess
9import tempfile
10import urllib.request
11import xml.etree.ElementTree as ET
12
13# These URLs are the Google repositories containing the list of available
14# packages and their versions. The list has been generated by listing the URLs
15# fetched while executing `tools/bin/sdkmanager --list`
16BASE_REPOSITORY = "https://dl.google.com/android/repository/"
17REPOSITORIES = [
18    "sys-img/android/sys-img2-1.xml",
19    "sys-img/android-wear/sys-img2-1.xml",
20    "sys-img/android-wear-cn/sys-img2-1.xml",
21    "sys-img/android-tv/sys-img2-1.xml",
22    "sys-img/google_apis/sys-img2-1.xml",
23    "sys-img/google_apis_playstore/sys-img2-1.xml",
24    "addon2-1.xml",
25    "glass/addon2-1.xml",
26    "extras/intel/addon2-1.xml",
27    "repository2-1.xml",
28]
29
30# Available hosts: linux, macosx and windows
31HOST_OS = "linux"
32
33# Mirroring options
34MIRROR_BUCKET = "rust-lang-ci-mirrors"
35MIRROR_BUCKET_REGION = "us-west-1"
36MIRROR_BASE_DIR = "rustc/android/"
37
38class Package:
39    def __init__(self, path, url, sha1, deps=None):
40        if deps is None:
41            deps = []
42        self.path = path.strip()
43        self.url = url.strip()
44        self.sha1 = sha1.strip()
45        self.deps = deps
46
47    def download(self, base_url):
48        _, file = tempfile.mkstemp()
49        url = base_url + self.url
50        subprocess.run(["curl", "-o", file, url], check=True)
51        # Ensure there are no hash mismatches
52        with open(file, "rb") as f:
53            sha1 = hashlib.sha1(f.read()).hexdigest()
54            if sha1 != self.sha1:
55                raise RuntimeError(
56                    "hash mismatch for package " + self.path + ": " +
57                    sha1 + " vs " + self.sha1 + " (known good)"
58                )
59        return file
60
61    def __repr__(self):
62        return "<Package "+self.path+" at "+self.url+" (sha1="+self.sha1+")"
63
64def fetch_url(url):
65    page = urllib.request.urlopen(url)
66    return page.read()
67
68def fetch_repository(base, repo_url):
69    packages = {}
70    root = ET.fromstring(fetch_url(base + repo_url))
71    for package in root:
72        if package.tag != "remotePackage":
73            continue
74        path = package.attrib["path"]
75
76        for archive in package.find("archives"):
77            host_os = archive.find("host-os")
78            if host_os is not None and host_os.text != HOST_OS:
79                continue
80            complete = archive.find("complete")
81            url = os.path.join(os.path.dirname(repo_url), complete.find("url").text)
82            sha1 = complete.find("checksum").text
83
84            deps = []
85            dependencies = package.find("dependencies")
86            if dependencies is not None:
87                for dep in dependencies:
88                    deps.append(dep.attrib["path"])
89
90            packages[path] = Package(path, url, sha1, deps)
91            break
92
93    return packages
94
95def fetch_repositories():
96    packages = {}
97    for repo in REPOSITORIES:
98        packages.update(fetch_repository(BASE_REPOSITORY, repo))
99    return packages
100
101class Lockfile:
102    def __init__(self, path):
103        self.path = path
104        self.packages = {}
105        if os.path.exists(path):
106            with open(path) as f:
107                for line in f:
108                    path, url, sha1 = line.split(" ")
109                    self.packages[path] = Package(path, url, sha1)
110
111    def add(self, packages, name, *, update=True):
112        if name not in packages:
113            raise NameError("package not found: " + name)
114        if not update and name in self.packages:
115            return
116        self.packages[name] = packages[name]
117        for dep in packages[name].deps:
118            self.add(packages, dep, update=False)
119
120    def save(self):
121        packages = list(sorted(self.packages.values(), key=lambda p: p.path))
122        with open(self.path, "w") as f:
123            for package in packages:
124                f.write(package.path + " " + package.url + " " + package.sha1 + "\n")
125
126def cli_add_to_lockfile(args):
127    lockfile = Lockfile(args.lockfile)
128    packages = fetch_repositories()
129    for package in args.packages:
130        lockfile.add(packages, package)
131    lockfile.save()
132
133def cli_update_mirror(args):
134    lockfile = Lockfile(args.lockfile)
135    for package in lockfile.packages.values():
136        path = package.download(BASE_REPOSITORY)
137        subprocess.run([
138            "aws", "s3", "mv", path,
139            "s3://" + MIRROR_BUCKET + "/" + MIRROR_BASE_DIR + package.url,
140            "--profile=" + args.awscli_profile,
141        ], check=True)
142
143def cli_install(args):
144    lockfile = Lockfile(args.lockfile)
145    for package in lockfile.packages.values():
146        # Download the file from the mirror into a temp file
147        url = "https://" + MIRROR_BUCKET + ".s3-" + MIRROR_BUCKET_REGION + \
148              ".amazonaws.com/" + MIRROR_BASE_DIR
149        downloaded = package.download(url)
150        # Extract the file in a temporary directory
151        extract_dir = tempfile.mkdtemp()
152        subprocess.run([
153            "unzip", "-q", downloaded, "-d", extract_dir,
154        ], check=True)
155        # Figure out the prefix used in the zip
156        subdirs = [d for d in os.listdir(extract_dir) if not d.startswith(".")]
157        if len(subdirs) != 1:
158            raise RuntimeError("extracted directory contains more than one dir")
159        # Move the extracted files in the proper directory
160        dest = os.path.join(args.dest, package.path.replace(";", "/"))
161        os.makedirs("/".join(dest.split("/")[:-1]), exist_ok=True)
162        os.rename(os.path.join(extract_dir, subdirs[0]), dest)
163        os.unlink(downloaded)
164
165def cli():
166    parser = argparse.ArgumentParser()
167    subparsers = parser.add_subparsers()
168
169    add_to_lockfile = subparsers.add_parser("add-to-lockfile")
170    add_to_lockfile.add_argument("lockfile")
171    add_to_lockfile.add_argument("packages", nargs="+")
172    add_to_lockfile.set_defaults(func=cli_add_to_lockfile)
173
174    update_mirror = subparsers.add_parser("update-mirror")
175    update_mirror.add_argument("lockfile")
176    update_mirror.add_argument("--awscli-profile", default="default")
177    update_mirror.set_defaults(func=cli_update_mirror)
178
179    install = subparsers.add_parser("install")
180    install.add_argument("lockfile")
181    install.add_argument("dest")
182    install.set_defaults(func=cli_install)
183
184    args = parser.parse_args()
185    if not hasattr(args, "func"):
186        print("error: a subcommand is required (see --help)")
187        exit(1)
188    args.func(args)
189
190if __name__ == "__main__":
191    cli()