fork/rust

git clone https://git.julia.blue/fork/rust

borsAuto merge of #71868 - Mark-Simulacrum:stable-1.43.1, r=pietroalbini8d69840ab92

1.43.1
6.5 KiB192 linesraw
1#!/usr/bin/env python3
2# Simpler reimplementation of Android's sdkmanager
3# Extra features of this implementation are pinning and mirroring
4
5# These URLs are the Google repositories containing the list of available
6# packages and their versions. The list has been generated by listing the URLs
7# fetched while executing `tools/bin/sdkmanager --list`
8BASE_REPOSITORY = "https://dl.google.com/android/repository/"
9REPOSITORIES = [
10    "sys-img/android/sys-img2-1.xml",
11    "sys-img/android-wear/sys-img2-1.xml",
12    "sys-img/android-wear-cn/sys-img2-1.xml",
13    "sys-img/android-tv/sys-img2-1.xml",
14    "sys-img/google_apis/sys-img2-1.xml",
15    "sys-img/google_apis_playstore/sys-img2-1.xml",
16    "addon2-1.xml",
17    "glass/addon2-1.xml",
18    "extras/intel/addon2-1.xml",
19    "repository2-1.xml",
20]
21
22# Available hosts: linux, macosx and windows
23HOST_OS = "linux"
24
25# Mirroring options
26MIRROR_BUCKET = "rust-lang-ci-mirrors"
27MIRROR_BUCKET_REGION = "us-west-1"
28MIRROR_BASE_DIR = "rustc/android/"
29
30import argparse
31import hashlib
32import os
33import subprocess
34import sys
35import tempfile
36import urllib.request
37import xml.etree.ElementTree as ET
38
39class Package:
40    def __init__(self, path, url, sha1, deps=None):
41        if deps is None:
42            deps = []
43        self.path = path.strip()
44        self.url = url.strip()
45        self.sha1 = sha1.strip()
46        self.deps = deps
47
48    def download(self, base_url):
49        _, file = tempfile.mkstemp()
50        url = base_url + self.url
51        subprocess.run(["curl", "-o", file, url], check=True)
52        # Ensure there are no hash mismatches
53        with open(file, "rb") as f:
54            sha1 = hashlib.sha1(f.read()).hexdigest()
55            if sha1 != self.sha1:
56                raise RuntimeError(
57                    "hash mismatch for package " + self.path + ": " +
58                    sha1 + " vs " + self.sha1 + " (known good)"
59                )
60        return file
61
62    def __repr__(self):
63        return "<Package "+self.path+" at "+self.url+" (sha1="+self.sha1+")"
64
65def fetch_url(url):
66    page = urllib.request.urlopen(url)
67    return page.read()
68
69def fetch_repository(base, repo_url):
70    packages = {}
71    root = ET.fromstring(fetch_url(base + repo_url))
72    for package in root:
73        if package.tag != "remotePackage":
74            continue
75        path = package.attrib["path"]
76
77        for archive in package.find("archives"):
78            host_os = archive.find("host-os")
79            if host_os is not None and host_os.text != HOST_OS:
80                continue
81            complete = archive.find("complete")
82            url = os.path.join(os.path.dirname(repo_url), complete.find("url").text)
83            sha1 = complete.find("checksum").text
84
85            deps = []
86            dependencies = package.find("dependencies")
87            if dependencies is not None:
88                for dep in dependencies:
89                    deps.append(dep.attrib["path"])
90
91            packages[path] = Package(path, url, sha1, deps)
92            break
93
94    return packages
95
96def fetch_repositories():
97    packages = {}
98    for repo in REPOSITORIES:
99        packages.update(fetch_repository(BASE_REPOSITORY, repo))
100    return packages
101
102class Lockfile:
103    def __init__(self, path):
104        self.path = path
105        self.packages = {}
106        if os.path.exists(path):
107            with open(path) as f:
108                for line in f:
109                    path, url, sha1 = line.split(" ")
110                    self.packages[path] = Package(path, url, sha1)
111
112    def add(self, packages, name, *, update=True):
113        if name not in packages:
114            raise NameError("package not found: " + name)
115        if not update and name in self.packages:
116            return
117        self.packages[name] = packages[name]
118        for dep in packages[name].deps:
119            self.add(packages, dep, update=False)
120
121    def save(self):
122        packages = list(sorted(self.packages.values(), key=lambda p: p.path))
123        with open(self.path, "w") as f:
124            for package in packages:
125                f.write(package.path + " " + package.url + " " + package.sha1 + "\n")
126
127def cli_add_to_lockfile(args):
128    lockfile = Lockfile(args.lockfile)
129    packages = fetch_repositories()
130    for package in args.packages:
131        lockfile.add(packages, package)
132    lockfile.save()
133
134def cli_update_mirror(args):
135    lockfile = Lockfile(args.lockfile)
136    for package in lockfile.packages.values():
137        path = package.download(BASE_REPOSITORY)
138        subprocess.run([
139            "aws", "s3", "mv", path,
140            "s3://" + MIRROR_BUCKET + "/" + MIRROR_BASE_DIR + package.url,
141            "--profile=" + args.awscli_profile,
142        ], check=True)
143
144def cli_install(args):
145    lockfile = Lockfile(args.lockfile)
146    for package in lockfile.packages.values():
147        # Download the file from the mirror into a temp file
148        url = "https://" + MIRROR_BUCKET + ".s3-" + MIRROR_BUCKET_REGION + \
149              ".amazonaws.com/" + MIRROR_BASE_DIR
150        downloaded = package.download(url)
151        # Extract the file in a temporary directory
152        extract_dir = tempfile.mkdtemp()
153        subprocess.run([
154            "unzip", "-q", downloaded, "-d", extract_dir,
155        ], check=True)
156        # Figure out the prefix used in the zip
157        subdirs = [d for d in os.listdir(extract_dir) if not d.startswith(".")]
158        if len(subdirs) != 1:
159            raise RuntimeError("extracted directory contains more than one dir")
160        # Move the extracted files in the proper directory
161        dest = os.path.join(args.dest, package.path.replace(";", "/"))
162        os.makedirs("/".join(dest.split("/")[:-1]), exist_ok=True)
163        os.rename(os.path.join(extract_dir, subdirs[0]), dest)
164        os.unlink(downloaded)
165
166def cli():
167    parser = argparse.ArgumentParser()
168    subparsers = parser.add_subparsers()
169
170    add_to_lockfile = subparsers.add_parser("add-to-lockfile")
171    add_to_lockfile.add_argument("lockfile")
172    add_to_lockfile.add_argument("packages", nargs="+")
173    add_to_lockfile.set_defaults(func=cli_add_to_lockfile)
174
175    update_mirror = subparsers.add_parser("update-mirror")
176    update_mirror.add_argument("lockfile")
177    update_mirror.add_argument("--awscli-profile", default="default")
178    update_mirror.set_defaults(func=cli_update_mirror)
179
180    install = subparsers.add_parser("install")
181    install.add_argument("lockfile")
182    install.add_argument("dest")
183    install.set_defaults(func=cli_install)
184
185    args = parser.parse_args()
186    if not hasattr(args, "func"):
187        print("error: a subcommand is required (see --help)")
188        exit(1)
189    args.func(args)
190
191if __name__ == "__main__":
192    cli()