fork/nix-minecraft

git clone https://git.julia.blue/fork/nix-minecraft

Nathan Regnerneoforge-servers: initcc15499

lazymc
8.9 KiB281 linesraw
1#!/usr/bin/env nix-shell
2#!nix-shell -i python3 -p python3Packages.packaging python3Packages.requests python3Packages.requests-cache
3
4import argparse
5import base64
6import concurrent.futures
7import json
8import re
9import subprocess
10import zipfile
11from collections import defaultdict
12from pathlib import Path
13from typing import Any, Dict, List, NotRequired, TypedDict
14
15import requests
16import requests_cache
17from packaging.version import InvalidVersion, Version
18from requests.adapters import HTTPAdapter, Retry
19
20# Versions before 20.5 do not always support the "fat jar" feature. As such,
21# they always try to download server mappings, and there's no way to bypass it.
22MIN_SUPPORTED_VERSION = Version("20.4.240")
23
24MINECRAFT_MANIFEST = "https://launchermeta.mojang.com/mc/game/version_manifest_v2.json"
25NEOFORGE_API = (
26    "https://maven.neoforged.net/api/maven/versions/releases/net/neoforged/neoforge"
27)
28NEOFORGE_MAVEN = "https://maven.neoforged.net/releases/net/neoforged/neoforge"
29
30TIMEOUT = 5
31RETRIES = 5
32THREADS = 8
33
34
35class FetchUrl(TypedDict):
36    name: NotRequired[str]
37    url: str
38    hash: str
39
40
41class GameVersionLock(TypedDict):
42    mappings: FetchUrl
43
44
45class LoaderLock(TypedDict):
46    src: FetchUrl
47    libraries: List[str]
48
49
50# game version -> build version -> version details
51LoaderLocks = Dict[str, Dict[str, LoaderLock]]
52
53
54class TimeoutHTTPAdapter(HTTPAdapter):
55    def __init__(self, *args, **kwargs):
56        self.timeout = TIMEOUT
57        if "timeout" in kwargs:
58            self.timeout = kwargs["timeout"]
59            del kwargs["timeout"]
60        super().__init__(*args, **kwargs)
61
62    def send(self, request, **kwargs):
63        timeout = kwargs.get("timeout")
64        if timeout is None:
65            kwargs["timeout"] = self.timeout
66        return super().send(request, **kwargs)
67
68
69def make_client() -> requests_cache.CachedSession:
70    # configure a cache for faster local development
71    # mostly for ".sha256" files that should never change
72    client = requests_cache.CachedSession(backend="filesystem", cache_control=True)
73    retries = Retry(
74        total=RETRIES, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504]
75    )
76    client.mount("https://", TimeoutHTTPAdapter(max_retries=retries))
77    return client
78
79
80def sri_hash(alg: str, hex: str):
81    return f"{alg}-{base64.b64encode(bytes.fromhex(hex)).decode('utf-8')}"
82
83
84def fetch_game_versions(client: requests_cache.CachedSession) -> Dict[str, str]:
85    print("Fetching game versions")
86    response = client.get(MINECRAFT_MANIFEST, expire_after=requests_cache.DO_NOT_CACHE)
87    response.raise_for_status()
88    data = response.json()
89    return {v["id"]: v["url"] for v in data["versions"]}
90
91
92def fetch_mappings_hash(client: requests_cache.CachedSession, url: str):
93    print(f"Fetching manifest: {url}")
94    response = client.get(url)
95    response.raise_for_status()
96    data = response.json()
97    server_mappings = data["downloads"]["server_mappings"]
98    return GameVersionLock(
99        mappings=FetchUrl(
100            name=f"{data['id']}-server-mappings.txt",
101            url=str(server_mappings["url"]),
102            hash=sri_hash("sha1", server_mappings["sha1"]),
103        ),
104    )
105
106
107def fetch_installer_hash(client: requests.Session, version: str):
108    url = f"{NEOFORGE_MAVEN}/{version}/neoforge-{version}-installer.jar"
109    hash_url = f"{url}.sha256"
110    response = client.get(hash_url)
111    response.raise_for_status()
112    return FetchUrl(
113        url=url,
114        hash=sri_hash("sha256", response.text),
115    )
116
117
118def fetch_library_hashes(src: FetchUrl) -> Dict[str, FetchUrl]:
119    # the installer jar is used by the build derivation, so we might as well
120    # use nix to fetch/cache it ahead of time
121    proc = subprocess.run(
122        ["nix-prefetch-url", src["url"], src["hash"], "--print-path"],
123        check=True,
124        encoding="UTF-8",
125        stdout=subprocess.PIPE,
126        stderr=subprocess.STDOUT,
127    )
128    store_path = proc.stdout.splitlines()[1]
129
130    def library_src(library: Dict[str, Any]):
131        artifact = library["downloads"]["artifact"]
132        return FetchUrl(
133            url=artifact["url"],
134            hash=sri_hash("sha1", artifact["sha1"]),
135        )
136
137    with zipfile.ZipFile(store_path, "r") as zf:
138        with zf.open("install_profile.json", "r") as f:
139            profile_data = json.load(f)
140            libraries = profile_data["libraries"]
141
142        with zf.open("version.json", "r") as f:
143            version_data = json.load(f)
144            libraries += version_data["libraries"]
145
146    return {str(lib["name"]): library_src(lib) for lib in libraries}
147
148
149def fetch_loader_versions(
150    client: requests_cache.CachedSession,
151) -> Dict[str, List[str]]:  # game version -> build versions
152    print("Fetching installer versions")
153    response = client.get(NEOFORGE_API, expire_after=requests_cache.DO_NOT_CACHE)
154    response.raise_for_status()
155
156    versions = defaultdict(list)
157    data = response.json()
158    for version in data["versions"]:
159        try:
160            version = Version(version)
161        except InvalidVersion:
162            print(f"Skipping unparseable version: {version}")
163            continue
164        if version.is_prerelease:
165            print(f"Skipping pre-release version: {version}")
166            continue
167        if MIN_SUPPORTED_VERSION > version:
168            print(f"Skipping unsupported version: {version}")
169            continue
170
171        # first two digits == 1.x game version
172        versions[f"1.{version.major}.{version.minor}".removesuffix(".0")].append(
173            str(version)
174        )
175    return versions
176
177
178def main(
179    loader_versions: LoaderLocks,
180    game_versions: Dict[str, GameVersionLock],
181    library_versions: Dict[str, FetchUrl],
182    version_regex,
183    client,
184):
185    print("Starting fetch")
186
187    game_manifest = fetch_game_versions(client)
188    loader_manifest = fetch_loader_versions(client)
189
190    to_fetch = []
191
192    for game_version, build_versions in loader_manifest.items():
193        if game_version not in game_versions:
194            game_versions[game_version] = fetch_mappings_hash(
195                client, game_manifest[game_version]
196            )
197
198        for build_version in build_versions:
199            if re.match(version_regex, build_version) is None:
200                print(f"Skip fetching {build_version}: does not match --version")
201                continue
202            if (
203                game_version not in loader_versions
204                or build_version not in loader_versions[game_version]
205            ):
206                to_fetch.append((game_version, build_version))
207
208    print(f"Fetching {len(to_fetch)} loader versions...")
209
210    def fetch_build(versions: tuple[str, str]):
211        game_version, version = versions
212        print(f"Fetching {version}")
213        installer = fetch_installer_hash(client, version)
214        return game_version, version, installer, fetch_library_hashes(installer)
215
216    try:
217        with concurrent.futures.ThreadPoolExecutor(max_workers=THREADS) as p:
218            for game_version, version, src, library_srcs in p.map(
219                fetch_build, to_fetch
220            ):
221                if game_version not in loader_versions:
222                    loader_versions[game_version] = {}
223                loader_versions[game_version][version] = LoaderLock(
224                    libraries=sorted(library_srcs.keys()),
225                    src=src,
226                )
227                library_versions |= library_srcs
228    except KeyboardInterrupt:
229        print("Cancelled fetching. Writing and exiting")
230
231    return (loader_versions, game_versions, library_versions)
232
233
234if __name__ == "__main__":
235    parser = argparse.ArgumentParser()
236    parser.add_argument("--version", type=str, default=r".*", required=False)
237    args = parser.parse_args()
238
239    folder = Path(__file__).parent
240    loader_path = folder / "loader_locks.json"
241    game_path = folder / "game_locks.json"
242    library_path = folder / "library_locks.json"
243    with (
244        open(loader_path, "r") as loader_locks,
245        open(game_path, "r") as game_locks,
246        open(library_path, "r") as library_locks,
247    ):
248        loader_versions = (
249            {} if loader_path.stat().st_size == 0 else json.load(loader_locks)
250        )
251        game_versions = {} if game_path.stat().st_size == 0 else json.load(game_locks)
252        library_versions = (
253            {} if library_path.stat().st_size == 0 else json.load(library_locks)
254        )
255
256    (loader_versions, game_versions, library_versions) = main(
257        loader_versions,
258        game_versions,
259        library_versions,
260        args.version,
261        make_client(),
262    )
263
264    with (
265        open(loader_path, "w") as loader_locks,
266        open(game_path, "w") as game_locks,
267        open(library_path, "w") as library_locks,
268    ):
269        json.dump(
270            loader_versions,
271            loader_locks,
272            indent=2,
273            sort_keys=True,
274        )
275        json.dump(game_versions, game_locks, indent=2, sort_keys=True)
276        json.dump(
277            library_versions,
278            library_locks,
279            indent=2,
280            sort_keys=True,
281        )