wei/Launcher
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add module that about java runtime management to core_lib.

Browse Source
This commit is contained in:
Wei Hong
2026-07-13 22:06:39 +08:00
parent 648cde6e33
commit bd5a1e8974
11 changed files with 1034 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core_lib/game/asset.py
+176 -47
View File
@@ -2,18 +2,20 @@ import json
import logging
import shutil
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
import requests
from core_lib.common.common import FileObject, download_multiple_files
from core_lib.common.exception import VersionDataKeyNotFoundException, AssetManifestFetchException, \
HashMismatchException, AssetManifestSaveException, AssetDataKeyNotFoundException
HashMismatchException, AssetManifestSaveException, AssetDataKeyNotFoundException, DownloadException
from core_lib.game.version_info import find_useful_part_in_specific_version_manifest
MOJANG_RESOURCE_ENDPOINT = "https://resources.download.minecraft.net/{beginning}/{hash}"
ASSET_DOWNLOAD_MAX_WORKERS = 20
ASSET_CHECK_MAX_WORKERS = 20
logger = logging.getLogger("Launcher.CoreLib")
@@ -101,12 +103,72 @@ def fetch_and_save_asset_manifest(target_version_manifest: dict, destination: Pa
"Error saving version manifest:\n{}".format(e)
)
def collect_assets_from_manifest(asset_id: str, manifest: dict, assets_dir: Path) -> tuple[list[FileObject], list[dict]]:
def _check_asset(asset_id: str, asset_data: dict, relative_path: str, assets_dir: Path, virtual: bool, no_hash_check: bool=False)\
-> tuple[FileObject | None, dict | None]:
"""
Collect assets from asset manifest.
:param asset_id:
:param manifest:
Check if asset exists.
:param asset_data:
:param relative_path:
:param assets_dir:
:param virtual:
:return:
asset_file: FileObject
legacy_context: dict
"""
hash_raw = asset_data.get("hash", "")
expected_size = asset_data.get("size")
if not hash_raw:
raise AssetDataKeyNotFoundException(
"Asset that inside assets section (objects) missing hash."
)
beginning = hash_raw[:2]
full_path = Path(assets_dir, "objects", beginning, hash_raw) # For newer assets
full_url = MOJANG_RESOURCE_ENDPOINT.format(beginning=beginning, hash=hash_raw)
file = FileObject(path=full_path, sha1=hash_raw, url=full_url)
needs_download = True
if full_path.exists() and full_path.is_file():
if no_hash_check and expected_size is not None:
needs_download = full_path.stat().st_size != expected_size
else:
needs_download = not file.verify_sha1(hash_raw)
if not needs_download:
suffix = " (no_hash_check=True)" if no_hash_check else ""
logger.debug(
"Asset hash {} (Aka {}) already exists.{}".format(
hash_raw,
relative_path,
suffix,
)
)
legacy_context = None
if virtual:
legacy_context = {
"file": file,
"target": full_path if virtual else None,
"sha1": hash_raw,
"destination": Path(assets_dir, "virtual", asset_id, relative_path) if virtual else None,
"size": expected_size,
}
return file if needs_download else None, legacy_context
def collect_assets_from_manifest(manifest: dict, asset_id: str, assets_dir: Path,
max_workers=ASSET_CHECK_MAX_WORKERS,
no_hash_check=False) -> tuple[list[FileObject], list[dict]]:
"""
Collect assets from asset manifest. (Only return items that does not exist)
:param no_hash_check:
:param manifest:
:param asset_id:
:param assets_dir:
:param max_workers:
:return:
assets: list[FileObject]
legacy_assets: list[dict] (Contains target and destination of the legacy asset)
@@ -122,64 +184,51 @@ def collect_assets_from_manifest(asset_id: str, manifest: dict, assets_dir: Path
"Assets section (objects) not found in asset manifest."
)
for relative_path in objects:
data = objects[relative_path]
hash_raw = data.get("hash", "")
if not hash_raw:
raise AssetDataKeyNotFoundException(
"Asset that inside assets section (objects) missing hash."
with ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = [
executor.submit(
_check_asset,
asset_id, objects[relative_path], relative_path, assets_dir, virtual,
no_hash_check=no_hash_check
)
for relative_path in objects
]
beginning = hash_raw[:2]
for future in as_completed(futures):
asset_file, legacy_context = future.result()
full_path = Path(assets_dir, "objects", beginning, hash_raw) # For newer assets
full_path.parent.mkdir(parents=True, exist_ok=True)
full_url = MOJANG_RESOURCE_ENDPOINT.format(beginning=beginning, hash=hash_raw)
file = FileObject(path=full_path, sha1=hash_raw, url=full_url)
if asset_file:
files.append(asset_file)
if virtual:
# For legacy assets
full_path_old = Path(assets_dir, "virtual", asset_id, relative_path)
legacy_assets.append({
"target": full_path,
"destination": full_path_old,
"sha1": hash_raw,
})
if (full_path.exists() and full_path.is_file()) and file.verify_sha1(hash_raw):
logger.debug("Asset hash {} (Aka {}) already exists.".format(hash_raw, relative_path))
continue
files.append(file)
# Also copy the asset into legacy directory if needed.
if legacy_context:
legacy_assets.append(legacy_context)
return files, legacy_assets
def download_assets_and_copy(assets: list[FileObject], legacy_assets: list[dict], progress_callback: Callable[str]) -> list[FileObject]:
def download_assets_and_copy(assets: list[FileObject], legacy_assets: list[dict],
no_hash_check: bool=False, allow_missing=False,
progress_callback: Callable[str]=None,
redownload_callback: Callable[[list[FileObject]], tuple[bool, list[FileObject]]]=None) -> list[FileObject]:
"""
Download assets from target version's asset manifest. (Not sure why this function exist, may for compatibly)
Also, this function will copy
:param assets:
:param legacy_assets:
:param no_hash_check:
:param allow_missing:
:param progress_callback:
:param redownload_callback:
:return:
files: list[FileObject] (files that download failed)
"""
fails , _ = download_multiple_files(assets, progress_callback=progress_callback,
max_workers=ASSET_DOWNLOAD_MAX_WORKERS)
if fails:
logger.warning(
"Download failed for assets: \n"
+"\n".join([asset.posix_path for asset in fails])
)
for asset in legacy_assets:
def __worker(asset: dict):
target = asset.get("target")
destination = asset.get("destination")
sha1 = asset.get("sha1")
size = asset.get("size")
if not target or not destination:
raise ValueError(
@@ -188,18 +237,60 @@ def download_assets_and_copy(assets: list[FileObject], legacy_assets: list[dict]
# Check source file
source_asset = FileObject(path=target)
if not source_asset.exists or (sha1 and not source_asset.verify("sha1", sha1)):
if (not source_asset.exists or
(not no_hash_check and sha1 and not source_asset.verify("sha1", sha1))):
logger.warning("Target asset {} not found.".format(target))
continue
return False, None
destination.parent.mkdir(parents=True, exist_ok=True)
# Check destination file
old_asset = FileObject(path=destination)
if old_asset.exists and (sha1 and old_asset.verify("sha1", sha1)):
continue
if old_asset.exists:
if no_hash_check and size is not None:
if old_asset.path.stat().st_size == size:
logger.debug("Asset {} already exists. (no_hash_check=True)".format(target))
return True, None
elif sha1 and old_asset.verify("sha1", sha1):
logger.debug("Asset {} already exists.".format(target))
return True, None
shutil.copyfile(target, destination)
return True, destination
fails , _ = download_multiple_files(assets, progress_callback=progress_callback,
max_workers=ASSET_DOWNLOAD_MAX_WORKERS)
download_ok = len(fails) == 0
if fails and callable(redownload_callback):
download_ok, fails = redownload_callback(fails)
if not download_ok:
logger.warning(
"Download failed for assets: \n"
+"\n".join([asset.posix_path for asset in fails])
)
if not allow_missing:
return fails
# Use thread pool to handle copy file process
with ThreadPoolExecutor(max_workers=ASSET_DOWNLOAD_MAX_WORKERS) as executor:
futures = [
executor.submit(
__worker,
asset
)
for asset in legacy_assets
]
for future in as_completed(futures):
result, dest = future.result()
if result and dest:
logger.debug("Created: {}".format(dest))
return fails
@@ -248,4 +339,42 @@ def correct_assets_name_and_copy(manifest: dict, assets_dir: Path, output_dir: P
shutil.copy(file.posix_path, full_path_new.as_posix())
return missing
return missing
def download_assets(manifest: dict, assets_dir: Path, no_hash_check: bool=False,
progress_callback: Callable[[str, float], None]=None,
redownload_callback: Callable[[list[FileObject]], tuple[bool, list[FileObject]]]=None) -> list[FileObject]:
_, asset_index, _, _, _, _, _ = find_useful_part_in_specific_version_manifest(manifest)
asset_id = asset_index.get("id")
if not asset_id:
raise VersionDataKeyNotFoundException(
"Asset ID is missing."
)
index_path = Path(assets_dir, "indexes", f"{asset_id}.json")
# Download and save asset manifest
asset_manifest = fetch_and_save_asset_manifest(
manifest,
index_path,
)
# Download assets files
assets, legacy_assets = collect_assets_from_manifest(
asset_manifest,
asset_id,
assets_dir,
no_hash_check=no_hash_check,
)
fails = download_assets_and_copy(
assets,
legacy_assets,
progress_callback=progress_callback,
no_hash_check=no_hash_check,
redownload_callback=redownload_callback,
)
return fails
core_lib/game/library.py
+47 -4
View File
@@ -14,7 +14,7 @@ from .version_info import (
NATIVE_OLD_TO_NEW_ARCH, find_useful_part_in_specific_version_manifest,
)
from ..common.exception import UnsupportedPlatformException, NativeResolveException, DependencyException, \
VersionDataKeyNotFoundException
VersionDataKeyNotFoundException, DownloadException
logger = logging.getLogger("Launcher.CoreLib")
@@ -439,7 +439,8 @@ def collect_native_artifacts(specific_version_manifest):
return items
def download_libraries(artifacts, output_dir, progress_callback: Callable[[str], None]):
def download_libraries(artifacts, output_dir, progress_callback: Callable[[str, float], None]=None) \
-> tuple[list[FileObject], list[FileObject]]:
"""
Download libraries from a list that contains artifacts
:param artifacts:
@@ -450,6 +451,7 @@ def download_libraries(artifacts, output_dir, progress_callback: Callable[[str],
successes: list (artifacts that were downloaded)
"""
files = []
successes = []
for artifact in artifacts:
sha1 = artifact.get("sha1")
@@ -475,15 +477,16 @@ def download_libraries(artifacts, output_dir, progress_callback: Callable[[str],
if file.exists and (sha1 is None or file.verify("sha1", sha1)):
logger.debug(f"File {full_path} exists.")
successes.append(file)
continue
files.append(file)
fails, successes = download_multiple_files(files, progress_callback=progress_callback)
fails, finished = download_multiple_files(files, progress_callback=progress_callback)
successes.extend(finished)
return fails, successes
def unzip_natives(filtered, libraries_dir, destination: Path):
"""
Unzip native libraries from a list that contains artifacts
@@ -642,3 +645,43 @@ def generate_classpath(manifest: dict, libraries_dir: Path, core_file_path: Path
return f"{separator}".join(classes)
def download_full_libraries(manifest: dict, libraries_dir: Path, natives_dir: Path,
progress_callback: Callable[[str, float], None]=None,
redownload_callback: Callable[[list[FileObject]], tuple[bool, list[FileObject]]]=None) -> None:
# libraries
libraries = collect_library_artifacts(manifest)
# natives
natives = collect_native_artifacts(manifest)
# all
full = list(libraries)
full.extend(natives)
logger.info(
"Supported native count: {}".format(len(natives))
)
# Start download
fails, successes = download_libraries(full, libraries_dir, progress_callback=progress_callback)
download_ok = len(fails) == 0
if redownload_callback and not download_ok:
download_ok, fails = redownload_callback(fails)
if not download_ok:
raise DownloadException(
"Unable to continue process due to certain libraries download failed:\n"
+ "\n".join(file.posix_path for file in fails)
)
if download_ok:
# Start unzip natives
unzip_natives(natives, libraries_dir, natives_dir)
else:
raise DependencyException(
"Certain libraries download failed. Unable to continue unzip process.\n"
)
core_lib/game/version_info.py
+91 -16
View File
@@ -1,11 +1,13 @@
import json
import logging
import re
import warnings
from collections.abc import Callable
from pathlib import Path
import requests
from ..common.common import FileObject
from ..common.common import FileObject, download_file
from ..common.exception import VersionManifestFetchException, NoVersionFoundException, \
NoSpecifiedVersionKeyException, VersionDataKeyNotFoundException, VersionManifestSaveException, HashMismatchException
@@ -61,6 +63,43 @@ NATIVE_NEW_TO_OLD_ARCH = {
"x86_64": "64",
}
RUNTIME_PLATFORM_MAP = {
"linux": "linux",
"darwin": "mac-os",
"macos": "mac-os",
"osx": "mac-os",
"windows": "windows",
"win32": "windows",
"cygwin": "windows",
"msys": "windows",
}
RUNTIME_ARCH_EMPTY = "$EMPTY"
RUNTIME_ARCH_MAP = {
"linux": {
"i386": "i386",
"i686": "i386",
"x86": "i386",
"amd64": RUNTIME_ARCH_EMPTY,
"x86_64": RUNTIME_ARCH_EMPTY,
},
"windows": {
"i386": "x86",
"i686": "x86",
"x86": "x86",
"amd64": "x64",
"x86_64": "x64",
"aarch64": "arm64",
"arm64": "arm64",
},
"mac-os": {
"amd64": RUNTIME_ARCH_EMPTY,
"x86_64": RUNTIME_ARCH_EMPTY,
"aarch64": "arm64",
"arm64": "arm64",
},
}
logger = logging.getLogger("Launcher.CoreLib")
@@ -227,7 +266,7 @@ def get_specific_version_manifest_url_and_hash(version_manifest: dict, version_i
for version in versions:
if version.get("id") == version_id:
specific_ver_manifest_url = version.get("url", None)
specific_ver_manifest_hash = version.get("hash", None)
specific_ver_manifest_hash = version.get("sha1", None)
if specific_ver_manifest_url is None:
raise VersionManifestFetchException(
@@ -244,13 +283,13 @@ def get_specific_version_manifest_url_and_hash(version_manifest: dict, version_i
)
def fetch_specific_version_manifest(version_manifest: dict, spec_version: str) -> tuple[dict, str | None]:
def fetch_specific_version_manifest(version_manifest: dict, spec_version: str) -> tuple[bytes, str | None]:
"""
Get specific version's manifest data and hash from version manifest.
:param version_manifest:
:param spec_version:
:return:
specific_version_manifest_data: str
specific_version_manifest_data: bytes
hash: str (sha1)
"""
ver_url, sha1 = get_specific_version_manifest_url_and_hash(version_manifest, spec_version)
@@ -263,14 +302,7 @@ def fetch_specific_version_manifest(version_manifest: dict, spec_version: str) -
"Failed to fetch version manifest URL: {}\n{}".format(ver_url, e)
)
try:
data = r.json()
logger.debug("SpecificVersionManifest: {}".format(json.dumps(data, indent=4)))
return data, sha1
except ValueError as e:
raise VersionManifestFetchException(
"Unable to deserialize response data into dictionary: {} (TargetVer: {})".format(e, ver_url)
)
return r.content, sha1
# mapping
get_specific_version_manifest = fetch_specific_version_manifest
@@ -400,8 +432,12 @@ def fetch_and_save_version_manifest(dest: Path, target_version: str=None, root_m
dest.parent.mkdir(parents=True, exist_ok=True)
try:
with dest.open("w") as f:
json.dump(data, f, indent=4)
method = "wb" if isinstance(data, bytes) else "w"
with dest.open(method) as f:
if isinstance(data, bytes):
f.write(data)
elif isinstance(data, dict):
json.dump(data, f)
if sha1 is not None and not file.verify_sha1(sha1):
raise HashMismatchException(
@@ -411,9 +447,48 @@ def fetch_and_save_version_manifest(dest: Path, target_version: str=None, root_m
file.posix_path
)
return data
if isinstance(data, bytes):
# Convert the data back to dict
return json.loads(data.decode("utf-8"))
else:
return data
except Exception as e:
logger.exception(e)
raise VersionManifestSaveException(
"Error saving version manifest:\n{}".format(e)
)
)
def download_core_file(manifest: dict, destination: Path, is_server=False, raise_for_null_hash=False,
progress_callback: Callable[str]=None) -> FileObject:
"""
Download core file from version manifest.
:param manifest:
:param destination:
:param is_server:
:param raise_for_null_hash:
:param progress_callback:
:return:
client_file: FileObject
"""
url, sha1 = get_core_file_download_url_and_hash(manifest, is_server=is_server)
destination.parent.mkdir(parents=True, exist_ok=True)
file = FileObject(destination, url=url)
if sha1 is None and raise_for_null_hash:
raise VersionDataKeyNotFoundException(
"Core file hash is missing. Enable \"raise_for_null_hash\" flag to ignore this exception.\n"
)
elif sha1 is None:
logger.warning("Client has no sha1.")
if file.exists and file.verify("sha1", sha1):
logger.info(f"Client existed and sha1 matches.")
elif url is not None:
download_file(file, progress_callback=progress_callback)
elif not url:
raise VersionDataKeyNotFoundException(
"Core file url is None. Unable to download it."
)
return file