455 lines
14 KiB
Python
455 lines
14 KiB
Python
import json
|
|
import logging
|
|
import shutil
|
|
from collections.abc import Callable
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
|
|
from core_lib.common.common import FileObject, download_multiple_files
|
|
from core_lib.common.exception import VersionDataKeyNotFoundException, AssetManifestFetchException, \
|
|
HashMismatchException, AssetManifestSaveException, AssetDataKeyNotFoundException, DownloadException, \
|
|
AssetManifestException
|
|
from core_lib.game.version_info import find_useful_part_in_specific_version_manifest
|
|
|
|
MOJANG_RESOURCE_ENDPOINT = "https://resources.download.minecraft.net/{beginning}/{hash}"
|
|
|
|
ASSET_DOWNLOAD_MAX_WORKERS = 20
|
|
ASSET_CHECK_MAX_WORKERS = 20
|
|
|
|
logger = logging.getLogger("Launcher.CoreLib")
|
|
|
|
def fetch_asset_manifest(target_version_manifest: dict, raw=False) -> tuple[dict | bytes, str]:
|
|
"""
|
|
Fetch the asset manifest from target version's manifest.
|
|
:param target_version_manifest:
|
|
:param raw: Return raw asset manifest.
|
|
:return:
|
|
asset_manifest: dict
|
|
hash: str (sha1)
|
|
"""
|
|
_, asset_index, _, _, _, _, _ = find_useful_part_in_specific_version_manifest(target_version_manifest)
|
|
|
|
if not asset_index:
|
|
raise VersionDataKeyNotFoundException(
|
|
"Version data section (assetIndex) not found."
|
|
)
|
|
|
|
asset_manifest_url = asset_index.get("url", "")
|
|
sha1 = asset_index.get("sha1", "")
|
|
|
|
if not asset_manifest_url:
|
|
raise VersionDataKeyNotFoundException(
|
|
"Asset manifest url not found in assetIndex."
|
|
)
|
|
|
|
try:
|
|
r = requests.get(asset_manifest_url)
|
|
r.raise_for_status()
|
|
except requests.exceptions.RequestException as e:
|
|
raise AssetManifestFetchException(
|
|
"Failed to fetch version manifest URL: {}\n{}".format(asset_manifest_url, e)
|
|
)
|
|
|
|
try:
|
|
if raw:
|
|
return r.content, sha1
|
|
|
|
data = r.json()
|
|
return data, sha1
|
|
except ValueError as e:
|
|
raise AssetManifestFetchException(
|
|
"Unable to deserialize asset response data into dictionary: {} ".format(e)
|
|
)
|
|
|
|
def fetch_and_save_asset_manifest(target_version_manifest: dict, destination: Path) -> dict:
|
|
"""
|
|
Fetch the asset manifest from target version's manifest and save it.
|
|
:param target_version_manifest:
|
|
:param destination:
|
|
:return:
|
|
asset_manifest: dict
|
|
"""
|
|
data, sha1 = fetch_asset_manifest(target_version_manifest, raw=True)
|
|
|
|
if not sha1:
|
|
logger.warning(
|
|
"Target version's manifest does not contain asset manifest sha1 hash. "
|
|
)
|
|
|
|
file = FileObject(
|
|
path=destination,
|
|
sha1=sha1,
|
|
)
|
|
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
try:
|
|
with file.path.open("wb") as f:
|
|
f.write(data)
|
|
|
|
if sha1 and not file.verify_sha1(sha1):
|
|
raise HashMismatchException(
|
|
"sha1",
|
|
file.sha1,
|
|
sha1,
|
|
file.posix_path
|
|
)
|
|
|
|
return json.loads(data.decode("utf-8"))
|
|
except Exception as e:
|
|
logger.exception(e)
|
|
raise AssetManifestSaveException(
|
|
"Error saving version manifest:\n{}".format(e)
|
|
)
|
|
|
|
def _check_asset(asset_id: str, asset_data: dict, relative_path: str, assets_dir: Path, launcher_root: Path, virtual: bool,
|
|
map_to_resources: bool, no_hash_check: bool=False)\
|
|
-> tuple[FileObject | None, dict | None]:
|
|
"""
|
|
Check if asset exists.
|
|
:param asset_data:
|
|
:param relative_path:
|
|
:param assets_dir:
|
|
:param launcher_root:
|
|
:param virtual:
|
|
:param map_to_resources:
|
|
:return:
|
|
asset_file: FileObject
|
|
legacy_context: dict
|
|
"""
|
|
hash_raw = asset_data.get("hash", "")
|
|
expected_size = asset_data.get("size")
|
|
|
|
if not hash_raw:
|
|
raise AssetDataKeyNotFoundException(
|
|
"Asset that inside assets section (objects) missing hash."
|
|
)
|
|
|
|
beginning = hash_raw[:2]
|
|
|
|
full_path = Path(assets_dir, "objects", beginning, hash_raw) # For newer assets
|
|
full_url = MOJANG_RESOURCE_ENDPOINT.format(beginning=beginning, hash=hash_raw)
|
|
file = FileObject(path=full_path, sha1=hash_raw, url=full_url)
|
|
|
|
needs_download = True
|
|
|
|
if full_path.exists() and full_path.is_file():
|
|
if no_hash_check and expected_size is not None:
|
|
needs_download = full_path.stat().st_size != expected_size
|
|
else:
|
|
needs_download = not file.verify_sha1(hash_raw)
|
|
|
|
if not needs_download:
|
|
suffix = " (no_hash_check=True)" if no_hash_check else ""
|
|
logger.debug(
|
|
"Asset hash {} (Aka {}) already exists.{}".format(
|
|
hash_raw,
|
|
relative_path,
|
|
suffix,
|
|
)
|
|
)
|
|
|
|
legacy_context = None
|
|
if virtual:
|
|
legacy_context = {
|
|
"file": file,
|
|
"target": full_path,
|
|
"sha1": hash_raw,
|
|
"destination": Path(assets_dir, "virtual", asset_id, relative_path),
|
|
"size": expected_size,
|
|
}
|
|
elif map_to_resources:
|
|
legacy_context = {
|
|
"file": file,
|
|
"target": full_path,
|
|
"sha1": hash_raw,
|
|
"destination": Path(launcher_root, "resources", relative_path),
|
|
"size": expected_size,
|
|
}
|
|
|
|
return file if needs_download else None, legacy_context
|
|
|
|
def collect_assets_from_manifest(manifest: dict, asset_id: str, assets_dir: Path, launcher_root: Path,
|
|
max_workers=ASSET_CHECK_MAX_WORKERS,
|
|
no_hash_check=False) -> tuple[list[FileObject], list[dict]]:
|
|
"""
|
|
Collect assets from asset manifest. (Only return items that does not exist)
|
|
:param no_hash_check:
|
|
:param manifest:
|
|
:param asset_id:
|
|
:param assets_dir:
|
|
:param launcher_root:
|
|
:param max_workers:
|
|
:return:
|
|
assets: list[FileObject]
|
|
legacy_assets: list[dict] (Contains target and destination of the legacy asset)
|
|
"""
|
|
objects = manifest.get("objects", [])
|
|
virtual = manifest.get("virtual", False)
|
|
map_to_resources = manifest.get("map_to_resources", False)
|
|
|
|
files = []
|
|
legacy_assets = []
|
|
|
|
if not objects:
|
|
raise AssetDataKeyNotFoundException(
|
|
"Assets section (objects) not found in asset manifest."
|
|
)
|
|
|
|
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
futures = [
|
|
executor.submit(
|
|
_check_asset,
|
|
asset_id, objects[relative_path], relative_path, assets_dir, launcher_root, virtual, map_to_resources,
|
|
no_hash_check=no_hash_check
|
|
)
|
|
for relative_path in objects
|
|
]
|
|
|
|
for future in as_completed(futures):
|
|
asset_file, legacy_context = future.result()
|
|
|
|
if asset_file:
|
|
files.append(asset_file)
|
|
|
|
# Also copy the asset into legacy directory if needed.
|
|
if legacy_context:
|
|
legacy_assets.append(legacy_context)
|
|
|
|
return files, legacy_assets
|
|
|
|
|
|
def download_assets_and_copy(assets: list[FileObject], legacy_assets: list[dict],
|
|
no_hash_check: bool=False, allow_missing=False,
|
|
progress_callback: Callable[str]=None,
|
|
redownload_callback: Callable[[list[FileObject]], tuple[bool, list[FileObject]]]=None) -> list[FileObject]:
|
|
"""
|
|
Download assets from target version's asset manifest. (Not sure why this function exist, may for compatibly)
|
|
Also, this function will copy
|
|
:param assets:
|
|
:param legacy_assets:
|
|
:param no_hash_check:
|
|
:param allow_missing:
|
|
:param progress_callback:
|
|
:param redownload_callback:
|
|
:return:
|
|
files: list[FileObject] (files that download failed)
|
|
"""
|
|
|
|
def __worker(asset: dict):
|
|
target = asset.get("target")
|
|
destination = asset.get("destination")
|
|
sha1 = asset.get("sha1")
|
|
size = asset.get("size")
|
|
|
|
if not target or not destination:
|
|
raise ValueError(
|
|
"Legacy asset missing target or destination key."
|
|
)
|
|
|
|
# Check source file
|
|
source_asset = FileObject(path=target)
|
|
|
|
if (not source_asset.exists or
|
|
(not no_hash_check and sha1 and not source_asset.verify("sha1", sha1))):
|
|
logger.warning("Target asset {} not found.".format(target))
|
|
return False, None
|
|
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Check destination file
|
|
old_asset = FileObject(path=destination)
|
|
if old_asset.exists:
|
|
if no_hash_check and size is not None:
|
|
if old_asset.path.stat().st_size == size:
|
|
logger.debug("Asset {} already exists. (no_hash_check=True)".format(target))
|
|
return True, None
|
|
elif sha1 and old_asset.verify("sha1", sha1):
|
|
logger.debug("Asset {} already exists.".format(target))
|
|
return True, None
|
|
|
|
shutil.copyfile(target, destination)
|
|
return True, destination
|
|
|
|
fails , _ = download_multiple_files(assets, progress_callback=progress_callback,
|
|
max_workers=ASSET_DOWNLOAD_MAX_WORKERS)
|
|
|
|
|
|
download_ok = len(fails) == 0
|
|
|
|
if fails and callable(redownload_callback):
|
|
download_ok, fails = redownload_callback(fails)
|
|
|
|
if not download_ok:
|
|
logger.warning(
|
|
"Download failed for assets: \n"
|
|
+"\n".join([asset.posix_path for asset in fails])
|
|
)
|
|
if not allow_missing:
|
|
return fails
|
|
|
|
# Use thread pool to handle copy file process
|
|
with ThreadPoolExecutor(max_workers=ASSET_DOWNLOAD_MAX_WORKERS) as executor:
|
|
futures = [
|
|
executor.submit(
|
|
__worker,
|
|
asset
|
|
)
|
|
for asset in legacy_assets
|
|
]
|
|
|
|
for future in as_completed(futures):
|
|
result, dest = future.result()
|
|
|
|
if result and dest:
|
|
logger.debug("Created: {}".format(dest))
|
|
|
|
return fails
|
|
|
|
def correct_assets_name_and_copy(manifest: dict, assets_dir: Path, output_dir: Path) -> list[str]:
|
|
"""
|
|
Correct the asset name and copy to output folder. (Left this for somebody who want)
|
|
IMPORTANT: This function will copy the target asset to another directory (rename to its original name).
|
|
And, you must download all assets inside the asset_manifest before you use this function.
|
|
:param manifest:
|
|
:param assets_dir:
|
|
:param output_dir:
|
|
:return:
|
|
missing: list[str] (Contains missing or incorrect hash asset keys)
|
|
"""
|
|
missing = []
|
|
|
|
objects = manifest.get("objects", [])
|
|
|
|
if not objects:
|
|
raise AssetDataKeyNotFoundException(
|
|
"Assets section (objects) not found in asset manifest."
|
|
)
|
|
|
|
for key in objects:
|
|
data = objects[key]
|
|
hash_raw = data.get("hash", "")
|
|
|
|
if not hash_raw:
|
|
raise AssetDataKeyNotFoundException(
|
|
"Asset that inside assets section (objects) missing hash."
|
|
)
|
|
|
|
beginning = hash_raw[:2]
|
|
|
|
full_path = Path(assets_dir, "objects", beginning, hash_raw)
|
|
|
|
full_url = MOJANG_RESOURCE_ENDPOINT.format(beginning=beginning, hash=hash_raw)
|
|
file = FileObject(path=full_path, sha1=hash_raw, url=full_url)
|
|
|
|
if not ((full_path.exists() and full_path.is_file()) and file.verify_sha1(hash_raw)):
|
|
missing.append(key)
|
|
continue
|
|
|
|
full_path_new = Path(output_dir, key)
|
|
full_path_new.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
shutil.copy(file.posix_path, full_path_new.as_posix())
|
|
|
|
return missing
|
|
|
|
def download_assets(manifest: dict, assets_dir: Path, launcher_root: Path, no_hash_check: bool=False,
|
|
progress_callback: Callable[[str, float], None]=None,
|
|
redownload_callback: Callable[[list[FileObject]], tuple[bool, list[FileObject]]]=None) -> list[FileObject]:
|
|
_, asset_index, _, _, _, _, _ = find_useful_part_in_specific_version_manifest(manifest)
|
|
|
|
asset_id = asset_index.get("id")
|
|
|
|
if not asset_id:
|
|
raise VersionDataKeyNotFoundException(
|
|
"Asset ID is missing."
|
|
)
|
|
|
|
index_path = Path(assets_dir, "indexes", f"{asset_id}.json")
|
|
|
|
# Download and save asset manifest
|
|
asset_manifest = fetch_and_save_asset_manifest(
|
|
manifest,
|
|
index_path,
|
|
)
|
|
|
|
# Download assets files
|
|
assets, legacy_assets = collect_assets_from_manifest(
|
|
asset_manifest,
|
|
asset_id,
|
|
assets_dir,
|
|
launcher_root,
|
|
no_hash_check=no_hash_check,
|
|
)
|
|
|
|
fails = download_assets_and_copy(
|
|
assets,
|
|
legacy_assets,
|
|
progress_callback=progress_callback,
|
|
no_hash_check=no_hash_check,
|
|
redownload_callback=redownload_callback,
|
|
)
|
|
|
|
return fails
|
|
|
|
def check_assets_exist(manifest: dict, assets_dir: Path, launcher_root: Path, no_hash_check: bool=False) -> bool:
|
|
_, asset_index, _, _, _, _, _ = find_useful_part_in_specific_version_manifest(manifest)
|
|
|
|
asset_id = asset_index.get("id")
|
|
|
|
if not asset_id:
|
|
raise VersionDataKeyNotFoundException(
|
|
"Asset ID is missing."
|
|
)
|
|
|
|
index_path = Path(assets_dir, "indexes", f"{asset_id}.json")
|
|
|
|
asset_manifest = None
|
|
if not (index_path.exists() and index_path.is_file()):
|
|
fetch_and_save_asset_manifest(
|
|
manifest,
|
|
index_path,
|
|
)
|
|
|
|
with index_path.open() as f:
|
|
asset_manifest = json.load(f)
|
|
|
|
# Download assets files
|
|
assets, legacy_assets = collect_assets_from_manifest(
|
|
asset_manifest,
|
|
asset_id,
|
|
assets_dir,
|
|
launcher_root,
|
|
no_hash_check=no_hash_check,
|
|
)
|
|
|
|
if not assets and not legacy_assets:
|
|
return True
|
|
|
|
return False
|
|
|
|
def read_exist_assets_indexes(asset_id: str, assets_dir: Path):
|
|
return json.loads(Path(assets_dir, "indexes", f"{asset_id}.json").open().read())
|
|
|
|
|
|
def find_correct_assets_dir(asset_id: str, assets_dir: Path, launcher_root: Path, asset_manifest: dict | None=None) -> Path:
|
|
try:
|
|
if not asset_manifest:
|
|
asset_manifest = read_exist_assets_indexes(asset_id, assets_dir)
|
|
except FileNotFoundError:
|
|
raise AssetManifestException(
|
|
"Asset manifest not found in: {} (Index: {})".format(assets_dir, asset_id),
|
|
)
|
|
|
|
virtual = asset_manifest.get("virtual")
|
|
map_to_resources = asset_manifest.get("map_to_resources")
|
|
|
|
if virtual:
|
|
return assets_dir / "virtual" / asset_id
|
|
|
|
if map_to_resources:
|
|
return launcher_root / "resources"
|
|
|
|
return assets_dir |