from __future__ import annotations import hashlib import logging import platform import zipfile from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from typing import List import requests from .exception import HashMismatchException DEFAULT_HASH_CHUNK_SIZE = 4096 DEFAULT_DOWNLOAD_CHUNK_SIZE = 4096 THREADED_DOWNLOAD_MAX_WORKERS = 10 logger = logging.getLogger("Launcher.CoreLib") class FileObject: def __init__(self, path: Path, url=None, sha1=None, sha256=None, md5=None): self.__path = path self.expected_sha1 = sha1 self.expected_sha256 = sha256 self.expected_md5 = md5 self.__sha1 = None self.__sha256 = None self.__md5 = None self.url = url def __eq__(self, other): """ THIS OPERATION ONLY COMPARE AND FILE SIZE! Use :param other: :return: """ if self.path == other.path: return True if self.size == other.size: return True return False def is_same(self, file: FileObject, hash_name="sha256") -> bool: if hash_name == "sha1": return self.sha1 == file.sha1 if hash_name == "md5": return self.md5 == file.md5 if hash_name != "sha256": logger.warning("Unsupported hash_name {}. Ignored and use sha256 instead.".format(hash_name)) return self.sha256 == file.sha256 @property def sha1(self): if self.__sha1 is None: self.__sha1 = self._calculate_hash("sha1") return self.__sha1 @property def sha256(self): if self.__sha256 is None: self.__sha256 = self._calculate_hash("sha256") return self.__sha256 @property def md5(self): if self.__md5 is None: self.__md5 = self._calculate_hash("md5") return self.__md5 def verify_sha1(self, expected_hash=None): return self.sha1 == expected_hash if expected_hash else self.expected_sha1 == self.sha1 def verify_sha256(self, expected_hash=None): return self.sha256 == expected_hash if expected_hash else self.expected_sha256 == self.sha256 def verify_md5(self, expected_hash=None): return self.sha1 == expected_hash if expected_hash else self.expected_sha1 == self.sha1 def verify(self, algorithm, expected_hash=None): if algorithm == "sha1": return self.verify_sha1(expected_hash if expected_hash else self.expected_sha1) if algorithm == "sha256": return self.verify_sha256(expected_hash if expected_hash else self.expected_sha256) if algorithm == "md5": return self.verify_md5(expected_hash if expected_hash else self.expected_md5) raise Exception("Unsupported algorithm: {}".format(algorithm)) def _calculate_hash(self, algorithm): h = hashlib.new(algorithm) with self.__path.open("rb") as f: for chunk in iter(lambda: f.read(DEFAULT_HASH_CHUNK_SIZE), b""): h.update(chunk) return h.hexdigest() @property def path(self): return self.__path @property def posix_path(self): return self.__path.absolute().as_posix() @property def size(self): return self.__path.stat().st_size @property def name(self): return self.__path.name @property def extension(self): return self.__path.suffix def __hash__(self): return hash(self.__path) @property def exists(self): return self.__path.exists() def download_file(file: FileObject, overwrite=True, progress_callback: Callable[int]=None) -> FileObject | Exception: """ Download file from url :param file: File (Objected) :param overwrite: Overwrite existing file :param progress_callback: Callback function called with progress information :return: """ file.path.parent.mkdir(parents=True, exist_ok=True) if file.exists and not overwrite: raise Exception("File {} already exists. Use overwrite=True to overwrite.".format(file)) if file.url is None: raise Exception("Target file {} URL cannot be None.".format(file)) logger.debug("Downloading file {} to {}".format(file.url, file)) with requests.get(file.url, stream=True) as r: current_chunk = 0 total_bytes = r.headers.get("content-length", None) logger.debug("URL: {}".format(file.url)) logger.debug("Response code: {}".format(r.status_code)) with file.path.open("wb") as f: for chunk in r.iter_content(chunk_size=DEFAULT_DOWNLOAD_CHUNK_SIZE): if not chunk: break f.write(chunk) current_chunk += 1 if callable(progress_callback) and total_bytes is not None: current_progress = ((current_chunk * DEFAULT_DOWNLOAD_CHUNK_SIZE) / int(total_bytes) ) * 100 progress_callback(file.name, current_progress) if file.expected_sha256 and not file.verify_sha256(): raise HashMismatchException("sha256", file.sha256, file.expected_sha256, file.posix_path) elif file.expected_sha1 and not file.verify_sha1(): raise HashMismatchException("sha1", file.sha1, file.expected_sha1, file.posix_path) elif file.expected_md5 and not file.verify_md5(): raise HashMismatchException("md5", file.md5, file.expected_md5, file.posix_path) if any([file.expected_sha1, file.expected_md5, file.expected_sha256]): logger.debug("File hash is match as the expected hash.") else: logger.warning("File hash is not checked.") return file def download_multiple_files(files: List[FileObject], download_callback: Callable[str]=None, max_workers=THREADED_DOWNLOAD_MAX_WORKERS, progress_callback: Callable[str]=None) \ -> List[FileObject]: """ Download multiple files :param download_callback: :param files: :param max_workers: :param progress_callback: :return: List of file objects that download failed. """ fails = [] files_count = 0 with ThreadPoolExecutor(max_workers=max_workers) as executor: future_to_file = {executor.submit(download_file, file): file for file in files} for future in as_completed(future_to_file): file = future_to_file[future] try: files_count += 1 future.result() if callable(download_callback): download_callback(file.posix_path) if callable(progress_callback): progress_callback(f"{len(files)} Files", files_count / len(files) * 100) except Exception as exc: logger.error("Unable to download file {}: {}".format(file.posix_path, exc)) fails.append(file) return fails def unzip(file: FileObject, destination: Path, target_extract_dir: Path | None=None, exclude: list[str] | None = None): with zipfile.ZipFile(file.posix_path, "r") as archive: if target_extract_dir is None and exclude is None: archive.extractall(path=destination) else: for member in archive.namelist(): # Ignore exclude file. if any(member.startswith(pattern) for pattern in exclude): continue if member.startswith(target_extract_dir.as_posix()): archive.extract(member, destination) def get_platform_info(): """ Get information about the current platform. :return: platform_name: str, architecture: str, os_version: str """ platform_name = platform.system().lower() arch = platform.uname().machine.lower() os_version = platform.version() if platform_name == "darwin": os_version = platform.mac_ver()[0] elif platform_name == "linux": os_version = platform.release() return platform_name, arch, os_version