From 391b61a43afd295ddecf8c899d5b77208dddb6d7 Mon Sep 17 00:00:00 2001 From: yyc12345 Date: Fri, 3 Feb 2023 15:34:16 +0800 Subject: [PATCH] update shit --- PyCmo/PyCmo.py | 8 +- PyCmo/PyCmo.pyproj | 8 +- PyCmo/PyCmoMisc.py | 14 +++ .../{VirtoolsConstants.py => VTConstants.py} | 43 +++++---- PyCmo/VTReader.py | 92 +++++++++++++++++++ PyCmo/VTStruct.py | 60 ++++++++++++ PyCmo/VTUtils.py | 75 +++++++++++++++ PyCmo/VirtoolsReader.py | 47 ---------- PyCmo/VirtoolsStruct.py | 43 --------- PyCmo/VirtoolsUtils.py | 48 ---------- 10 files changed, 273 insertions(+), 165 deletions(-) rename PyCmo/{VirtoolsConstants.py => VTConstants.py} (95%) create mode 100644 PyCmo/VTReader.py create mode 100644 PyCmo/VTStruct.py create mode 100644 PyCmo/VTUtils.py delete mode 100644 PyCmo/VirtoolsReader.py delete mode 100644 PyCmo/VirtoolsStruct.py delete mode 100644 PyCmo/VirtoolsUtils.py diff --git a/PyCmo/PyCmo.py b/PyCmo/PyCmo.py index a2f0e2b..d43f291 100644 --- a/PyCmo/PyCmo.py +++ b/PyCmo/PyCmo.py @@ -1,6 +1,6 @@ -import VirtoolsReader -import VirtoolsStruct +import VTReader +import VTStruct with open("D:\\libcmo21\\PyCmo\\Gameplay.nmo", 'rb') as fs: - composition = VirtoolsReader.ReadCKComposition(fs) - print(str(composition.Header)) \ No newline at end of file + composition = VTReader.ReadCKComposition(fs) + print(composition.Header) \ No newline at end of file diff --git a/PyCmo/PyCmo.pyproj b/PyCmo/PyCmo.pyproj index 0f6f076..19861d1 100644 --- a/PyCmo/PyCmo.pyproj +++ b/PyCmo/PyCmo.pyproj @@ -25,16 +25,16 @@ Code - + Code - + Code - + Code - + Code diff --git a/PyCmo/PyCmoMisc.py b/PyCmo/PyCmoMisc.py index a4b2dba..7e991ef 100644 --- a/PyCmo/PyCmoMisc.py +++ b/PyCmo/PyCmoMisc.py @@ -1,3 +1,4 @@ +import functools, inspect def OutputSizeHumanReadable(storage_size: int): probe = storage_size @@ -30,3 +31,16 @@ def BcdCodeToDecCode(bcd_num: int): pow *= 10 return result + +def ClsMethodRegister(cls): + def decorator(func): + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + return func(self, *args, **kwargs) + if inspect.getattr_static(cls, func.__name__, None): + msg = 'Error. method name REPEAT, {} has exist'.format(func.__name__) + raise NameError(msg) + else: + setattr(cls, func.__name__, wrapper) + return func + return decorator diff --git a/PyCmo/VirtoolsConstants.py b/PyCmo/VTConstants.py similarity index 95% rename from PyCmo/VirtoolsConstants.py rename to PyCmo/VTConstants.py index cf360be..c69a075 100644 --- a/PyCmo/VirtoolsConstants.py +++ b/PyCmo/VTConstants.py @@ -1,27 +1,31 @@ import enum -class PyEnum(object): - @staticmethod - def Contain(val: int, probe: int): - return bool(val & probe) +class CKEnum(object): + def __init__(self, val: int): + self.m_Value: int = val - @staticmethod - def Add(val: int, data: int): - return val | data - - @staticmethod - def PrintEnum(val: int, _enum: enum.IntEnum): - for i in _enum: + + def __repr__(self): + for i in self: if i.value == val: return i.name - return "" - - @staticmethod - def PrintEnumFlag(val: int, _enum: enum.IntEnum): + return "[None]" + + def __str__(self): + return self.__repr__() + +class CKFlagEnum(CKEnum): + def Contain(self: CKFlagEnum, probe: int): + return bool(self.m_Value & probe) + + def Add(self: CKFlagEnum, data: int): + self.m_Value = self.m_Value | data + + def __repr__(self): pending = [] - for i in _enum: + for i in self: # if it have exactly same entry, return directly if i.value == val: return i.name @@ -30,16 +34,17 @@ class PyEnum(object): if bool(val & i.value): pending.append(i.name) - return ', '.join(pending) + result = ', '.join(pending) + return result if len(result) != 9 else "[None]" -class CK_FILE_WRITEMODE(enum.IntEnum): +class CK_FILE_WRITEMODE(CKFlagEnum, enum.IntEnum): CKFILE_UNCOMPRESSED =0 # Save data uncompressed CKFILE_CHUNKCOMPRESSED_OLD =1 # Obsolete CKFILE_EXTERNALTEXTURES_OLD=2 # Obsolete : use CKContext::SetGlobalImagesSaveOptions instead. CKFILE_FORVIEWER =4 # Don't save Interface Data within the file, the level won't be editable anymore in the interface CKFILE_WHOLECOMPRESSED =8 # Compress the whole file -class CK_LOAD_FLAGS(enum.IntEnum): +class CK_LOAD_FLAGS(CKFlagEnum, enum.IntEnum): CK_LOAD_ANIMATION =1<<0 # Load animations CK_LOAD_GEOMETRY =1<<1 # Load geometry. CK_LOAD_DEFAULT =CK_LOAD_GEOMETRY|CK_LOAD_ANIMATION # Load animations & geometry diff --git a/PyCmo/VTReader.py b/PyCmo/VTReader.py new file mode 100644 index 0000000..5e7e6d6 --- /dev/null +++ b/PyCmo/VTReader.py @@ -0,0 +1,92 @@ +import VTStruct, VTUtils, VTConstants +import PyCmoMisc +import struct, io, datetime, zlib +import typing + +g_HeaderPacker = struct.Struct('<' + 'i' * 8) + +class CKFileReader(): + + @staticmethod + def ReadFileHeaders(self: VTStruct.CKFile) -> VTConstants.CKERROR: + if self.m_Parser is None: + return VTConstants.CKERR.CKERR_INVALIDPARAMETER + header = VTStruct.CKFileHeader() + + # check magic words + magic_words = self.m_Parser.GetReader()[0:4] + if magic_words != b'Nemo': + return VTConstants.CKERR.CKERR_INVALIDFILE + + # read header1 + if self.m_Parser.GetSize() < 0x20: + return VTConstants.CKERR.CKERR_INVALIDFILE + header1 = g_HeaderPacker.unpack(self.m_Parser.GetReader().read(8 * 4)) + + # check header1 + if header1[5]: # i don't know what is this fields stands for + header1 = tuple(0 for _ in range(8)) + + # virtools is too old to open this file. file is too new. + if header1[4] > 9: # file version + return VTConstants.CKERR.CKERR_OBSOLETEVIRTOOLS + + # read header2 + # file ver < 5 do not have second header + if header1[4] < 5: + header2 = tuple(0 for _ in range(8)) + else: + if self.m_Parser.GetSize() < 0x40: + return VTConstants.CKERR.CKERR_INVALIDFILE + + header2 = g_HeaderPacker.unpack(self.m_Parser.GetReader().read(8 * 4)) + + # forcely reset too big product ver + if header2[5] >= 12: # product version + header2[5] = 0 + header2[6] = 0x1010000 # product build + + # assign value + self.m_FileInfo.ProductVersion = header2[5] + self.m_FileInfo.ProductBuild = header2[6] + self.m_FileInfo.FileWriteMode.m_Value = header1[6] + self.m_FileInfo.CKVersion = header1[3] + self.m_FileInfo.FileVersion = header1[4] + self.m_FileInfo.FileSize = self.m_Parser.GetSize() + self.m_FileInfo.ManagerCount = header2[2] + self.m_FileInfo.ObjectCount = header2[3] + self.m_FileInfo.MaxIDSaved = header2[4] + self.m_FileInfo.Hdr1PackSize = header1[7] + self.m_FileInfo.Hdr1UnPackSize = header2[7] + self.m_FileInfo.DataPackSize = header2[0] + self.m_FileInfo.DataUnPackSize = header2[1] + self.m_FileInfo.Crc = header1[2] + + + # process date independently + # date is in BCD code + day = PyCmoMisc.BcdCodeToDecCode((raw_date >> 24) & 0xff) + month = PyCmoMisc.BcdCodeToDecCode((raw_date >> 16) & 0xff - 1) + month = (month % 12) + 1 + year = PyCmoMisc.BcdCodeToDecCode(raw_date & 0xffff) + header.Timestamp = datetime.date(year, month, day) + + if header.FileVersion >= 8: + # check crc + gotten_crc = zlib.adler32(b'Nemo Fi\0', 0) + gotten_crc = zlib.adler32(struct.pack("<6I", 0, raw_date, header.FileVersion, header.FileVersion2, header.SaveFlags, header.PrewHdrPackSize), gotten_crc) # reset crc as zero + gotten_crc = zlib.adler32(struct.pack("<8I", header.DataPackSize, header.DataUnpackSize, header.ManagerCount, header.ObjectCount, header.MaxIDSaved, header.ProductVersion, header.ProductBuild, header.PrewHdrUnpackSize), gotten_crc) + gotten_crc = zlib.adler32(fs.read(header.PrewHdrPackSize), gotten_crc) + gotten_crc = zlib.adler32(fs.read(header.DataPackSize), gotten_crc) + if gotten_crc != header.Crc: + raise Exception("Crc Error") + + return header + + + @staticmethod + def ReadCKComposition(fs: io.BufferedReader): + composition = VTStruct.CKComposition() + + composition.Header = ReadCKFileHeader(fs) + return composition diff --git a/PyCmo/VTStruct.py b/PyCmo/VTStruct.py new file mode 100644 index 0000000..ea22c65 --- /dev/null +++ b/PyCmo/VTStruct.py @@ -0,0 +1,60 @@ +import VTConstants, VTUtils +import PyCmoMisc +import datetime + +class CKFileInfo: + def __init__(self: CKFileInfo): + self.ProductVersion: int = 0 + self.ProductBuild: int = 0 + + self.FileWriteMode: VTConstants.CK_FILE_WRITEMODE = 0 + + self.FileVersion: int = 0 + self.CKVersion: int = 0 + + self.FileSize: int = 0 + self.ObjectCount: int = 0 + self.ManagerCount: int = 0 + self.MaxIDSaved: int = 0 + + self.Crc: int = 0 + + self.Hdr1PackSize: int = 0 + self.Hdr1UnPackSize: int = 0 + self.DataPackSize: int = 0 + self.DataUnpackSize: int = 0 + + def GetProductBuildTuple(self: CKFileInfo) -> tuple[int]: + return ( + (self.ProductBuild >> 24) & 0xff, + (self.ProductBuild >> 16) & 0xff, + (self.ProductBuild >> 8) & 0xff, + self.ProductBuild & 0xff + ) + + def __repr__(self: CKFileInfo) -> str: + return f"""Version (File / CK): {self.FileVersion:08X} / {self.CKVersion:08X} +Product (Version / Build): {self.ProductVersion:d} / {'.0'.join(self.GetProductBuildTuple())} +Save Flags: {self.SaveFlags} +File Size: {PyCmoMisc.OutputSizeHumanReadable(self.FileSize)} +Crc: 0x{self.Crc:08X} + +Preview Header (Pack / Unpack): {PyCmoMisc.OutputSizeHumanReadable(self.PrewHdrPackSize)} / {PyCmoMisc.OutputSizeHumanReadable(self.PrewHdrUnpackSize)} +Data (Pack / Unpack): {PyCmoMisc.OutputSizeHumanReadable(self.DataPackSize)} / {PyCmoMisc.OutputSizeHumanReadable(self.DataUnpackSize)} + +Manager Count: {self.ManagerCount:d} +Object Count: {self.ObjectCount:d} +Max ID Saved: {self.MaxIDSaved:d} +""" + + + +class CKFile(object): + def __init__(self): + self.m_FileName: str = '' + self.m_FileInfo: CKFileInfo = CKFileInfo() + self.m_Parser: VTUtils.UniversalFileReader = None + + def __repr__(self: CKFile) -> str: + return self.m_FileInfo + diff --git a/PyCmo/VTUtils.py b/PyCmo/VTUtils.py new file mode 100644 index 0000000..40d5f01 --- /dev/null +++ b/PyCmo/VTUtils.py @@ -0,0 +1,75 @@ +import PyCmoMisc +import zlib, io, mmap, os +import typing + +class RawFileReader(): + def __init__(self, filename: str): + self.__size: int = os.path.getsize(filename) + self.__fs = open(filename, 'rb') + self.__mm: mmap.mmap = mmap.mmap(self.__fs.fileno, 0, access = mmap.ACCESS_READ) + + def __del__(self): + self.__mm.close() + del self.__mm + self.__fs.close() + del self.__fs + + def GetSize(self) -> int: + return self.__size + + def GetReader(self) -> mmap.mmap: + return self.__mm + +class LargeZlibFileReader(): + def __init__(self, raw_reader: RawFileReader, comp_size: int, uncomp_size: int): + # set size + self.__size: int = uncomp_size + + # create mmap + self.__mm: mmap.mmap = mmap.mmap(-1, -1, access = mmap.ACCESS_WRITE) + + # decompress data + reader = raw_reader.GetReader() + parser: zlib._Decompress = zlib.decompressobj() + + buf = reader.read(io.DEFAULT_BUFFER_SIZE) + while buf: + self.__mm.write(parser.decompress(buf)) + buf = reader.read(io.DEFAULT_BUFFER_SIZE) + self._mm.write(parser.flush()) + + def __del__(self): + self.__mm.close() + del self.__mm + + def GetSize(self) -> int: + return self.__size + + def GetReader(self) -> mmap.mmap: + return self.__mm + +class SmallZlibFileReader(): + def __init__(self): + # create io + self.__ss: io.BytesIO = io.BytesIO() + + # decompress data + reader = raw_reader.GetReader() + parser: zlib._Decompress = zlib.decompressobj() + + buf = reader.read(io.DEFAULT_BUFFER_SIZE) + while buf: + self.__ss.write(parser.decompress(buf)) + buf = reader.read(io.DEFAULT_BUFFER_SIZE) + self._ss.write(parser.flush()) + + def __del__(self): + del self.__ss + + def GetSize(self) -> int: + return len(self.__ss.getvalue()) + + def GetReader(self) -> io.BytesIO: + return self.__ss + +UniversalFileReader = typing.Union[RawFileReader, LargeZlibFileReader, SmallZlibFileReader] diff --git a/PyCmo/VirtoolsReader.py b/PyCmo/VirtoolsReader.py deleted file mode 100644 index a1735fb..0000000 --- a/PyCmo/VirtoolsReader.py +++ /dev/null @@ -1,47 +0,0 @@ -import VirtoolsStruct -import VirtoolsUtils -import PyCmoMisc -import struct, io, datetime, zlib - -g_dword = struct.Struct("> 24) & 0xff) - month = PyCmoMisc.BcdCodeToDecCode((raw_date >> 16) & 0xff - 1) - month = (month % 12) + 1 - year = PyCmoMisc.BcdCodeToDecCode(raw_date & 0xffff) - header.Timestamp = datetime.date(year, month, day) - - if header.FileVersion >= 8: - # check crc - gotten_crc = zlib.adler32(b'Nemo Fi\0', 0) - gotten_crc = zlib.adler32(struct.pack("<6I", 0, raw_date, header.FileVersion, header.FileVersion2, header.SaveFlags, header.PrewHdrPackSize), gotten_crc) # reset crc as zero - gotten_crc = zlib.adler32(struct.pack("<8I", header.DataPackSize, header.DataUnpackSize, header.ManagerCount, header.ObjectCount, header.MaxIDSaved, header.ProductVersion, header.ProductBuild, header.PrewHdrUnpackSize), gotten_crc) - gotten_crc = zlib.adler32(fs.read(header.PrewHdrPackSize), gotten_crc) - gotten_crc = zlib.adler32(fs.read(header.DataPackSize), gotten_crc) - if gotten_crc != header.Crc: - raise Exception("Crc Error") - - return header - - -def ReadCKComposition(fs: io.BufferedReader): - composition = VirtoolsStruct.CKComposition() - - composition.Header = ReadCKFileHeader(fs) - return composition diff --git a/PyCmo/VirtoolsStruct.py b/PyCmo/VirtoolsStruct.py deleted file mode 100644 index 3f9df31..0000000 --- a/PyCmo/VirtoolsStruct.py +++ /dev/null @@ -1,43 +0,0 @@ -import VirtoolsConstants -import datetime -import PyCmoMisc - -class CKFileHeader: - def __init__(self): - self.Signature: bytes = b'Nemo Fi\0' - self.Crc: int = 0 - self.Timestamp: datetime.date = datetime.date.today() - self.FileVersion: int = 0 - self.FileVersion2: int = 0 - self.SaveFlags: int = 0 - self.PrewHdrPackSize: int = 0 - - self.DataPackSize: int = 0 - self.DataUnpackSize: int = 0 - self.ManagerCount: int = 0 - self.ObjectCount: int = 0 - self.MaxIDSaved: int = 0 - self.ProductVersion: int = 0 - self.ProductBuild: int = 0 - self.PrewHdrUnpackSize: int = 0 - - def __str__(self): - return f"""File Version: {self.FileVersion:d} / {self.FileVersion2:d} -Production (Version / Build): {self.ProductVersion:d} / {(self.ProductBuild >> 24) & 0xff:d}.{(self.ProductBuild >> 16) & 0xff:d}.{(self.ProductBuild >> 8) & 0xff:d}.{self.ProductBuild & 0xff:d} -Crc: 0x{self.Crc:08X} -Timestamp: {str(self.Timestamp)} -Save Flags: {VirtoolsConstants.PyEnum.PrintEnumFlag(self.SaveFlags, VirtoolsConstants.CK_FILE_WRITEMODE)} - -Preview Header (Pack / Unpack): {PyCmoMisc.OutputSizeHumanReadable(self.PrewHdrPackSize)} / {PyCmoMisc.OutputSizeHumanReadable(self.PrewHdrUnpackSize)} -Data (Pack / Unpack): {PyCmoMisc.OutputSizeHumanReadable(self.DataPackSize)} / {PyCmoMisc.OutputSizeHumanReadable(self.DataUnpackSize)} - -Manager Count: {self.ManagerCount:d} -Object Count: {self.ObjectCount:d} -Max ID Saved: {self.MaxIDSaved:d} -""" - - -class CKComposition(object): - def __init__(self): - self.Header: CKFileHeader = None - diff --git a/PyCmo/VirtoolsUtils.py b/PyCmo/VirtoolsUtils.py deleted file mode 100644 index 50cca3e..0000000 --- a/PyCmo/VirtoolsUtils.py +++ /dev/null @@ -1,48 +0,0 @@ -import zlib -import io - -class ZlibDecompressBuffer(object): - def __init__(self, _fs: io.BufferedReader, _len: int, _is_compressed: bool): - self.__fs: io.BufferedReader = _fs - self.__len: int = _len - self.__compressed: bool = _is_compressed - - self.__pos: int = 0 - self.__parser: zlib._Decompress = zlib.decompressobj() - self.__cache: bytes = b'' - self.__cachelen: int = 0 - - def __ParseOnce(self) -> bytes: - # check remain - remain: int = self.__len - self.__pos - if remain <= 0: - return None - - # read it and increase pos - read_count: int = min(remain, 1024) - gotten_uncompressed: bytes = self.__parser.decompress(self.__fs.read(read_count)) - self.__pos += read_count - - # everything has done, no more data, flush it and get it remained data - if self.__pos >= self.__len: - gotten_uncompressed += self.__parser.flush() - - return gotten_uncompressed - - def Read(self, expected: int): - # try enrich cache - while self.__cachelen < expected: - new_data = self.__ParseOnce() - if new_data is None: - # no more data - raise Exception("No more data.") - else: - self.__cache += new_data - self.__cachelen += len(new_data) - - # change data - returned_data = self.__cache[:expected] - self.__cache = self.__cache[expected:] - self.__cachelen -= expected - - return returned_data \ No newline at end of file