update shit

PyCmo/PyCmo.py

@@ -1,6 +1,6 @@
-import VirtoolsReader
+import VTReader
-import VirtoolsStruct
+import VTStruct
 
 with open("D:\\libcmo21\\PyCmo\\Gameplay.nmo", 'rb') as fs:
-    composition = VirtoolsReader.ReadCKComposition(fs)
+    composition = VTReader.ReadCKComposition(fs)
-    print(str(composition.Header))
+    print(composition.Header)

PyCmo/PyCmo.pyproj

@@ -25,16 +25,16 @@
     <Compile Include="PyCmoMisc.py">
       <SubType>Code</SubType>
     </Compile>
-    <Compile Include="VirtoolsConstants.py">
+    <Compile Include="VTConstants.py">
       <SubType>Code</SubType>
     </Compile>
-    <Compile Include="VirtoolsReader.py">
+    <Compile Include="VTReader.py">
       <SubType>Code</SubType>
     </Compile>
-    <Compile Include="VirtoolsStruct.py">
+    <Compile Include="VTStruct.py">
       <SubType>Code</SubType>
     </Compile>
-    <Compile Include="VirtoolsUtils.py">
+    <Compile Include="VTUtils.py">
       <SubType>Code</SubType>
     </Compile>
   </ItemGroup>

PyCmo/PyCmoMisc.py

@@ -1,3 +1,4 @@
+import functools, inspect
 
 def OutputSizeHumanReadable(storage_size: int):
     probe = storage_size
@@ -30,3 +31,16 @@ def BcdCodeToDecCode(bcd_num: int):
         pow *= 10
 
     return result
+
+def ClsMethodRegister(cls):
+    def decorator(func):
+        @functools.wraps(func) 
+        def wrapper(self, *args, **kwargs): 
+            return func(self, *args, **kwargs)
+        if inspect.getattr_static(cls, func.__name__, None):
+            msg = 'Error. method name REPEAT, {} has exist'.format(func.__name__)
+            raise NameError(msg)
+        else:
+            setattr(cls, func.__name__, wrapper)
+        return func 
+    return decorator

PyCmo/VTConstants.py

@@ -1,27 +1,31 @@
 import enum
 
-class PyEnum(object):
+class CKEnum(object):
-    @staticmethod
+    def __init__(self, val: int):
-    def Contain(val: int, probe: int):
+        self.m_Value: int = val
-        return bool(val & probe)
 
-    @staticmethod
+
-    def Add(val: int, data: int):
+    def __repr__(self):
-        return val | data
+        for i in self:
-    
-    @staticmethod
-    def PrintEnum(val: int, _enum: enum.IntEnum):
-        for i in _enum:
             if i.value == val:
                 return i.name
 
-        return ""
+        return "[None]"
-    
+
-    @staticmethod
+    def __str__(self):
-    def PrintEnumFlag(val: int, _enum: enum.IntEnum):
+        return self.__repr__()
+
+class CKFlagEnum(CKEnum):
+    def Contain(self: CKFlagEnum, probe: int):
+        return bool(self.m_Value & probe)
+
+    def Add(self: CKFlagEnum, data: int):
+        self.m_Value = self.m_Value | data
+
+    def __repr__(self):
         pending = []
 
-        for i in _enum:
+        for i in self:
             # if it have exactly same entry, return directly
             if i.value == val:
                 return i.name
@@ -30,16 +34,17 @@ class PyEnum(object):
             if bool(val & i.value):
                 pending.append(i.name)
 
-        return ', '.join(pending)
+        result = ', '.join(pending)
+        return result if len(result) != 9 else "[None]"
 
-class CK_FILE_WRITEMODE(enum.IntEnum):
+class CK_FILE_WRITEMODE(CKFlagEnum, enum.IntEnum):
     CKFILE_UNCOMPRESSED	       =0	# Save data uncompressed
     CKFILE_CHUNKCOMPRESSED_OLD =1	# Obsolete
     CKFILE_EXTERNALTEXTURES_OLD=2	# Obsolete : use CKContext::SetGlobalImagesSaveOptions instead.
     CKFILE_FORVIEWER           =4	# Don't save Interface Data within the file, the level won't be editable anymore in the interface
     CKFILE_WHOLECOMPRESSED     =8	# Compress the whole file
 
-class CK_LOAD_FLAGS(enum.IntEnum):
+class CK_LOAD_FLAGS(CKFlagEnum, enum.IntEnum):
     CK_LOAD_ANIMATION					=1<<0	# Load animations
     CK_LOAD_GEOMETRY					=1<<1	# Load geometry.
     CK_LOAD_DEFAULT						=CK_LOAD_GEOMETRY|CK_LOAD_ANIMATION	# Load animations & geometry

PyCmo/VTReader.py

@@ -0,0 +1,92 @@
+import VTStruct, VTUtils, VTConstants
+import PyCmoMisc
+import struct, io, datetime, zlib
+import typing
+
+g_HeaderPacker = struct.Struct('<' + 'i' * 8)
+
+class CKFileReader():
+
+    @staticmethod
+    def ReadFileHeaders(self: VTStruct.CKFile) -> VTConstants.CKERROR:
+        if self.m_Parser is None:
+            return VTConstants.CKERR.CKERR_INVALIDPARAMETER
+        header = VTStruct.CKFileHeader()
+
+        # check magic words
+        magic_words = self.m_Parser.GetReader()[0:4]
+        if magic_words != b'Nemo':
+            return VTConstants.CKERR.CKERR_INVALIDFILE
+        
+        # read header1
+        if self.m_Parser.GetSize() < 0x20:
+            return VTConstants.CKERR.CKERR_INVALIDFILE
+        header1 = g_HeaderPacker.unpack(self.m_Parser.GetReader().read(8 * 4))
+
+        # check header1
+        if header1[5]:  # i don't know what is this fields stands for
+            header1 = tuple(0 for _ in range(8))
+
+        # virtools is too old to open this file. file is too new.
+        if header1[4] > 9:  # file version
+            return VTConstants.CKERR.CKERR_OBSOLETEVIRTOOLS
+
+        # read header2
+        # file ver < 5 do not have second header
+        if header1[4] < 5:
+            header2 = tuple(0 for _ in range(8))
+        else:
+            if self.m_Parser.GetSize() < 0x40:
+                return VTConstants.CKERR.CKERR_INVALIDFILE
+
+            header2 = g_HeaderPacker.unpack(self.m_Parser.GetReader().read(8 * 4))
+
+        # forcely reset too big product ver
+        if header2[5] >= 12:    # product version
+            header2[5] = 0
+            header2[6] = 0x1010000   # product build
+
+        # assign value
+        self.m_FileInfo.ProductVersion = header2[5]
+        self.m_FileInfo.ProductBuild = header2[6]
+        self.m_FileInfo.FileWriteMode.m_Value = header1[6]
+        self.m_FileInfo.CKVersion = header1[3]
+        self.m_FileInfo.FileVersion = header1[4]
+        self.m_FileInfo.FileSize = self.m_Parser.GetSize()
+        self.m_FileInfo.ManagerCount = header2[2]
+        self.m_FileInfo.ObjectCount = header2[3]
+        self.m_FileInfo.MaxIDSaved = header2[4]
+        self.m_FileInfo.Hdr1PackSize = header1[7]
+        self.m_FileInfo.Hdr1UnPackSize = header2[7]
+        self.m_FileInfo.DataPackSize = header2[0]
+        self.m_FileInfo.DataUnPackSize = header2[1]
+        self.m_FileInfo.Crc = header1[2]
+
+
+        # process date independently
+        # date is in BCD code
+        day = PyCmoMisc.BcdCodeToDecCode((raw_date >> 24) & 0xff)
+        month = PyCmoMisc.BcdCodeToDecCode((raw_date >> 16) & 0xff - 1)
+        month = (month % 12) + 1
+        year = PyCmoMisc.BcdCodeToDecCode(raw_date & 0xffff)
+        header.Timestamp = datetime.date(year, month, day)
+
+        if header.FileVersion >= 8:
+            # check crc
+            gotten_crc = zlib.adler32(b'Nemo Fi\0', 0)
+            gotten_crc = zlib.adler32(struct.pack("<6I", 0, raw_date, header.FileVersion, header.FileVersion2, header.SaveFlags, header.PrewHdrPackSize), gotten_crc)   # reset crc as zero
+            gotten_crc = zlib.adler32(struct.pack("<8I", header.DataPackSize, header.DataUnpackSize, header.ManagerCount, header.ObjectCount, header.MaxIDSaved, header.ProductVersion, header.ProductBuild, header.PrewHdrUnpackSize), gotten_crc)
+            gotten_crc = zlib.adler32(fs.read(header.PrewHdrPackSize), gotten_crc)
+            gotten_crc = zlib.adler32(fs.read(header.DataPackSize), gotten_crc)
+            if gotten_crc != header.Crc:
+                raise Exception("Crc Error")
+
+        return header
+
+    
+    @staticmethod
+    def ReadCKComposition(fs: io.BufferedReader):
+        composition = VTStruct.CKComposition()
+
+        composition.Header = ReadCKFileHeader(fs)
+        return composition

PyCmo/VTStruct.py

@@ -0,0 +1,60 @@
+import VTConstants, VTUtils
+import PyCmoMisc
+import datetime
+
+class CKFileInfo:
+    def __init__(self: CKFileInfo):
+        self.ProductVersion: int = 0
+        self.ProductBuild: int = 0
+
+        self.FileWriteMode: VTConstants.CK_FILE_WRITEMODE = 0
+        
+        self.FileVersion: int = 0
+        self.CKVersion: int = 0
+
+        self.FileSize: int = 0
+        self.ObjectCount: int = 0
+        self.ManagerCount: int = 0
+        self.MaxIDSaved: int = 0
+
+        self.Crc: int = 0
+        
+        self.Hdr1PackSize: int = 0
+        self.Hdr1UnPackSize: int = 0
+        self.DataPackSize: int = 0
+        self.DataUnpackSize: int = 0
+
+    def GetProductBuildTuple(self: CKFileInfo) -> tuple[int]:
+        return (
+            (self.ProductBuild >> 24) & 0xff, 
+            (self.ProductBuild >> 16) & 0xff, 
+            (self.ProductBuild >> 8) & 0xff, 
+            self.ProductBuild & 0xff
+        )
+
+    def __repr__(self: CKFileInfo) -> str:
+        return f"""Version (File / CK): {self.FileVersion:08X} / {self.CKVersion:08X}
+Product (Version / Build): {self.ProductVersion:d} / {'.0'.join(self.GetProductBuildTuple())}
+Save Flags: {self.SaveFlags}
+File Size: {PyCmoMisc.OutputSizeHumanReadable(self.FileSize)}
+Crc: 0x{self.Crc:08X}
+
+Preview Header (Pack / Unpack): {PyCmoMisc.OutputSizeHumanReadable(self.PrewHdrPackSize)} / {PyCmoMisc.OutputSizeHumanReadable(self.PrewHdrUnpackSize)}
+Data (Pack / Unpack): {PyCmoMisc.OutputSizeHumanReadable(self.DataPackSize)} / {PyCmoMisc.OutputSizeHumanReadable(self.DataUnpackSize)}
+
+Manager Count: {self.ManagerCount:d}
+Object Count: {self.ObjectCount:d}
+Max ID Saved: {self.MaxIDSaved:d}
+"""
+
+
+
+class CKFile(object):
+    def __init__(self):
+        self.m_FileName: str = ''
+        self.m_FileInfo: CKFileInfo = CKFileInfo()
+        self.m_Parser: VTUtils.UniversalFileReader = None
+
+    def __repr__(self: CKFile) -> str:
+        return self.m_FileInfo
+

PyCmo/VTUtils.py

@@ -0,0 +1,75 @@
+import PyCmoMisc
+import zlib, io, mmap, os
+import typing
+
+class RawFileReader():
+    def __init__(self, filename: str):
+        self.__size: int = os.path.getsize(filename)
+        self.__fs = open(filename, 'rb')
+        self.__mm: mmap.mmap = mmap.mmap(self.__fs.fileno, 0, access = mmap.ACCESS_READ)
+    
+    def __del__(self):
+        self.__mm.close()
+        del self.__mm
+        self.__fs.close()
+        del self.__fs
+
+    def GetSize(self) -> int:
+        return self.__size
+
+    def GetReader(self) -> mmap.mmap:
+        return self.__mm
+
+class LargeZlibFileReader():
+    def __init__(self, raw_reader: RawFileReader, comp_size: int, uncomp_size: int):
+        # set size
+        self.__size: int = uncomp_size
+
+        # create mmap
+        self.__mm: mmap.mmap = mmap.mmap(-1, -1, access = mmap.ACCESS_WRITE)
+        
+        # decompress data
+        reader = raw_reader.GetReader()
+        parser: zlib._Decompress = zlib.decompressobj()
+
+        buf = reader.read(io.DEFAULT_BUFFER_SIZE)
+        while buf:
+            self.__mm.write(parser.decompress(buf))
+            buf = reader.read(io.DEFAULT_BUFFER_SIZE)
+        self._mm.write(parser.flush())
+
+    def __del__(self):
+        self.__mm.close()
+        del self.__mm
+
+    def GetSize(self) -> int:
+        return self.__size
+
+    def GetReader(self) -> mmap.mmap:
+        return self.__mm
+
+class SmallZlibFileReader():
+    def __init__(self):
+        # create io
+        self.__ss: io.BytesIO = io.BytesIO()
+
+        # decompress data
+        reader = raw_reader.GetReader()
+        parser: zlib._Decompress = zlib.decompressobj()
+
+        buf = reader.read(io.DEFAULT_BUFFER_SIZE)
+        while buf:
+            self.__ss.write(parser.decompress(buf))
+            buf = reader.read(io.DEFAULT_BUFFER_SIZE)
+        self._ss.write(parser.flush())
+
+    def __del__(self):
+        del self.__ss
+
+    def GetSize(self) -> int:
+        return len(self.__ss.getvalue())
+
+    def GetReader(self) -> io.BytesIO:
+        return self.__ss
+
+UniversalFileReader = typing.Union[RawFileReader, LargeZlibFileReader, SmallZlibFileReader]

PyCmo/VirtoolsReader.py

@@ -1,47 +0,0 @@
-import VirtoolsStruct
-import VirtoolsUtils
-import PyCmoMisc
-import struct, io, datetime, zlib
-
-g_dword = struct.Struct("<I")
-
-def ReadCKFileHeader(fs: io.BufferedReader):
-    header = VirtoolsStruct.CKFileHeader()
-
-    # check magic words
-    magic_words = fs.read(8)
-    if magic_words != b'Nemo Fi\0':
-        raise Exception("Fail to read file header magic words")
-    header.Signature = magic_words
-
-    # assign data
-    (header.Crc, raw_date, header.FileVersion, header.FileVersion2, header.SaveFlags, header.PrewHdrPackSize,
-    header.DataPackSize, header.DataUnpackSize, header.ManagerCount, header.ObjectCount, header.MaxIDSaved, 
-    header.ProductVersion, header.ProductBuild, header.PrewHdrUnpackSize) = struct.unpack("<14I", fs.read(14 * 4))
-
-    # process date independently
-    # date is in BCD code
-    day = PyCmoMisc.BcdCodeToDecCode((raw_date >> 24) & 0xff)
-    month = PyCmoMisc.BcdCodeToDecCode((raw_date >> 16) & 0xff - 1)
-    month = (month % 12) + 1
-    year = PyCmoMisc.BcdCodeToDecCode(raw_date & 0xffff)
-    header.Timestamp = datetime.date(year, month, day)
-
-    if header.FileVersion >= 8:
-        # check crc
-        gotten_crc = zlib.adler32(b'Nemo Fi\0', 0)
-        gotten_crc = zlib.adler32(struct.pack("<6I", 0, raw_date, header.FileVersion, header.FileVersion2, header.SaveFlags, header.PrewHdrPackSize), gotten_crc)   # reset crc as zero
-        gotten_crc = zlib.adler32(struct.pack("<8I", header.DataPackSize, header.DataUnpackSize, header.ManagerCount, header.ObjectCount, header.MaxIDSaved, header.ProductVersion, header.ProductBuild, header.PrewHdrUnpackSize), gotten_crc)
-        gotten_crc = zlib.adler32(fs.read(header.PrewHdrPackSize), gotten_crc)
-        gotten_crc = zlib.adler32(fs.read(header.DataPackSize), gotten_crc)
-        if gotten_crc != header.Crc:
-            raise Exception("Crc Error")
-
-    return header
-
-
-def ReadCKComposition(fs: io.BufferedReader):
-    composition = VirtoolsStruct.CKComposition()
-
-    composition.Header = ReadCKFileHeader(fs)
-    return composition

PyCmo/VirtoolsStruct.py

@@ -1,43 +0,0 @@
-import VirtoolsConstants
-import datetime
-import PyCmoMisc
-
-class CKFileHeader:
-    def __init__(self):
-        self.Signature: bytes = b'Nemo Fi\0'
-        self.Crc: int = 0
-        self.Timestamp: datetime.date = datetime.date.today()
-        self.FileVersion: int = 0
-        self.FileVersion2: int = 0
-        self.SaveFlags: int = 0
-        self.PrewHdrPackSize: int = 0
-
-        self.DataPackSize: int = 0
-        self.DataUnpackSize: int = 0
-        self.ManagerCount: int = 0
-        self.ObjectCount: int = 0
-        self.MaxIDSaved: int = 0
-        self.ProductVersion: int = 0
-        self.ProductBuild: int = 0
-        self.PrewHdrUnpackSize: int = 0
-
-    def __str__(self):
-        return f"""File Version: {self.FileVersion:d} / {self.FileVersion2:d}
-Production (Version / Build): {self.ProductVersion:d} / {(self.ProductBuild >> 24) & 0xff:d}.{(self.ProductBuild >> 16) & 0xff:d}.{(self.ProductBuild >> 8) & 0xff:d}.{self.ProductBuild & 0xff:d}
-Crc: 0x{self.Crc:08X}
-Timestamp: {str(self.Timestamp)}
-Save Flags: {VirtoolsConstants.PyEnum.PrintEnumFlag(self.SaveFlags, VirtoolsConstants.CK_FILE_WRITEMODE)}
-
-Preview Header (Pack / Unpack): {PyCmoMisc.OutputSizeHumanReadable(self.PrewHdrPackSize)} / {PyCmoMisc.OutputSizeHumanReadable(self.PrewHdrUnpackSize)}
-Data (Pack / Unpack): {PyCmoMisc.OutputSizeHumanReadable(self.DataPackSize)} / {PyCmoMisc.OutputSizeHumanReadable(self.DataUnpackSize)}
-
-Manager Count: {self.ManagerCount:d}
-Object Count: {self.ObjectCount:d}
-Max ID Saved: {self.MaxIDSaved:d}
-"""
-
-
-class CKComposition(object):
-    def __init__(self):
-        self.Header: CKFileHeader = None
-

PyCmo/VirtoolsUtils.py

@@ -1,48 +0,0 @@
-import zlib
-import io
-
-class ZlibDecompressBuffer(object):
-    def __init__(self, _fs: io.BufferedReader, _len: int, _is_compressed: bool):
-        self.__fs: io.BufferedReader = _fs
-        self.__len: int = _len
-        self.__compressed: bool = _is_compressed
-
-        self.__pos: int = 0
-        self.__parser: zlib._Decompress = zlib.decompressobj()
-        self.__cache: bytes = b''
-        self.__cachelen: int = 0
-
-    def __ParseOnce(self) -> bytes:
-        # check remain
-        remain: int = self.__len - self.__pos
-        if remain <= 0:
-            return None
-
-        # read it and increase pos
-        read_count: int = min(remain, 1024)
-        gotten_uncompressed: bytes = self.__parser.decompress(self.__fs.read(read_count))
-        self.__pos += read_count
-
-        # everything has done, no more data, flush it and get it remained data
-        if self.__pos >= self.__len:
-            gotten_uncompressed += self.__parser.flush()
-
-        return gotten_uncompressed
-
-    def Read(self, expected: int):
-        # try enrich cache
-        while self.__cachelen < expected:
-            new_data = self.__ParseOnce()
-            if new_data is None:
-                # no more data
-                raise Exception("No more data.")
-            else:
-                self.__cache += new_data
-                self.__cachelen += len(new_data)
-
-        # change data
-        returned_data = self.__cache[:expected]
-        self.__cache = self.__cache[expected:]
-        self.__cachelen -= expected
-        
-        return returned_data