From f3e7cdf004433c50c4b8583a1c729f3ec0674f34 Mon Sep 17 00:00:00 2001 From: yyc12345 Date: Mon, 28 Aug 2023 21:21:40 +0800 Subject: [PATCH] finish writing --- LibCmo/CK2/CKFile.hpp | 7 +- LibCmo/CK2/CKFileOthers.cpp | 6 +- LibCmo/CK2/CKFileReader.cpp | 10 ++- LibCmo/CK2/CKFileWriter.cpp | 154 ++++++++++++++++++++++++++++++++++-- LibCmo/CK2/CKGlobals.cpp | 8 +- LibCmo/CK2/CKGlobals.hpp | 4 +- LibCmo/VTUtils.cpp | 32 ++++---- LibCmo/VTUtils.hpp | 8 +- 8 files changed, 186 insertions(+), 43 deletions(-) diff --git a/LibCmo/CK2/CKFile.hpp b/LibCmo/CK2/CKFile.hpp index 98f88f1..c38a032 100644 --- a/LibCmo/CK2/CKFile.hpp +++ b/LibCmo/CK2/CKFile.hpp @@ -42,7 +42,8 @@ namespace LibCmo::CK2 { } LIBCMO_DISABLE_COPY_MOVE(CKBufferParser); - const void* GetPtr(void) { return (this->m_MemBegin + m_MemPos); } + const void* GetPtr(ptrdiff_t extraoff = 0) { return (this->m_MemBegin + m_MemPos + extraoff); } + void* GetMutablePtr(ptrdiff_t extraoff = 0) { return (this->m_MemBegin + m_MemPos + extraoff); } void Read(void* data, size_t data_size) { std::memcpy(data, (this->m_MemBegin + m_MemPos), data_size); this->m_MemPos += data_size; @@ -54,7 +55,7 @@ namespace LibCmo::CK2 { void* GetBase(void) { return this->m_MemBegin; } size_t GetSize(void) { return this->m_MemSize; } size_t GetCursor(void) { return this->m_MemPos; } - void MoveCursor(size_t off) { this->m_MemPos += off; } + void MoveCursor(ptrdiff_t off) { this->m_MemPos += off; } void SetCursor(size_t off) { this->m_MemPos = off; } }; @@ -239,7 +240,7 @@ namespace LibCmo::CK2 { XContainer::XArray m_ManagersData; /**< Manager Data loaded */ XContainer::XArray m_PluginsDep; /**< Plugins dependencies for this file */ XContainer::XArray m_IncludedFiles; /**< List of files that should be inserted in the CMO file. */ - //CKFileInfo m_FileInfo; /**< Headers summary */ + CKFileInfo m_FileInfo; /**< Headers summary */ CKERROR PrepareFile(CKSTRING filename); diff --git a/LibCmo/CK2/CKFileOthers.cpp b/LibCmo/CK2/CKFileOthers.cpp index 3d3e8ca..64ad70b 100644 --- a/LibCmo/CK2/CKFileOthers.cpp +++ b/LibCmo/CK2/CKFileOthers.cpp @@ -164,14 +164,16 @@ namespace LibCmo::CK2 { m_Ctx(ctx), m_Visitor(this), m_Done(false), m_IsCopyFromReader(false), m_SaveIDMax(0), - m_FileObjects(), m_ManagersData(), m_PluginsDep(), m_IncludedFiles() + m_FileObjects(), m_ManagersData(), m_PluginsDep(), m_IncludedFiles(), + m_FileInfo() {} CKFileWriter::CKFileWriter(CKContext* ctx, CKFileReader* reader) : m_Ctx(ctx), m_Visitor(this), m_Done(false), m_IsCopyFromReader(true), m_SaveIDMax(0), - m_FileObjects(), m_ManagersData(), m_PluginsDep(), m_IncludedFiles() + m_FileObjects(), m_ManagersData(), m_PluginsDep(), m_IncludedFiles(), + m_FileInfo() { // sync save id max this->m_SaveIDMax = reader->GetSaveIdMax(); diff --git a/LibCmo/CK2/CKFileReader.cpp b/LibCmo/CK2/CKFileReader.cpp index 6e984a7..835f1b5 100644 --- a/LibCmo/CK2/CKFileReader.cpp +++ b/LibCmo/CK2/CKFileReader.cpp @@ -100,9 +100,11 @@ namespace LibCmo::CK2 { parser->SetCursor(sizeof(CKRawFileInfo)); // compare size to decide wheher use compress feature - void* decomp_buffer = CKUnPackData(this->m_FileInfo.Hdr1UnPackSize, parser->GetPtr(), this->m_FileInfo.Hdr1PackSize); - if (decomp_buffer != nullptr) { - parser = std::unique_ptr(new CKBufferParser(decomp_buffer, this->m_FileInfo.Hdr1UnPackSize, true)); + if (this->m_FileInfo.Hdr1PackSize != this->m_FileInfo.Hdr1UnPackSize) { + void* decomp_buffer = CKUnPackData(this->m_FileInfo.Hdr1UnPackSize, parser->GetPtr(), this->m_FileInfo.Hdr1PackSize); + if (decomp_buffer != nullptr) { + parser = std::unique_ptr(new CKBufferParser(decomp_buffer, this->m_FileInfo.Hdr1UnPackSize, true)); + } } } @@ -312,7 +314,7 @@ namespace LibCmo::CK2 { // read file body FILE* fp = m_Ctx->OpenTempFile(file.c_str(), "wb"); if (fp != nullptr) { - StreamHelper::CopyStream(parser->GetPtr(), fp, filebodylen); + fwrite(parser->GetPtr(), sizeof(char), filebodylen, fp); fclose(fp); } diff --git a/LibCmo/CK2/CKFileWriter.cpp b/LibCmo/CK2/CKFileWriter.cpp index 0211429..db7b5a1 100644 --- a/LibCmo/CK2/CKFileWriter.cpp +++ b/LibCmo/CK2/CKFileWriter.cpp @@ -12,17 +12,20 @@ namespace LibCmo::CK2 { // check document status if (this->m_Done) CKERROR::CKERR_CANCELLED; + // encoding conv helper + std::string name_conv; + // try detect filename legality CKERROR err = PrepareFile(u8_filename); if (err != CKERROR::CKERR_OK) return err; - + // ========== Prepare Stage ========== // todo: add TOBEDELETED flag for all Referenced objects's m_ObjectFlags // MARK: ignore the notification to all CKBehavior based objects. // ========== StateChunk convertion ========== - + // iterate all objects and transform it into CKStateChunk // MARK: Drop the support of collecting the sum of InterfaceChunk's size. // because it is useless. @@ -128,18 +131,20 @@ namespace LibCmo::CK2 { // calc the remains for (size_t i = 1; i < m_FileObjects.size(); ++i) { // prev obj PackSize + prev obj FileIndex + prev obj chunk size - m_FileObjects[i].FileIndex = m_FileObjects[i - 1].FileIndex + m_FileObjects[i - 1].PackSize + sizeof(CKDWORD); + m_FileObjects[i].FileIndex = m_FileObjects[i - 1].FileIndex + m_FileObjects[i - 1].PackSize + sizeof(CKDWORD); } } // ========== Construct File Header ========== + CK_FILE_WRITEMODE fileWriteMode = m_Ctx->GetFileWriteMode(); + CKRawFileInfo rawHeader; std::memcpy(&rawHeader.NeMo, CKNEMOFI, sizeof(CKRawFileInfo::NeMo)); rawHeader.Crc = 0; rawHeader.Zero = 0; rawHeader.CKVersion = CKVERSION; rawHeader.FileVersion = 8; - rawHeader.FileWriteMode = static_cast(m_Ctx->GetFileWriteMode()); + rawHeader.FileWriteMode = static_cast(fileWriteMode); rawHeader.ObjectCount = static_cast(m_FileObjects.size()); rawHeader.ManagerCount = static_cast(m_ManagersData.size()); rawHeader.Hdr1UnPackSize = sumHdrSize; @@ -151,10 +156,7 @@ namespace LibCmo::CK2 { rawHeader.MaxIDSaved = m_SaveIDMax; // crc will filled later - // create a encoding conversion helper string - std::string name_conv; - - // ========== Writing header ========== + // ========== Write header ========== // create a buffer std::unique_ptr hdrparser(new CKBufferParser(sumHdrSize)); @@ -190,6 +192,142 @@ namespace LibCmo::CK2 { } } + // write included file + { + CKDWORD cache = sizeof(CKDWORD); + hdrparser->Write(&cache, sizeof(CKDWORD)); + + cache = static_cast(m_IncludedFiles.size()); + hdrparser->Write(&cache, sizeof(CKDWORD)); + } + + // compress header if needed + if (EnumsHelper::Has(fileWriteMode, CK_FILE_WRITEMODE::CKFILE_CHUNKCOMPRESSED_OLD) || + EnumsHelper::Has(fileWriteMode, CK_FILE_WRITEMODE::CKFILE_WHOLECOMPRESSED)) { + + CKDWORD comp_buf_size = 0; + void* comp_buffer = CKPackData(hdrparser->GetBase(), static_cast(hdrparser->GetSize()), comp_buf_size, m_Ctx->GetCompressionLevel()); + if (comp_buffer != nullptr) { + hdrparser = std::unique_ptr(new CKBufferParser(comp_buffer, comp_buf_size, true)); + rawHeader.Hdr1PackSize = comp_buf_size; + } else { + // fallback + rawHeader.Hdr1PackSize = rawHeader.Hdr1UnPackSize; + } + } + + // ========== Write data ========== + // create a buffer + std::unique_ptr datparser(new CKBufferParser(sumDataSize)); + + // write manager + for (auto& mgr : m_ManagersData) { + datparser->Write(&mgr.Manager, sizeof(CKGUID)); + + CKDWORD writtenSize = 0; + if (mgr.Data != nullptr) { + writtenSize = mgr.Data->ConvertToBuffer(datparser->GetMutablePtr(sizeof(CKDWORD))); + delete mgr.Data; + mgr.Data = nullptr; + } + + datparser->Write(&writtenSize, sizeof(CKDWORD)); + datparser->MoveCursor(writtenSize); + } + + // write object + for (auto& obj : m_FileObjects) { + datparser->Write(&obj.PackSize, sizeof(CKDWORD)); + + if (obj.Data != nullptr) { + obj.Data->ConvertToBuffer(datparser->GetMutablePtr()); + delete obj.Data; + obj.Data = nullptr; + } + + datparser->MoveCursor(obj.PackSize); + } + + // compress header if needed + if (EnumsHelper::Has(fileWriteMode, CK_FILE_WRITEMODE::CKFILE_CHUNKCOMPRESSED_OLD) || + EnumsHelper::Has(fileWriteMode, CK_FILE_WRITEMODE::CKFILE_WHOLECOMPRESSED)) { + + CKDWORD comp_buf_size = 0; + void* comp_buffer = CKPackData(datparser->GetBase(), static_cast(datparser->GetSize()), comp_buf_size, m_Ctx->GetCompressionLevel()); + if (comp_buffer != nullptr) { + datparser = std::unique_ptr(new CKBufferParser(comp_buffer, comp_buf_size, true)); + rawHeader.DataPackSize = comp_buf_size; + } else { + // fallback + rawHeader.DataPackSize = rawHeader.DataUnPackSize; + } + } + + // ========== Construct File Info ========== + // compute crc + CKDWORD computedcrc = CKComputeDataCRC(&rawHeader, sizeof(CKRawFileInfo), 0u); + computedcrc = CKComputeDataCRC(hdrparser->GetBase(), hdrparser->GetSize(), computedcrc); + computedcrc = CKComputeDataCRC(datparser->GetBase(), datparser->GetSize(), computedcrc); + + // copy to file info + this->m_FileInfo.ProductVersion = rawHeader.ProductVersion; + this->m_FileInfo.ProductBuild = rawHeader.ProductBuild; + this->m_FileInfo.FileWriteMode = static_cast(rawHeader.FileWriteMode); + this->m_FileInfo.CKVersion = rawHeader.CKVersion; + this->m_FileInfo.FileVersion = rawHeader.FileVersion; + this->m_FileInfo.Hdr1PackSize = rawHeader.Hdr1PackSize; + this->m_FileInfo.Hdr1UnPackSize = rawHeader.Hdr1UnPackSize; + this->m_FileInfo.DataPackSize = rawHeader.DataPackSize; + this->m_FileInfo.DataUnPackSize = rawHeader.DataUnPackSize; + this->m_FileInfo.ManagerCount = rawHeader.ManagerCount; + this->m_FileInfo.ObjectCount = rawHeader.ObjectCount; + this->m_FileInfo.MaxIDSaved = rawHeader.MaxIDSaved; + // fill file size and crc + this->m_FileInfo.FileSize = sizeof(CKRawFileInfo) + rawHeader.DataPackSize + rawHeader.Hdr1PackSize; + this->m_FileInfo.Crc = computedcrc; + rawHeader.Crc = computedcrc; + + // ========== Open File & Write Essential Data ========== + // open file and test + FILE* fs = m_Ctx->OpenFile(u8_filename, "wb"); + if (fs == nullptr) return CKERROR::CKERR_CANTWRITETOFILE; + // write small header + header + data + std::fwrite(&rawHeader, sizeof(CKRawFileInfo), 1, fs); + std::fwrite(hdrparser->GetBase(), sizeof(char), hdrparser->GetSize(), fs); + std::fwrite(datparser->GetBase(), sizeof(char), datparser->GetSize(), fs); + // free buffer + hdrparser.reset(); + datparser.reset(); + + // ========== Included Files ========== + for (auto& fentry : m_IncludedFiles) { + // write filename + m_Ctx->GetNativeString(fentry, name_conv); + CKDWORD filenamelen = static_cast(name_conv.size()); + fwrite(&filenamelen, sizeof(CKDWORD), 1, fs); + fwrite(name_conv.data(), sizeof(char), filenamelen, fs); + + // try mapping file. + std::unique_ptr mappedFile(new VxMath::VxMemoryMappedFile(fentry.c_str())); + + // write file length + CKDWORD filebodylen = static_cast(mappedFile->IsValid() ? mappedFile->GetFileSize() : 0); + fwrite(&filebodylen, sizeof(CKDWORD), 1, fs); + + // write file body + if (mappedFile->IsValid()) { + fwrite(mappedFile->GetBase(), sizeof(char), filebodylen, fs); + } + + // release mapped file + mappedFile.reset(); + + } + + // close file + fclose(fs); + + return CKERROR::CKERR_OK; } CKERROR CKFileWriter::PrepareFile(CKSTRING filename) { diff --git a/LibCmo/CK2/CKGlobals.cpp b/LibCmo/CK2/CKGlobals.cpp index 87409de..f185e47 100644 --- a/LibCmo/CK2/CKGlobals.cpp +++ b/LibCmo/CK2/CKGlobals.cpp @@ -16,7 +16,7 @@ namespace LibCmo::CK2 { #pragma region Compression utilities - void* CKPackData(const void* Data, CKINT size, CKINT& NewSize, CKINT compressionlevel) { + void* CKPackData(const void* Data, CKDWORD size, CKDWORD& NewSize, CKINT compressionlevel) { uLong boundary = compressBound(static_cast(size)); char* DestBuffer = new char[boundary]; @@ -30,17 +30,17 @@ namespace LibCmo::CK2 { return nullptr; } - NewSize = static_cast(_destLen); + NewSize = static_cast(_destLen); return DestBuffer; } - void* CKUnPackData(CKINT DestSize, const void* SrcBuffer, CKINT SrcSize) { + void* CKUnPackData(CKDWORD DestSize, const void* SrcBuffer, CKDWORD SrcSize) { char* DestBuffer = new char[DestSize]; uLongf cache = DestSize; if (uncompress( reinterpret_cast(DestBuffer), &cache, - reinterpret_cast(SrcBuffer), SrcSize) != Z_OK) { + reinterpret_cast(SrcBuffer), static_cast(SrcSize)) != Z_OK) { delete[] DestBuffer; return nullptr; } diff --git a/LibCmo/CK2/CKGlobals.hpp b/LibCmo/CK2/CKGlobals.hpp index de31d93..d652b6e 100644 --- a/LibCmo/CK2/CKGlobals.hpp +++ b/LibCmo/CK2/CKGlobals.hpp @@ -20,7 +20,7 @@ namespace LibCmo::CK2 { * NewSize only indicate the size of the part storing useful data in return value. * @see CKUnPackData, CKComputeDataCRC */ - void* CKPackData(const void* Data, CKINT size, CKINT& NewSize, CKINT compressionlevel); + void* CKPackData(const void* Data, CKDWORD size, CKDWORD& NewSize, CKINT compressionlevel); /** * @brief Decompress a buffer * @param[in] DestSize Expected size of the decompressed buffer. @@ -31,7 +31,7 @@ namespace LibCmo::CK2 { * The return pointer should be freed by `delete[]` manually. * @see CKPackData, CKComputeDataCRC */ - void* CKUnPackData(CKINT DestSize, const void* SrcBuffer, CKINT SrcSize); + void* CKUnPackData(CKDWORD DestSize, const void* SrcBuffer, CKDWORD SrcSize); /** * @brief Computes a CRC for a buffer. * @param[in] data A pointer to the buffer to create a CRC for. diff --git a/LibCmo/VTUtils.cpp b/LibCmo/VTUtils.cpp index 963e093..9da1d7c 100644 --- a/LibCmo/VTUtils.cpp +++ b/LibCmo/VTUtils.cpp @@ -8,23 +8,23 @@ namespace LibCmo { std::abort(); } - namespace StreamHelper { + //namespace StreamHelper { - static constexpr const size_t CHUNK_SIZE = 10240; - void CopyStream(const void* src, FILE* dest, size_t len) { - fwrite(src, sizeof(char), len, dest); - } - void CopyStream(FILE* src, void* dest, size_t len) { - size_t expected_size = 0u; - char* p = reinterpret_cast(dest); + // static constexpr const size_t CHUNK_SIZE = 10240; + // void CopyStream(const void* src, FILE* dest, size_t len) { + // fwrite(src, sizeof(char), len, dest); + // } + // void CopyStream(FILE* src, void* dest, size_t len) { + // size_t expected_size = 0u; + // char* p = reinterpret_cast(dest); - while (len != 0) { - expected_size = len < CHUNK_SIZE ? len : CHUNK_SIZE; - fread(p, sizeof(char), expected_size, src); - p += expected_size; - len -= expected_size; - } - } + // while (len != 0) { + // expected_size = len < CHUNK_SIZE ? len : CHUNK_SIZE; + // fread(p, sizeof(char), expected_size, src); + // p += expected_size; + // len -= expected_size; + // } + // } - } + //} } diff --git a/LibCmo/VTUtils.hpp b/LibCmo/VTUtils.hpp index 8c27bb9..b185001 100644 --- a/LibCmo/VTUtils.hpp +++ b/LibCmo/VTUtils.hpp @@ -164,10 +164,10 @@ namespace LibCmo { } } - namespace StreamHelper { + //namespace StreamHelper { - void CopyStream(const void* src, FILE* dest, size_t len); - void CopyStream(FILE* src, void* dest, size_t len); + // void CopyStream(const void* src, FILE* dest, size_t len); + // void CopyStream(FILE* src, void* dest, size_t len); - } + //} }