From 11d05615bae5936ad85a5047d2597a463ab37df0 Mon Sep 17 00:00:00 2001 From: yyc12345 Date: Wed, 1 Mar 2023 15:51:56 +0800 Subject: [PATCH] finish iconv code --- LibCmo/CKFileReader.cpp | 4 +- LibCmo/VTEncoding.cpp | 117 ++++++++++++++++++++++++++++++++++++++-- LibCmo/VTEncoding.hpp | 31 +++++------ 3 files changed, 127 insertions(+), 25 deletions(-) diff --git a/LibCmo/CKFileReader.cpp b/LibCmo/CKFileReader.cpp index 3799154..9cceb34 100644 --- a/LibCmo/CKFileReader.cpp +++ b/LibCmo/CKFileReader.cpp @@ -116,7 +116,7 @@ namespace LibCmo::CK2 { if (decomp_buffer != nullptr) { parser = std::unique_ptr(new(std::nothrow) CKBufferParser(decomp_buffer, doc->m_FileInfo.Hdr1UnPackSize, true)); if (parser == nullptr) { - delete[] decomp_buffer; + delete[] reinterpret_cast(decomp_buffer); return CKERROR::CKERR_OUTOFMEMORY; } } @@ -221,7 +221,7 @@ namespace LibCmo::CK2 { parser = std::unique_ptr(new(std::nothrow) CKBufferParser(decomp_buffer, doc->m_FileInfo.DataUnPackSize, true)); if (parser == nullptr) { - delete[] decomp_buffer; + delete[] reinterpret_cast(decomp_buffer); return CKERROR::CKERR_OUTOFMEMORY; } } diff --git a/LibCmo/VTEncoding.cpp b/LibCmo/VTEncoding.cpp index 4c45050..50c6b77 100644 --- a/LibCmo/VTEncoding.cpp +++ b/LibCmo/VTEncoding.cpp @@ -71,7 +71,70 @@ namespace LibCmo::EncodingHelper { #else - //todo: linux implementation + static constexpr const IconvInc = 16; + bool DoIconv(const char* enc_from, const char* enc_to, + std::string& str_from, std::string& str_to) { + iconv_t cd; + char *inbuf = nullptr, *outbuf = nullptr; + size_t inbytesleft, outbytesleft, nchars, result_len; + + // check empty + if (str_from.empty()) { + str_to.clear(); + return true; + } + + // create iconv descriptor + cd = iconv_open(enc_to, enc_from); + if (cd == (iconv_t) -1) { + // fail to create iconv descriptor + return false; + } + + // pre-resize + str_to.resize(str_from.size() + IconvInc); + // setup some variables + inbytesleft = str_from.size(); + inbuf = str_from.data(); + + outbytesleft = str_to_size(); + outbuf = str_to.data(); + + result_len = str_to.size(); + + // conv core + nchars = iconv(cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft); + while (nchars == (size_t)-1 && errno == E2BIG) { + // record the length has been converted + size_t len = outbuf - str_to.data(); + + // resize for variables + result_len += IconvInc; + outbytesleft += IconvInc; + + // resize for container + str_to.resize(result_len); + + // assign new outbuf from failed position + outbuf = str_to.data() + len; + nchars = iconv(cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft); + } + + // close iconv descriptor + iconv_close(cd); + + // check error + if (nchars == (size_t)-1) { + // failed + return false; + } else { + // success + // resize result to get correct data + str_to.resize(result_len - outbytesleft); + return true; + } + + } #endif @@ -81,8 +144,6 @@ namespace LibCmo::EncodingHelper { #if defined(LIBCMO_OS_WIN32) - const ENCODING_TOKEN ENCODING_TOKEN_DEFAULT = nullptr; - ENCODING_TOKEN CreateEncodingToken(std::string& token_string) { ENCODING_TOKEN token = new(std::nothrow) UINT(); if (token == nullptr) return ENCODING_TOKEN_DEFAULT; @@ -138,9 +199,55 @@ namespace LibCmo::EncodingHelper { #else - const ENCODING_TOKEN ENCODING_TOKEN_DEFAULT = nullptr; + static const char UTF8_SYMBOL[] = "UTF-8"; + + ENCODING_TOKEN CreateEncodingToken(std::string& token_string) { + ENCODING_TOKEN token = new(std::nothrow) char[token_string.size() + 1]; + if (token == nullptr) return ENCODING_TOKEN_DFAULT; + + std::memcpy(token, token_string.c_str(), token_string.size() + 1); + return token; + } + + void DestroyEncodingToken(ENCODING_TOKEN token) { + if (token != ENCODING_TOKEN_DEFAULT) { + delete[] token; + } + } + + void GetUtf8VirtoolsName(std::string& native_name, std::string& u8_name, ENCODING_TOKEN token) { + if (token == ENCODING_TOKEN_DEFAULT) { + u8_name = native_name.c_str(); + return; + } + + // convert with fallback + if (!DoIconv(token, UTF8_SYMBOL, native_name, u8_name)) { + u8_name = native_name.c_str(); + } + } + + void GetNativeVirtoolsName(std::string& u8_name, std::string& native_name, ENCODING_TOKEN token) { + + if (token == ENCODING_TOKEN_DEFAULT) { + native_name = u8_name.c_str(); + return; + } + + // convert with fallback + if (!DoIconv(UTF8_SYMBOL, token, u8_name, native_name)) { + native_name = u8_name.c_str(); + } + } + + void SetStdPathFromU8Path(std::filesystem::path& stdpath, const char* u8_path) { + stdpath = u8_path; + } + + FILE* OpenStdPathFile(std::filesystem::path& u8_filepath, bool is_read) { + return fopen(u8_filepath.string().c_str(), is_read ? "rb" : "wb"); + } - //todo: linux implementation #endif diff --git a/LibCmo/VTEncoding.hpp b/LibCmo/VTEncoding.hpp index 1709f47..d4b486e 100644 --- a/LibCmo/VTEncoding.hpp +++ b/LibCmo/VTEncoding.hpp @@ -29,7 +29,9 @@ namespace LibCmo::EncodingHelper { bool CharToChar(std::string& src, std::string& dest, UINT src_codepage, UINT dest_codepage); #else -#error NO IMPLEMENTATION FOR LINUX ENCODING! + + bool DoIconv(const char* enc_from, const char* enc_to, std::string& str_from, std::string& str_to); + #endif #pragma endregion @@ -38,12 +40,17 @@ namespace LibCmo::EncodingHelper { #if defined(LIBCMO_OS_WIN32) - /// - /// Token is the ticket for using encoding functions. - /// It should be created by "GenerateEncodingToken" and free by "DestroyEncodingToken". - /// + // Token is the ticket for using encoding functions. + // It should be created by "GenerateEncodingToken" and free by "DestroyEncodingToken". using ENCODING_TOKEN = UINT*; - extern const ENCODING_TOKEN ENCODING_TOKEN_DEFAULT; + constexpr const ENCODING_TOKEN ENCODING_TOKEN_DEFAULT = nullptr; + +#else + + using ENCODING_TOKEN = char*; + constexpr const ENCODING_TOKEN ENCODING_TOKEN_DEFAULT = nullptr; + +#endif ENCODING_TOKEN CreateEncodingToken(std::string& token_string); void DestroyEncodingToken(ENCODING_TOKEN token); @@ -54,18 +61,6 @@ namespace LibCmo::EncodingHelper { void SetStdPathFromU8Path(std::filesystem::path& stdpath, const char* u8_path); FILE* OpenStdPathFile(std::filesystem::path& u8_filepath, bool is_read); -#else - - /// - /// Token is the ticket for using encoding functions. - /// It should be created by "GenerateEncodingToken" and free by "DestroyEncodingToken". - /// - using ENCODING_TOKEN = char*; - extern const ENCODING_TOKEN ENCODING_TOKEN_DEFAULT; - -#error NO IMPLEMENTATION FOR LINUX ENCODING! -#endif - #pragma endregion }