diff --git a/materializer/DataTypes.hpp b/materializer/DataTypes.hpp index 637bef9..a9fd5f5 100644 --- a/materializer/DataTypes.hpp +++ b/materializer/DataTypes.hpp @@ -4,6 +4,11 @@ namespace VSW::Materializer::DataTypes { + struct BlobDescriptor { + const void* ptr; + int length; + }; + namespace Script { struct Table_script { @@ -130,7 +135,7 @@ namespace VSW::Materializer::DataTypes { struct Table_data { YYCC::yycc_u8string field; - YYCC::yycc_u8string data; + BlobDescriptor data; CK_ID parent; }; diff --git a/materializer/ExportDocument.cpp b/materializer/ExportDocument.cpp index 1423eb8..afb6c04 100644 --- a/materializer/ExportDocument.cpp +++ b/materializer/ExportDocument.cpp @@ -2,7 +2,7 @@ #include "Database.hpp" #include "DataTypes.hpp" #include "Utilities.hpp" -#include +#include namespace VSW::Materializer::ExportDocument { diff --git a/materializer/ExportScript.cpp b/materializer/ExportScript.cpp index 792ecfd..135429c 100644 --- a/materializer/ExportScript.cpp +++ b/materializer/ExportScript.cpp @@ -1,12 +1,175 @@ #include "ExportCore.hpp" +#include "Database.hpp" +#include "DataTypes.hpp" +#include "Utilities.hpp" +#include +#include namespace VSW::Materializer::ExportScript { + struct ExportContext { + ExportContext(CKContext* ctx, const YYCC::yycc_u8string_view& db_path, UINT code_page) : + db(db_path), cache(), reporter(ctx), cp(code_page), attr_set(), param_mgr(ctx->GetParameterManager()) {} + Database::ScriptDatabase db; + DataTypes::Script::DataCache cache; + Utilities::EnhancedReporter reporter; + UINT cp; + /// @brief Variable for removing duplicated exported attributes. + std::set attr_set; + CKParameterManager* param_mgr; + }; + +#pragma region Assist Functions + + static void DataDictWriter(ExportContext& expctx, const YYCC::yycc_u8string_view& field, const void* data, size_t data_length, CK_ID parent) { + // check given data length + using db_size_t = decltype(expctx.cache.data.data.length); + if (data_length > static_cast(std::numeric_limits::max())) + throw std::runtime_error("Too long data length when exporting to data dictionary."); + // write data + expctx.cache.data.field = field; + expctx.cache.data.data.ptr = data; + expctx.cache.data.data.length = static_cast(data_length); + expctx.cache.data.parent = parent; + expctx.db.Write(expctx.cache.data); + } + + static void DataDictWriter(ExportContext& expctx, const YYCC::yycc_u8string_view& field, const YYCC::yycc_u8string& data, CK_ID parent) { + DataDictWriter(expctx, field, data.c_str(), data.length(), parent); + } + + template, int> = 0> + static void DataDictWriter(ExportContext& expctx, const YYCC::yycc_u8string_view& field, const _Ty& data, CK_ID parent) { + DataDictWriter(expctx, field, &data, sizeof(_Ty), parent); + } + +#define REC_RAW_DATA(field, data, len) DataDictWriter(expctx, YYCC_U8(field), data, len, parent) +#define REC_DATA(field, data) DataDictWriter(expctx, YYCC_U8(field), data, parent) + + void DigParameterData(ExportContext& expctx, CKParameter* p, CK_ID parent) { + // According to our algorithm, CKParameter passed in this function can not be duplicated. + // So we don't need check it anymore. + // Get CKGUID and Parameter Type first + CKGUID t = p->GetGUID(); + CKParameterType pt = p->GetType(); + + // Record GUID + int64_t exported_t; + CP_GUID(exported_t, t); + REC_DATA("dumper-guid", exported_t); + // Record Parameter Type name + YYCC::yycc_u8string exported_pt; + CP_CKSTR(exported_pt, expctx.param_mgr->ParameterTypeToName(pt)); + REC_DATA("dumper-type-name", exported_pt); + + // Detect value object scenario (associated with an existing CKObject) + if (p->GetParameterClassID() && p->GetValueObject(false)) { + // Record CK_ID of associated object + CKObject* assoc_obj = p->GetValueObject(false); + REC_DATA("dumper-assoc-ckobj", assoc_obj->GetID()); + return; + } + + // Nothing + if (t == CKPGUID_NONE) return; + + // Float value + if (t == CKPGUID_FLOAT || t == CKPGUID_ANGLE || t == CKPGUID_PERCENTAGE || t == CKPGUID_TIME +#if defined(VIRTOOLS_50) || defined(VIRTOOLS_40) || defined(VIRTOOLS_35) + || t == CKPGUID_FLOATSLIDER +#endif + ) { + REC_DATA("dumper-float", *static_cast(p->GetReadDataPtr(false))); + return; + } + // Integral value + if (t == CKPGUID_INT || t == CKPGUID_KEY || t == CKPGUID_BOOL || t == CKPGUID_ID || t == CKPGUID_POINTER + || t == CKPGUID_MESSAGE || t == CKPGUID_ATTRIBUTE || t == CKPGUID_BLENDMODE || t == CKPGUID_FILTERMODE + || t == CKPGUID_BLENDFACTOR || t == CKPGUID_FILLMODE || t == CKPGUID_LITMODE || t == CKPGUID_SHADEMODE + || t == CKPGUID_ADDRESSMODE || t == CKPGUID_WRAPMODE || t == CKPGUID_3DSPRITEMODE || t == CKPGUID_FOGMODE + || t == CKPGUID_LIGHTTYPE || t == CKPGUID_SPRITEALIGN || t == CKPGUID_DIRECTION || t == CKPGUID_LAYERTYPE + || t == CKPGUID_COMPOPERATOR || t == CKPGUID_BINARYOPERATOR || t == CKPGUID_SETOPERATOR + || t == CKPGUID_OBSTACLEPRECISION || t == CKPGUID_OBSTACLEPRECISIONBEH) { + REC_DATA("dumper-int", *static_cast(p->GetReadDataPtr(false))); + return; + } + // Vector value + if (t == CKPGUID_VECTOR) { + REC_DATA("dumper-vector", *static_cast(p->GetReadDataPtr(false))); + return; + } + if (t == CKPGUID_2DVECTOR) { + REC_DATA("dumper-2dvector", *static_cast(p->GetReadDataPtr(false))); + return; + } + if (t == CKPGUID_MATRIX) { + REC_DATA("dumper-matrix", *static_cast(p->GetReadDataPtr(false))); + return; + } + if (t == CKPGUID_COLOR) { + REC_DATA("dumper-color", *static_cast(p->GetReadDataPtr(false))); + return; + } + // 2D Curve value + if (t == CKPGUID_2DCURVE) { + // Get instance + CK2dCurve* c = static_cast(p->GetReadDataPtr(false)); + // We build our unique binary 2d curve data. + Utilities::Curve2DBuilder builder(c); + REC_RAW_DATA("dumper-2d-curve", builder.GetDataPtr(), builder.GetDataLength()); + return; + } + // String value + if (t == CKPGUID_STRING) { + // Virtools internal data may not have null terminal, + // so we use need copy it into container first. + std::string native_string( + static_cast(p->GetReadDataPtr(false)), + static_cast(p->GetDataSize()) + ); + // Then do encoding convertion + YYCC::yycc_u8string utf8_string; + if (!YYCC::EncodingHelper::CharToUTF8(native_string, utf8_string, expctx.cp)) { + // If failed, report error + expctx.reporter.Err(YYCC_U8("Fail to convert string encoding for data of CKParameter. Some value may be empty!")); + // reset to blank string + utf8_string.clear(); + } + REC_DATA("dumper-string", utf8_string); + return; + } + // If it gets here, we have no idea what it really is. so simply dump it. + // Buffer-like + if (t == CKPGUID_VOIDBUF +#if defined(VIRTOOLS_50) || defined(VIRTOOLS_40) || defined(VIRTOOLS_35) + || t == CKPGUID_SHADER || t == CKPGUID_TECHNIQUE || t == CKPGUID_PASS +#endif + || true // All unknown type goes there. + ) { + // Raw data is similar with string, + // but we don't need do encoding convertion. + const void* data_ptr = p->GetReadDataPtr(false); + size_t data_len = static_cast(p->GetDataSize()); + REC_RAW_DATA("dumper-raw", data_ptr, data_len); + return; + } + + } + + +#pragma endregion void Export(CKContext* ctx, const YYCC::yycc_u8string_view& db_path, UINT code_page) { + // create export context + ExportContext expctx(ctx, db_path, code_page); + if (!expctx.db.IsValid()) { + expctx.reporter.Err(YYCC_U8("Fail to open database. Export process aborted.")); + return; + } + // export script } } diff --git a/materializer/Utilities.cpp b/materializer/Utilities.cpp index b506efc..49c9bb6 100644 --- a/materializer/Utilities.cpp +++ b/materializer/Utilities.cpp @@ -14,6 +14,79 @@ namespace VSW::Materializer::Utilities { m_Ctx->OutputToConsole(const_cast(YYCC::EncodingHelper::UTF8ToChar(strl, CP_ACP).c_str()), FALSE); } +#pragma endregion + +#pragma region Curve 2D Builder + + Curve2DBuilder::Curve2DBuilder(CK2dCurve* curve_2d) : m_Cache() { + BuildCurve(curve_2d); + } + + Curve2DBuilder::~Curve2DBuilder() {} + + const void* Curve2DBuilder::GetDataPtr() const { return m_Cache.c_str(); } + size_t Curve2DBuilder::GetDataLength() const { return m_Cache.size(); } + + void Curve2DBuilder::BuildCurve(CK2dCurve* c) { + // check curve + if (c == nullptr) return; + // get curve control point count + int cp_count = c->GetControlPointCount(); + // reserve enough space + // count * (x + y + is_linear + is_tcb + tcb_tuple) + m_Cache.reserve(static_cast(cp_count) * (sizeof(float) * 2u + sizeof(float) * 3u + sizeof(uint32_t) + sizeof(uint32_t))); + + // iterate control point + for (int i = 0; i < cp_count; ++i) { + BuildCurvePoint(c->GetControlPoint(i)); + } + } + + void Curve2DBuilder::BuildCurvePoint(CK2dCurvePoint* cp) { + // check control point + if (cp == nullptr) return; + // prepare variable + uint32_t int_cache; + Vx2DVector vector_cache; + float float_cache; + +#define APPEND_DATA(data) m_Cache.append(reinterpret_cast(&data), sizeof(data)) + + // x y value + vector_cache = cp->GetPosition(); + APPEND_DATA(vector_cache); + + // is linear + int_cache = static_cast(cp->IsLinear()); + APPEND_DATA(int_cache); + // is tcb + int_cache = static_cast(cp->IsTCB()); + APPEND_DATA(int_cache); + + if (cp->IsTCB()) { + // TCB control point + float_cache = cp->GetTension(); + APPEND_DATA(float_cache); + float_cache = cp->GetContinuity(); + APPEND_DATA(float_cache); + float_cache = cp->GetBias(); + APPEND_DATA(float_cache); + } else { + // non-TCB control point + float_cache = cp->GetInTangent(); + APPEND_DATA(float_cache); + float_cache = cp->GetOutTangent(); + APPEND_DATA(float_cache); + // To keep balance with TCB control point, + // We add a blank 0.0f in there + float_cache = 0.0f; + APPEND_DATA(float_cache); + } + +#undef APPEND_DATA + + } + #pragma endregion void RelativeAddress(const EnhancedReporter& reporter, YYCC::yycc_u8string& relative_addr_str, const void* absolute_addr) { @@ -59,6 +132,35 @@ namespace VSW::Materializer::Utilities { } } + //void GetBase64(YYCC::yycc_u8string& dst, const char* data, size_t data_len) { + // // Reference: https://stackoverflow.com/questions/342409/how-do-i-base64-encode-decode-in-c + // static const YYCC::yycc_char8_t* BASE64_TABLE = YYCC_U8("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"); + // static const int MOD_TABLE[] = { 0, 2, 1 }; + + // // compute size + // size_t output_length = 4u * ((data_len + 2u) / 3u); + // dst.resize(output_length); + + // // compute + // for (size_t i = 0, j = 0; i < data_len;) { + + // uint32_t octet_a = i < data_len ? (unsigned char)data[i++] : 0; + // uint32_t octet_b = i < data_len ? (unsigned char)data[i++] : 0; + // uint32_t octet_c = i < data_len ? (unsigned char)data[i++] : 0; + + // uint32_t triple = (octet_a << 0x10) + (octet_b << 0x08) + octet_c; + + // dst[j++] = BASE64_TABLE[(triple >> 3 * 6) & 0x3F]; + // dst[j++] = BASE64_TABLE[(triple >> 2 * 6) & 0x3F]; + // dst[j++] = BASE64_TABLE[(triple >> 1 * 6) & 0x3F]; + // dst[j++] = BASE64_TABLE[(triple >> 0 * 6) & 0x3F]; + // } + + // // fill blank + // for (int i = 0; i < MOD_TABLE[data_len % 3]; i++) + // dst[output_length - 1 - i] = '='; + //} + void CopyStrGuid(const EnhancedReporter& reporter, YYCC::yycc_u8string& dst, const CKGUID& src) { if (!YYCC::StringHelper::Printf(dst, YYCC_U8("<0x%08" PRIX32 ", 0x%08" PRIX32 ">"), src.d1, src.d2)) { reporter.Err(YYCC_U8("Fail to format CKGUID. Some stringified GUID may be empty.")); @@ -66,7 +168,7 @@ namespace VSW::Materializer::Utilities { } } - void CopyGuid(const EnhancedReporter& reporter, int64_t& dst, const CKGUID& src) { + void CopyGuid(int64_t& dst, const CKGUID& src) { // reset dst to zero uint64_t* pdst = reinterpret_cast(&dst); // CKGUID.d1 to high 32 bits diff --git a/materializer/Utilities.hpp b/materializer/Utilities.hpp index 4473b8e..81c715b 100644 --- a/materializer/Utilities.hpp +++ b/materializer/Utilities.hpp @@ -20,6 +20,21 @@ namespace VSW::Materializer::Utilities { CKContext* m_Ctx; }; + class Curve2DBuilder { + public: + Curve2DBuilder(CK2dCurve* curve_2d); + ~Curve2DBuilder(); + + public: + const void* GetDataPtr() const; + size_t GetDataLength() const; + + private: + void BuildCurve(CK2dCurve* c); + void BuildCurvePoint(CK2dCurvePoint* cp); + std::basic_string m_Cache; + }; + /** * @brief Get relative address from given absolute address * @details This function is used when exporting function pointer into database. @@ -30,7 +45,7 @@ namespace VSW::Materializer::Utilities { */ void RelativeAddress(const EnhancedReporter& reporter, YYCC::yycc_u8string& relative_addr_str, const void* absolute_addr); void CopyStrGuid(const EnhancedReporter& reporter, YYCC::yycc_u8string& dst, const CKGUID& src); - void CopyGuid(const EnhancedReporter& reporter, int64_t& dst, const CKGUID& src); + void CopyGuid(int64_t& dst, const CKGUID& src); void CopyCKString( const EnhancedReporter& reporter, YYCC::yycc_u8string& storage, @@ -43,7 +58,7 @@ namespace VSW::Materializer::Utilities { #define CP_ADDR(dst, src) ::VSW::Materializer::Utilities::RelativeAddress(expctx.reporter, (dst), (src)) #define CP_STR_GUID(dst, src) ::VSW::Materializer::Utilities::CopyStrGuid(expctx.reporter, (dst), (src)) -#define CP_GUID(dst, src) ::VSW::Materializer::Utilities::CopyGuid(expctx.reporter, (dst), (src)) +#define CP_GUID(dst, src) ::VSW::Materializer::Utilities::CopyGuid((dst), (src)) #define CP_CKSTR(dst, src, ...) ::VSW::Materializer::Utilities::CopyCKString(expctx.reporter, (dst), (src), expctx.cp, ##__VA_ARGS__) #pragma endregion