From 1188a48c944aeb650ced19008646b6529c341c22 Mon Sep 17 00:00:00 2001 From: Shauren Date: Fri, 16 Mar 2018 18:44:57 +0100 Subject: Core/DataStores: Implemented WDC2 --- src/common/DataStores/DB2FileLoader.cpp | 424 ++++++++++++++++++++++---------- 1 file changed, 288 insertions(+), 136 deletions(-) (limited to 'src/common/DataStores/DB2FileLoader.cpp') diff --git a/src/common/DataStores/DB2FileLoader.cpp b/src/common/DataStores/DB2FileLoader.cpp index 712c96eb3fe..d429195445b 100644 --- a/src/common/DataStores/DB2FileLoader.cpp +++ b/src/common/DataStores/DB2FileLoader.cpp @@ -29,11 +29,24 @@ enum class DB2ColumnCompression : uint32 Immediate, CommonData, Pallet, - PalletArray + PalletArray, + SignedImmediate }; #pragma pack(push, 1) +struct DB2SectionHeader +{ + uint64 TactKeyId; + uint32 FileOffset; + uint32 RecordCount; + uint32 StringTableSize; + uint32 CopyTableSize; + uint32 CatalogDataOffset; + uint32 IdTableSize; + uint32 ParentLookupDataSize; +}; + struct DB2FieldEntry { int16 UnusedBits; @@ -107,6 +120,11 @@ struct DB2IndexData std::unique_ptr Entries; }; +DB2FieldMeta::DB2FieldMeta(bool isSigned, DBCFormer type, char const* name) + : IsSigned(isSigned), Type(type), Name(name) +{ +} + DB2FileLoadInfo::DB2FileLoadInfo() : Fields(nullptr), FieldCount(0), Meta(nullptr) { } @@ -134,7 +152,7 @@ std::pair DB2FileLoadInfo::GetFieldIndexByName(char const* fieldNa std::size_t ourIndex = Meta->HasIndexFieldInData() ? 0 : 1; for (uint32 i = 0; i < Meta->FieldCount; ++i) { - for (uint8 arr = 0; arr < Meta->ArraySizes[i]; ++arr) + for (uint8 arr = 0; arr < Meta->Fields[i].ArraySize; ++arr) { if (!strcmp(Fields[ourIndex].Name, fieldName)) return std::make_pair(int32(i), int32(arr)); @@ -154,12 +172,12 @@ class DB2FileLoaderImpl { public: virtual ~DB2FileLoaderImpl() { } - virtual bool LoadTableData(DB2FileSource* source) = 0; - virtual bool LoadCatalogData(DB2FileSource* source) = 0; - virtual void SetAdditionalData(std::unique_ptr fields, std::unique_ptr idTable, std::unique_ptr copyTable, - std::unique_ptr columnMeta, std::unique_ptr[]> palletValues, - std::unique_ptr[]> palletArrayValues, std::unique_ptr[]> commonValues, - std::unique_ptr parentIndexes) = 0; + virtual void LoadColumnData(std::unique_ptr sections, std::unique_ptr fields, std::unique_ptr columnMeta, + std::unique_ptr[]> palletValues, std::unique_ptr[]> palletArrayValues, + std::unique_ptr[]> commonValues) = 0; + virtual bool LoadTableData(DB2FileSource* source, uint32 section) = 0; + virtual bool LoadCatalogData(DB2FileSource* source, uint32 section) = 0; + virtual void SetAdditionalData(std::vector idTable, std::vector copyTable, std::vector parentIndexes) = 0; virtual char* AutoProduceData(uint32& count, char**& indexTable, std::vector& stringPool) = 0; virtual char* AutoProduceStrings(char** indexTable, uint32 indexTableSize, uint32 locale) = 0; virtual void AutoProduceRecordCopies(uint32 records, char** indexTable, char* dataTable) = 0; @@ -169,6 +187,8 @@ public: virtual uint32 GetRecordCopyCount() const = 0; virtual uint32 GetMaxId() const = 0; virtual DB2FileLoadInfo const* GetLoadInfo() const = 0; + virtual DB2SectionHeader const& GetSection(uint32 section) const = 0; + virtual bool IsSignedField(uint32 field) const = 0; private: friend class DB2Record; @@ -191,12 +211,12 @@ public: DB2FileLoaderRegularImpl(char const* fileName, DB2FileLoadInfo const* loadInfo, DB2Header const* header); ~DB2FileLoaderRegularImpl(); - bool LoadTableData(DB2FileSource* source) override; - bool LoadCatalogData(DB2FileSource* /*source*/) override { return true; } - void SetAdditionalData(std::unique_ptr fields, std::unique_ptr idTable, std::unique_ptr copyTable, - std::unique_ptr columnMeta, std::unique_ptr[]> palletValues, - std::unique_ptr[]> palletArrayValues, std::unique_ptr[]> commonValues, - std::unique_ptr parentIndexes) override; + void LoadColumnData(std::unique_ptr sections, std::unique_ptr fields, std::unique_ptr columnMeta, + std::unique_ptr[]> palletValues, std::unique_ptr[]> palletArrayValues, + std::unique_ptr[]> commonValues) override; + bool LoadTableData(DB2FileSource* source, uint32 section) override; + bool LoadCatalogData(DB2FileSource* /*source*/, uint32 /*section*/) override { return true; } + void SetAdditionalData(std::vector idTable, std::vector copyTable, std::vector parentIndexes) override; char* AutoProduceData(uint32& count, char**& indexTable, std::vector& stringPool) override; char* AutoProduceStrings(char** indexTable, uint32 indexTableSize, uint32 locale) override; void AutoProduceRecordCopies(uint32 records, char** indexTable, char* dataTable) override; @@ -206,6 +226,8 @@ public: uint32 GetRecordCopyCount() const override; uint32 GetMaxId() const override; DB2FileLoadInfo const* GetLoadInfo() const override; + DB2SectionHeader const& GetSection(uint32 section) const override; + bool IsSignedField(uint32 field) const override; private: void FillParentLookup(char* dataTable); @@ -230,13 +252,14 @@ private: DB2Header const* _header; std::unique_ptr _data; uint8* _stringTable; - std::unique_ptr _idTable; - std::unique_ptr _copyTable; + std::unique_ptr _sections; std::unique_ptr _columnMeta; std::unique_ptr[]> _palletValues; std::unique_ptr[]> _palletArrayValues; std::unique_ptr[]> _commonValues; - std::unique_ptr _parentIndexes; + std::vector _idTable; + std::vector _copyTable; + std::vector _parentIndexes; }; class DB2FileLoaderSparseImpl final : public DB2FileLoaderImpl @@ -245,12 +268,12 @@ public: DB2FileLoaderSparseImpl(char const* fileName, DB2FileLoadInfo const* loadInfo, DB2Header const* header); ~DB2FileLoaderSparseImpl(); - bool LoadTableData(DB2FileSource* source) override; - bool LoadCatalogData(DB2FileSource* source) override; - void SetAdditionalData(std::unique_ptr fields, std::unique_ptr idTable, std::unique_ptr copyTable, - std::unique_ptr columnMeta, std::unique_ptr[]> palletValues, - std::unique_ptr[]> palletArrayValues, std::unique_ptr[]> commonValues, - std::unique_ptr parentIndexes) override; + void LoadColumnData(std::unique_ptr sections, std::unique_ptr fields, std::unique_ptr columnMeta, + std::unique_ptr[]> palletValues, std::unique_ptr[]> palletArrayValues, + std::unique_ptr[]> commonValues) override; + bool LoadTableData(DB2FileSource* source, uint32 section) override; + bool LoadCatalogData(DB2FileSource* source, uint32 section) override; + void SetAdditionalData(std::vector /*idTable*/, std::vector /*copyTable*/, std::vector /*parentIndexes*/) override { } char* AutoProduceData(uint32& records, char**& indexTable, std::vector& stringPool) override; char* AutoProduceStrings(char** indexTable, uint32 indexTableSize, uint32 locale) override; void AutoProduceRecordCopies(uint32 /*records*/, char** /*indexTable*/, char* /*dataTable*/) override { } @@ -260,6 +283,8 @@ public: uint32 GetRecordCopyCount() const override; uint32 GetMaxId() const override; DB2FileLoadInfo const* GetLoadInfo() const override; + DB2SectionHeader const& GetSection(uint32 section) const override; + bool IsSignedField(uint32 field) const override; private: uint8 const* GetRawRecordData(uint32 recordNumber) const override; @@ -286,6 +311,7 @@ private: DB2Header const* _header; std::size_t _dataStart; std::unique_ptr _data; + std::unique_ptr _sections; std::unique_ptr _fields; std::unique_ptr _fieldAndArrayOffsets; std::unique_ptr _catalog; @@ -299,24 +325,46 @@ DB2FileLoaderRegularImpl::DB2FileLoaderRegularImpl(char const* fileName, DB2File { } -bool DB2FileLoaderRegularImpl::LoadTableData(DB2FileSource* source) +void DB2FileLoaderRegularImpl::LoadColumnData(std::unique_ptr sections, std::unique_ptr /*fields*/, std::unique_ptr columnMeta, + std::unique_ptr[]> palletValues, std::unique_ptr[]> palletArrayValues, + std::unique_ptr[]> commonValues) { - _data = Trinity::make_unique(_header->RecordSize * _header->RecordCount + _header->StringTableSize); - _stringTable = &_data[_header->RecordSize * _header->RecordCount]; - return source->Read(_data.get(), _header->RecordSize * _header->RecordCount + _header->StringTableSize); + _sections = std::move(sections); + _columnMeta = std::move(columnMeta); + _palletValues = std::move(palletValues); + _palletArrayValues = std::move(palletArrayValues); + _commonValues = std::move(commonValues); +} + +bool DB2FileLoaderRegularImpl::LoadTableData(DB2FileSource* source, uint32 section) +{ + if (!_data) + { + _data = Trinity::make_unique(_header->RecordSize * _header->RecordCount + _header->StringTableSize + 8); + _stringTable = &_data[_header->RecordSize * _header->RecordCount]; + } + + uint32 sectionDataStart = 0; + uint32 sectionStringTableStart = 0; + for (uint32 i = 0; i < section; ++i) + { + sectionDataStart += _header->RecordSize * _sections[i].RecordCount; + sectionStringTableStart += _sections[i].StringTableSize; + } + + if (_sections[section].RecordCount && !source->Read(&_data[sectionDataStart], _header->RecordSize * _sections[section].RecordCount)) + return false; + + if (_sections[section].StringTableSize && !source->Read(&_stringTable[sectionStringTableStart], _sections[section].StringTableSize)) + return false; + + return true; } -void DB2FileLoaderRegularImpl::SetAdditionalData(std::unique_ptr /*fields*/, std::unique_ptr idTable, std::unique_ptr copyTable, - std::unique_ptr columnMeta, std::unique_ptr[]> palletValues, - std::unique_ptr[]> palletArrayValues, std::unique_ptr[]> commonValues, - std::unique_ptr parentIndexes) +void DB2FileLoaderRegularImpl::SetAdditionalData(std::vector idTable, std::vector copyTable, std::vector parentIndexes) { _idTable = std::move(idTable); _copyTable = std::move(copyTable); - _columnMeta = std::move(columnMeta); - _palletValues = std::move(palletValues); - _palletArrayValues = std::move(palletArrayValues); - _commonValues = std::move(commonValues); _parentIndexes = std::move(parentIndexes); } @@ -339,7 +387,7 @@ char* DB2FileLoaderRegularImpl::AutoProduceData(uint32& records, char**& indexTa indexTable = new index_entry_t[maxi]; memset(indexTable, 0, maxi * sizeof(index_entry_t)); - char* dataTable = new char[(_header->RecordCount + (_header->CopyTableSize / 8)) * recordsize]; + char* dataTable = new char[(_header->RecordCount + _copyTable.size()) * recordsize]; // we store flat holders pool as single memory block std::size_t stringFields = _loadInfo->GetStringFieldCount(false); @@ -384,7 +432,7 @@ char* DB2FileLoaderRegularImpl::AutoProduceData(uint32& records, char**& indexTa for (uint32 x = 0; x < _header->FieldCount; ++x) { - for (uint32 z = 0; z < _loadInfo->Meta->ArraySizes[x]; ++z) + for (uint32 z = 0; z < _loadInfo->Meta->Fields[x].ArraySize; ++z) { switch (_loadInfo->TypesString[fieldIndex]) { @@ -423,7 +471,8 @@ char* DB2FileLoaderRegularImpl::AutoProduceData(uint32& records, char**& indexTa break; } default: - ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->TypesString[x], _fileName); + ASSERT(false, "Unknown format character '%c' found in %s meta for field %s", + _loadInfo->TypesString[fieldIndex], _fileName, _loadInfo->Fields[fieldIndex].Name); break; } ++fieldIndex; @@ -432,7 +481,7 @@ char* DB2FileLoaderRegularImpl::AutoProduceData(uint32& records, char**& indexTa for (uint32 x = _header->FieldCount; x < _loadInfo->Meta->FieldCount; ++x) { - for (uint32 z = 0; z < _loadInfo->Meta->ArraySizes[x]; ++z) + for (uint32 z = 0; z < _loadInfo->Meta->Fields[x].ArraySize; ++z) { switch (_loadInfo->TypesString[fieldIndex]) { @@ -471,7 +520,8 @@ char* DB2FileLoaderRegularImpl::AutoProduceData(uint32& records, char**& indexTa break; } default: - ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->TypesString[x], _fileName); + ASSERT(false, "Unknown format character '%c' found in %s meta for field %s", + _loadInfo->TypesString[fieldIndex], _fileName, _loadInfo->Fields[fieldIndex].Name); break; } ++fieldIndex; @@ -479,7 +529,7 @@ char* DB2FileLoaderRegularImpl::AutoProduceData(uint32& records, char**& indexTa } } - if (_parentIndexes) + if (!_parentIndexes.empty()) FillParentLookup(dataTable); return dataTable; @@ -529,7 +579,7 @@ char* DB2FileLoaderRegularImpl::AutoProduceStrings(char** indexTable, uint32 ind for (uint32 x = 0; x < _loadInfo->Meta->FieldCount; ++x) { - for (uint32 z = 0; z < _loadInfo->Meta->ArraySizes[x]; ++z) + for (uint32 z = 0; z < _loadInfo->Meta->Fields[x].ArraySize; ++z) { switch (_loadInfo->TypesString[fieldIndex]) { @@ -568,7 +618,8 @@ char* DB2FileLoaderRegularImpl::AutoProduceStrings(char** indexTable, uint32 ind break; } default: - ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->TypesString[x], _fileName); + ASSERT(false, "Unknown format character '%c' found in %s meta for field %s", + _loadInfo->TypesString[fieldIndex], _fileName, _loadInfo->Fields[fieldIndex].Name); break; } ++fieldIndex; @@ -606,7 +657,7 @@ void DB2FileLoaderRegularImpl::FillParentLookup(char* dataTable) uint32 parentId = _parentIndexes[0].Entries[i].ParentId; char* recordData = &dataTable[_parentIndexes[0].Entries[i].RecordIndex * recordSize]; - switch (_loadInfo->Meta->Types[_loadInfo->Meta->ParentIndexField]) + switch (_loadInfo->Meta->Fields[_loadInfo->Meta->ParentIndexField].Type) { case FT_SHORT: *reinterpret_cast(&recordData[parentIdOffset]) = uint16(parentId); @@ -618,7 +669,7 @@ void DB2FileLoaderRegularImpl::FillParentLookup(char* dataTable) *reinterpret_cast(&recordData[parentIdOffset]) = parentId; break; default: - ASSERT(false, "Unhandled parent id type '%c' found in %s", _loadInfo->Meta->Types[_loadInfo->Meta->ParentIndexField], _fileName); + ASSERT(false, "Unhandled parent id type '%c' found in %s", _loadInfo->Meta->Fields[_loadInfo->Meta->ParentIndexField].Type, _fileName); break; } } @@ -644,7 +695,7 @@ uint32 DB2FileLoaderRegularImpl::GetRecordCount() const uint32 DB2FileLoaderRegularImpl::GetRecordCopyCount() const { - return _header->CopyTableSize / sizeof(DB2RecordCopy); + return _copyTable.size(); } uint8 const* DB2FileLoaderRegularImpl::GetRawRecordData(uint32 recordNumber) const @@ -695,16 +746,18 @@ float DB2FileLoaderRegularImpl::RecordGetFloat(uint8 const* record, uint32 field char const* DB2FileLoaderRegularImpl::RecordGetString(uint8 const* record, uint32 field, uint32 arrayIndex) const { + uint32 fieldOffset = GetFieldOffset(field) + sizeof(uint32) * arrayIndex; uint32 stringOffset = RecordGetVarInt(record, field, arrayIndex); - ASSERT(stringOffset < _header->StringTableSize); - return reinterpret_cast(_stringTable + stringOffset); + ASSERT(stringOffset < _header->RecordSize * _header->RecordCount + _header->StringTableSize); + return reinterpret_cast(record + fieldOffset + stringOffset); } template T DB2FileLoaderRegularImpl::RecordGetVarInt(uint8 const* record, uint32 field, uint32 arrayIndex) const { ASSERT(field < _header->FieldCount); - switch (_columnMeta[field].CompressionType) + DB2ColumnCompression compressionType = _columnMeta ? _columnMeta[field].CompressionType : DB2ColumnCompression::None; + switch (compressionType) { case DB2ColumnCompression::None: { @@ -718,11 +771,6 @@ T DB2FileLoaderRegularImpl::RecordGetVarInt(uint8 const* record, uint32 field, u uint64 immediateValue = RecordGetPackedValue(record + GetFieldOffset(field), _columnMeta[field].CompressionData.immediate.BitWidth, _columnMeta[field].CompressionData.immediate.BitOffset); EndianConvert(immediateValue); - if (_columnMeta[field].CompressionData.immediate.Signed) - { - uint64 mask = UI64LIT(1) << (_columnMeta[field].CompressionData.immediate.BitWidth - 1); - immediateValue = (immediateValue ^ mask) - mask; - } T value; memcpy(&value, &immediateValue, std::min(sizeof(T), sizeof(immediateValue))); return value; @@ -761,6 +809,18 @@ T DB2FileLoaderRegularImpl::RecordGetVarInt(uint8 const* record, uint32 field, u memcpy(&value, &palletValue, std::min(sizeof(T), sizeof(palletValue))); return value; } + case DB2ColumnCompression::SignedImmediate: + { + ASSERT(arrayIndex == 0); + uint64 immediateValue = RecordGetPackedValue(record + GetFieldOffset(field), + _columnMeta[field].CompressionData.immediate.BitWidth, _columnMeta[field].CompressionData.immediate.BitOffset); + EndianConvert(immediateValue); + uint64 mask = UI64LIT(1) << (_columnMeta[field].CompressionData.immediate.BitWidth - 1); + immediateValue = (immediateValue ^ mask) - mask; + T value; + memcpy(&value, &immediateValue, std::min(sizeof(T), sizeof(immediateValue))); + return value; + } default: ASSERT(false, "Unhandled compression type %u in %s", uint32(_columnMeta[field].CompressionType), _fileName); break; @@ -778,11 +838,13 @@ uint64 DB2FileLoaderRegularImpl::RecordGetPackedValue(uint8 const* packedRecordD uint16 DB2FileLoaderRegularImpl::GetFieldOffset(uint32 field) const { ASSERT(field < _header->FieldCount); - switch (_columnMeta[field].CompressionType) + DB2ColumnCompression compressionType = _columnMeta ? _columnMeta[field].CompressionType : DB2ColumnCompression::None; + switch (compressionType) { case DB2ColumnCompression::None: return _columnMeta[field].BitOffset / 8; case DB2ColumnCompression::Immediate: + case DB2ColumnCompression::SignedImmediate: return _columnMeta[field].CompressionData.immediate.BitOffset / 8 + _header->PackedDataOffset; case DB2ColumnCompression::CommonData: return 0xFFFF; @@ -831,6 +893,40 @@ DB2FileLoadInfo const* DB2FileLoaderRegularImpl::GetLoadInfo() const return _loadInfo; } +DB2SectionHeader const& DB2FileLoaderRegularImpl::GetSection(uint32 section) const +{ + return _sections[section]; +} + +bool DB2FileLoaderRegularImpl::IsSignedField(uint32 field) const +{ + if (field >= _header->TotalFieldCount) + { + ASSERT(field == _header->TotalFieldCount); + ASSERT(int32(field) == _loadInfo->Meta->ParentIndexField); + return _loadInfo->Meta->IsSignedField(field); + } + + DB2ColumnCompression compressionType = _columnMeta ? _columnMeta[field].CompressionType : DB2ColumnCompression::None; + switch (compressionType) + { + case DB2ColumnCompression::None: + case DB2ColumnCompression::CommonData: + case DB2ColumnCompression::Pallet: + case DB2ColumnCompression::PalletArray: + return _loadInfo->Meta->IsSignedField(field); + case DB2ColumnCompression::SignedImmediate: + return field != uint32(_loadInfo->Meta->IndexField); + case DB2ColumnCompression::Immediate: + return false; + default: + ASSERT(false, "Unhandled compression type %u in %s", uint32(_columnMeta[field].CompressionType), _fileName); + break; + } + + return false; +} + DB2FileLoaderSparseImpl::DB2FileLoaderSparseImpl(char const* fileName, DB2FileLoadInfo const* loadInfo, DB2Header const* header) : _fileName(fileName), _loadInfo(loadInfo), @@ -844,25 +940,31 @@ DB2FileLoaderSparseImpl::~DB2FileLoaderSparseImpl() { } -bool DB2FileLoaderSparseImpl::LoadTableData(DB2FileSource* source) +void DB2FileLoaderSparseImpl::LoadColumnData(std::unique_ptr sections, std::unique_ptr fields, std::unique_ptr /*columnMeta*/, + std::unique_ptr[]> /*palletValues*/, std::unique_ptr[]> /*palletArrayValues*/, + std::unique_ptr[]> /*commonValues*/) { - _dataStart = source->GetPosition(); - _data = Trinity::make_unique(_header->CatalogDataOffset - _dataStart); - return source->Read(_data.get(), _header->CatalogDataOffset - _dataStart); + _sections = std::move(sections); + _fields = std::move(fields); } -bool DB2FileLoaderSparseImpl::LoadCatalogData(DB2FileSource* source) +bool DB2FileLoaderSparseImpl::LoadTableData(DB2FileSource* source, uint32 section) { - _catalog = Trinity::make_unique(_header->MaxId - _header->MinId + 1); - return source->Read(_catalog.get(), sizeof(DB2CatalogEntry) * (_header->MaxId - _header->MinId + 1)); + if (section != 0) + return false; + + _dataStart = source->GetPosition(); + _data = Trinity::make_unique(_sections[0].CatalogDataOffset - _dataStart); + return source->Read(_data.get(), _sections[0].CatalogDataOffset - _dataStart); } -void DB2FileLoaderSparseImpl::SetAdditionalData(std::unique_ptr fields, std::unique_ptr /*idTable*/, std::unique_ptr /*copyTable*/, - std::unique_ptr /*columnMeta*/, std::unique_ptr[]> /*palletValues*/, - std::unique_ptr[]> /*palletArrayValues*/, std::unique_ptr[]> /*commonValues*/, - std::unique_ptr /*parentIndexes*/) +bool DB2FileLoaderSparseImpl::LoadCatalogData(DB2FileSource* source, uint32 section) { - _fields = std::move(fields); + if (section != 0) + return false; + + _catalog = Trinity::make_unique(_header->MaxId - _header->MinId + 1); + return source->Read(_catalog.get(), sizeof(DB2CatalogEntry) * (_header->MaxId - _header->MinId + 1)); } char* DB2FileLoaderSparseImpl::AutoProduceData(uint32& maxId, char**& indexTable, std::vector& stringPool) @@ -935,7 +1037,7 @@ char* DB2FileLoaderSparseImpl::AutoProduceData(uint32& maxId, char**& indexTable uint32 stringFieldOffset = 0; for (uint32 x = 0; x < _header->FieldCount; ++x) { - for (uint32 z = 0; z < _loadInfo->Meta->ArraySizes[x]; ++z) + for (uint32 z = 0; z < _loadInfo->Meta->Fields[x].ArraySize; ++z) { switch (_loadInfo->TypesString[fieldIndex]) { @@ -984,7 +1086,8 @@ char* DB2FileLoaderSparseImpl::AutoProduceData(uint32& maxId, char**& indexTable break; } default: - ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->TypesString[x], _fileName); + ASSERT(false, "Unknown format character '%c' found in %s meta for field %s", + _loadInfo->TypesString[fieldIndex], _fileName, _loadInfo->Fields[fieldIndex].Name); break; } ++fieldIndex; @@ -1027,8 +1130,8 @@ char* DB2FileLoaderSparseImpl::AutoProduceStrings(char** indexTable, uint32 inde uint32 recordsize = _loadInfo->Meta->GetRecordSize(); std::size_t stringFields = _loadInfo->GetStringFieldCount(true); - char* stringTable = new char[_header->CatalogDataOffset - _dataStart - records * ((recordsize - (!_loadInfo->Meta->HasIndexFieldInData() ? 4 : 0)) - stringFields * sizeof(char*))]; - memset(stringTable, 0, _header->CatalogDataOffset - _dataStart - records * ((recordsize - (!_loadInfo->Meta->HasIndexFieldInData() ? 4 : 0)) - stringFields * sizeof(char*))); + char* stringTable = new char[_sections[0].CatalogDataOffset - _dataStart - records * ((recordsize - (!_loadInfo->Meta->HasIndexFieldInData() ? 4 : 0)) - stringFields * sizeof(char*))]; + memset(stringTable, 0, _sections[0].CatalogDataOffset - _dataStart - records * ((recordsize - (!_loadInfo->Meta->HasIndexFieldInData() ? 4 : 0)) - stringFields * sizeof(char*))); char* stringPtr = stringTable; for (uint32 y = 0; y < offsetCount; y++) @@ -1053,7 +1156,7 @@ char* DB2FileLoaderSparseImpl::AutoProduceStrings(char** indexTable, uint32 inde for (uint32 x = 0; x < _header->FieldCount; ++x) { - for (uint32 z = 0; z < _loadInfo->Meta->ArraySizes[x]; ++z) + for (uint32 z = 0; z < _loadInfo->Meta->Fields[x].ArraySize; ++z) { switch (_loadInfo->TypesString[fieldIndex]) { @@ -1085,7 +1188,8 @@ char* DB2FileLoaderSparseImpl::AutoProduceStrings(char** indexTable, uint32 inde offset += sizeof(char*); break; default: - ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->TypesString[x], _fileName); + ASSERT(false, "Unknown format character '%c' found in %s meta for field %s", + _loadInfo->TypesString[fieldIndex], _fileName, _loadInfo->Fields[fieldIndex].Name); break; } ++fieldIndex; @@ -1230,10 +1334,10 @@ void DB2FileLoaderSparseImpl::CalculateAndStoreFieldOffsets(uint8 const* rawReco for (uint32 field = 0; field < _loadInfo->Meta->FieldCount; ++field) { _fieldAndArrayOffsets[field] = combinedField; - for (uint32 arr = 0; arr < _loadInfo->Meta->ArraySizes[field]; ++arr) + for (uint32 arr = 0; arr < _loadInfo->Meta->Fields[field].ArraySize; ++arr) { _fieldAndArrayOffsets[combinedField] = offset; - switch (_loadInfo->Meta->Types[field]) + switch (_loadInfo->Meta->Fields[field].Type) { case FT_BYTE: case FT_SHORT: @@ -1245,10 +1349,11 @@ void DB2FileLoaderSparseImpl::CalculateAndStoreFieldOffsets(uint8 const* rawReco offset += sizeof(float); break; case FT_STRING: + case FT_STRING_NOT_LOCALIZED: offset += strlen(reinterpret_cast(rawRecord) + offset) + 1; break; default: - ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->Meta->Types[field], _fileName); + ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->Meta->Fields[field].Type, _fileName); break; } ++combinedField; @@ -1266,6 +1371,17 @@ DB2FileLoadInfo const* DB2FileLoaderSparseImpl::GetLoadInfo() const return _loadInfo; } +DB2SectionHeader const& DB2FileLoaderSparseImpl::GetSection(uint32 section) const +{ + return _sections[section]; +} + +bool DB2FileLoaderSparseImpl::IsSignedField(uint32 field) const +{ + ASSERT(field < _header->FieldCount); + return _loadInfo->Meta->IsSignedField(field); +} + DB2Record::DB2Record(DB2FileLoaderImpl const& db2, uint32 recordIndex, std::size_t* fieldOffsets) : _db2(db2), _recordIndex(recordIndex), _recordData(db2.GetRawRecordData(recordIndex)), _fieldOffsets(fieldOffsets) { @@ -1402,86 +1518,66 @@ bool DB2FileLoader::Load(DB2FileSource* source, DB2FileLoadInfo const* loadInfo) EndianConvert(_header.MinId); EndianConvert(_header.MaxId); EndianConvert(_header.Locale); - EndianConvert(_header.CopyTableSize); EndianConvert(_header.Flags); EndianConvert(_header.IndexField); EndianConvert(_header.TotalFieldCount); EndianConvert(_header.PackedDataOffset); EndianConvert(_header.ParentLookupCount); - EndianConvert(_header.CatalogDataOffset); - EndianConvert(_header.IdTableSize); EndianConvert(_header.ColumnMetaSize); EndianConvert(_header.CommonDataSize); EndianConvert(_header.PalletDataSize); - EndianConvert(_header.ParentLookupDataSize); + EndianConvert(_header.SectionCount); - if (_header.Signature != 0x31434457) //'WDC1' + if (_header.Signature != 0x32434457) //'WDC2' return false; if (_header.LayoutHash != loadInfo->Meta->LayoutHash) return false; + if (_header.ParentLookupCount > 1) + return false; + + if (_header.TotalFieldCount + (loadInfo->Meta->ParentIndexField >= int32(_header.TotalFieldCount) ? 1 : 0) != loadInfo->Meta->FieldCount) + return false; + + if (_header.ParentLookupCount && loadInfo->Meta->ParentIndexField == -1) + return false; + + std::unique_ptr sections = Trinity::make_unique(_header.SectionCount); + if (_header.SectionCount && !source->Read(sections.get(), sizeof(DB2SectionHeader) * _header.SectionCount)) + return false; + + uint32 totalCopyTableSize = 0; + uint32 totalParentLookupDataSize = 0; + for (uint32 i = 0; i < _header.SectionCount; ++i) + { + totalCopyTableSize += sections[i].CopyTableSize; + totalParentLookupDataSize += sections[i].ParentLookupDataSize; + } + if (!(_header.Flags & 0x1)) { std::size_t expectedFileSize = sizeof(DB2Header) + + sizeof(DB2SectionHeader) * _header.SectionCount + sizeof(DB2FieldEntry) * _header.FieldCount + _header.RecordCount * _header.RecordSize + _header.StringTableSize + - _header.IdTableSize + - _header.CopyTableSize + + (loadInfo->Meta->IndexField == -1 ? _header.RecordCount * sizeof(uint32) : 0) + + totalCopyTableSize + _header.ColumnMetaSize + _header.PalletDataSize + _header.CommonDataSize + - _header.ParentLookupDataSize; + totalParentLookupDataSize; if (source->GetFileSize() != expectedFileSize) return false; } - if (_header.ParentLookupCount > 1) - return false; - - if (_header.TotalFieldCount + (loadInfo->Meta->ParentIndexField >= int32(_header.TotalFieldCount) ? 1 : 0) != loadInfo->Meta->FieldCount) - return false; - - if (_header.ParentLookupCount && loadInfo->Meta->ParentIndexField == -1) - return false; - - if (!(_header.Flags & 0x1)) - _impl = new DB2FileLoaderRegularImpl(source->GetFileName(), loadInfo, &_header); - else - _impl = new DB2FileLoaderSparseImpl(source->GetFileName(), loadInfo, &_header); - std::unique_ptr fieldData = Trinity::make_unique(_header.FieldCount); if (!source->Read(fieldData.get(), sizeof(DB2FieldEntry) * _header.FieldCount)) return false; - if (!_impl->LoadTableData(source)) - return false; - - if (!_impl->LoadCatalogData(source)) - return false; - - ASSERT(!loadInfo->Meta->HasIndexFieldInData() || _header.IdTableSize == 0); - ASSERT(loadInfo->Meta->HasIndexFieldInData() || _header.IdTableSize == 4 * _header.RecordCount); - - std::unique_ptr idTable; - if (!loadInfo->Meta->HasIndexFieldInData() && _header.IdTableSize) - { - idTable = Trinity::make_unique(_header.RecordCount); - if (!source->Read(idTable.get(), _header.IdTableSize)) - return false; - } - - std::unique_ptr copyTable; - if (_header.CopyTableSize) - { - copyTable = Trinity::make_unique(_header.CopyTableSize / sizeof(DB2RecordCopy)); - if (!source->Read(copyTable.get(), _header.CopyTableSize)) - return false; - } - std::unique_ptr columnMeta; std::unique_ptr[]> palletValues; std::unique_ptr[]> palletArrayValues; @@ -1494,7 +1590,8 @@ bool DB2FileLoader::Load(DB2FileSource* source, DB2FileLoadInfo const* loadInfo) ASSERT(!loadInfo->Meta->HasIndexFieldInData() || columnMeta[loadInfo->Meta->IndexField].CompressionType == DB2ColumnCompression::None || - columnMeta[loadInfo->Meta->IndexField].CompressionType == DB2ColumnCompression::Immediate); + columnMeta[loadInfo->Meta->IndexField].CompressionType == DB2ColumnCompression::Immediate || + columnMeta[loadInfo->Meta->IndexField].CompressionType == DB2ColumnCompression::SignedImmediate); palletValues = Trinity::make_unique[]>(_header.TotalFieldCount); for (uint32 i = 0; i < _header.TotalFieldCount; ++i) @@ -1543,26 +1640,81 @@ bool DB2FileLoader::Load(DB2FileSource* source, DB2FileLoadInfo const* loadInfo) } } - std::unique_ptr parentIndexes; - if (_header.ParentLookupCount) + if (!(_header.Flags & 0x1)) + _impl = new DB2FileLoaderRegularImpl(source->GetFileName(), loadInfo, &_header); + else + _impl = new DB2FileLoaderSparseImpl(source->GetFileName(), loadInfo, &_header); + + _impl->LoadColumnData(std::move(sections), std::move(fieldData), std::move(columnMeta), std::move(palletValues), std::move(palletArrayValues), std::move(commonValues)); + + std::vector idTable; + std::vector copyTable; + std::vector parentIndexes; + if (!loadInfo->Meta->HasIndexFieldInData() && _header.RecordCount) + idTable.reserve(_header.RecordCount); + + for (uint32 i = 0; i < _header.SectionCount; ++i) { - parentIndexes = Trinity::make_unique(_header.ParentLookupCount); - for (uint32 i = 0; i < _header.ParentLookupCount; ++i) + DB2SectionHeader const& section = _impl->GetSection(i); + + source->SetPosition(section.FileOffset); + + if (!_impl->LoadTableData(source, i)) + return false; + + if (!_impl->LoadCatalogData(source, i)) + return false; + + ASSERT(!loadInfo->Meta->HasIndexFieldInData() || section.IdTableSize == 0); + ASSERT(loadInfo->Meta->HasIndexFieldInData() || section.IdTableSize == 4 * section.RecordCount); + + if (!loadInfo->Meta->HasIndexFieldInData() && section.IdTableSize) { - if (!source->Read(&parentIndexes[i].Info, sizeof(DB2IndexDataInfo))) + std::size_t idTableSize = idTable.size(); + idTable.resize(idTableSize + section.IdTableSize); + if (!source->Read(&idTable[idTableSize], section.IdTableSize)) return false; + } - if (!parentIndexes[i].Info.NumEntries) - continue; - - parentIndexes[i].Entries = Trinity::make_unique(parentIndexes[i].Info.NumEntries); - if (!source->Read(parentIndexes[i].Entries.get(), sizeof(DB2IndexEntry) * parentIndexes[i].Info.NumEntries)) + if (section.CopyTableSize) + { + std::size_t copyTableSize = copyTable.size(); + copyTable.resize(copyTableSize + section.CopyTableSize / sizeof(DB2RecordCopy)); + if (!source->Read(©Table[copyTableSize], section.CopyTableSize)) return false; } + + if (_header.ParentLookupCount) + { + parentIndexes.resize(_header.ParentLookupCount); + for (uint32 i = 0; i < _header.ParentLookupCount; ++i) + { + if (!source->Read(&parentIndexes[i].Info, sizeof(DB2IndexDataInfo))) + return false; + + if (!parentIndexes[i].Info.NumEntries) + continue; + + parentIndexes[i].Entries = Trinity::make_unique(parentIndexes[i].Info.NumEntries); + if (!source->Read(parentIndexes[i].Entries.get(), sizeof(DB2IndexEntry) * parentIndexes[i].Info.NumEntries)) + return false; + } + } } - _impl->SetAdditionalData(std::move(fieldData), std::move(idTable), std::move(copyTable), std::move(columnMeta), - std::move(palletValues), std::move(palletArrayValues), std::move(commonValues), std::move(parentIndexes)); + _impl->SetAdditionalData(std::move(idTable), std::move(copyTable), std::move(parentIndexes)); + + uint32 fieldIndex = 0; + if (!loadInfo->Meta->HasIndexFieldInData()) + { + ASSERT(!loadInfo->Fields[0].IsSigned, "ID must be unsigned"); + ++fieldIndex; + } + for (uint32 f = 0; f < loadInfo->FieldCount; ++f) + { + ASSERT(loadInfo->Fields[fieldIndex].IsSigned == _impl->IsSignedField(f), "Mismatched field signedness for field %u (%s)", f, loadInfo->Fields[fieldIndex].Name); + fieldIndex += loadInfo->Meta->Fields[f].ArraySize; + } return true; } -- cgit v1.2.3 From ccbe4404f1806f047c009d14ed1dea9806dd5736 Mon Sep 17 00:00:00 2001 From: Shauren Date: Tue, 11 Sep 2018 21:48:16 +0200 Subject: Tools: Updated db2 structures --- src/common/DataStores/DB2FileLoader.cpp | 5 +- src/common/DataStores/DB2FileLoader.h | 2 +- src/common/DataStores/DB2FileSystemSource.cpp | 4 +- src/common/DataStores/DB2FileSystemSource.h | 2 +- src/tools/extractor_common/CascHandles.cpp | 7 + src/tools/extractor_common/CascHandles.h | 1 + src/tools/extractor_common/DB2CascFileSource.cpp | 5 + src/tools/extractor_common/DB2CascFileSource.h | 1 + src/tools/extractor_common/ExtractorDB2LoadInfo.h | 183 +++++++++++++++------- 9 files changed, 148 insertions(+), 62 deletions(-) (limited to 'src/common/DataStores/DB2FileLoader.cpp') diff --git a/src/common/DataStores/DB2FileLoader.cpp b/src/common/DataStores/DB2FileLoader.cpp index d429195445b..d6138fa492e 100644 --- a/src/common/DataStores/DB2FileLoader.cpp +++ b/src/common/DataStores/DB2FileLoader.cpp @@ -1657,7 +1657,8 @@ bool DB2FileLoader::Load(DB2FileSource* source, DB2FileLoadInfo const* loadInfo) { DB2SectionHeader const& section = _impl->GetSection(i); - source->SetPosition(section.FileOffset); + if (!source->SetPosition(section.FileOffset)) + return false; if (!_impl->LoadTableData(source, i)) return false; @@ -1710,7 +1711,7 @@ bool DB2FileLoader::Load(DB2FileSource* source, DB2FileLoadInfo const* loadInfo) ASSERT(!loadInfo->Fields[0].IsSigned, "ID must be unsigned"); ++fieldIndex; } - for (uint32 f = 0; f < loadInfo->FieldCount; ++f) + for (uint32 f = 0; f < loadInfo->Meta->FieldCount; ++f) { ASSERT(loadInfo->Fields[fieldIndex].IsSigned == _impl->IsSignedField(f), "Mismatched field signedness for field %u (%s)", f, loadInfo->Fields[fieldIndex].Name); fieldIndex += loadInfo->Meta->Fields[f].ArraySize; diff --git a/src/common/DataStores/DB2FileLoader.h b/src/common/DataStores/DB2FileLoader.h index b1d42a2c954..7be5dc1aa39 100644 --- a/src/common/DataStores/DB2FileLoader.h +++ b/src/common/DataStores/DB2FileLoader.h @@ -88,7 +88,7 @@ struct TC_COMMON_API DB2FileSource // Returns current read position in file virtual std::size_t GetPosition() const = 0; - virtual void SetPosition(std::size_t position) = 0; + virtual bool SetPosition(std::size_t position) = 0; virtual std::size_t GetFileSize() const = 0; diff --git a/src/common/DataStores/DB2FileSystemSource.cpp b/src/common/DataStores/DB2FileSystemSource.cpp index b8a4e4258b6..9b929a8f5c9 100644 --- a/src/common/DataStores/DB2FileSystemSource.cpp +++ b/src/common/DataStores/DB2FileSystemSource.cpp @@ -45,9 +45,9 @@ std::size_t DB2FileSystemSource::GetPosition() const return ftell(_file); } -void DB2FileSystemSource::SetPosition(std::size_t position) +bool DB2FileSystemSource::SetPosition(std::size_t position) { - fseek(_file, position, SEEK_SET); + return fseek(_file, position, SEEK_SET) == 0; } std::size_t DB2FileSystemSource::GetFileSize() const diff --git a/src/common/DataStores/DB2FileSystemSource.h b/src/common/DataStores/DB2FileSystemSource.h index 94e4d3287b5..107aa2705fb 100644 --- a/src/common/DataStores/DB2FileSystemSource.h +++ b/src/common/DataStores/DB2FileSystemSource.h @@ -28,7 +28,7 @@ struct TC_COMMON_API DB2FileSystemSource : public DB2FileSource bool IsOpen() const override; bool Read(void* buffer, std::size_t numBytes) override; std::size_t GetPosition() const override; - void SetPosition(std::size_t position) override; + bool SetPosition(std::size_t position) override; std::size_t GetFileSize() const override; char const* GetFileName() const override; diff --git a/src/tools/extractor_common/CascHandles.cpp b/src/tools/extractor_common/CascHandles.cpp index 575f9fe3405..930d8eea420 100644 --- a/src/tools/extractor_common/CascHandles.cpp +++ b/src/tools/extractor_common/CascHandles.cpp @@ -127,6 +127,13 @@ DWORD CASC::GetFilePointer(FileHandle const& file) return ::CascSetFilePointer(file.get(), 0, nullptr, FILE_CURRENT); } +bool CASC::SetFilePointer(FileHandle const& file, LONGLONG position) +{ + LONG parts[2]; + memcpy(parts, &position, sizeof(parts)); + return ::CascSetFilePointer(file.get(), parts[0], &parts[1], FILE_BEGIN) != CASC_INVALID_POS; +} + bool CASC::ReadFile(FileHandle const& file, void* buffer, DWORD bytes, PDWORD bytesRead) { return ::CascReadFile(file.get(), buffer, bytes, bytesRead); diff --git a/src/tools/extractor_common/CascHandles.h b/src/tools/extractor_common/CascHandles.h index 0a8a1dd359c..a412976ce10 100644 --- a/src/tools/extractor_common/CascHandles.h +++ b/src/tools/extractor_common/CascHandles.h @@ -55,6 +55,7 @@ namespace CASC FileHandle OpenFile(StorageHandle const& storage, char const* fileName, DWORD localeMask, bool printErrors = false); DWORD GetFileSize(FileHandle const& file, PDWORD fileSizeHigh); DWORD GetFilePointer(FileHandle const& file); + bool SetFilePointer(FileHandle const& file, LONGLONG position); bool ReadFile(FileHandle const& file, void* buffer, DWORD bytes, PDWORD bytesRead); } diff --git a/src/tools/extractor_common/DB2CascFileSource.cpp b/src/tools/extractor_common/DB2CascFileSource.cpp index 5452c3e1518..b09f7f0b90e 100644 --- a/src/tools/extractor_common/DB2CascFileSource.cpp +++ b/src/tools/extractor_common/DB2CascFileSource.cpp @@ -40,6 +40,11 @@ std::size_t DB2CascFileSource::GetPosition() const return CASC::GetFilePointer(_fileHandle); } +bool DB2CascFileSource::SetPosition(std::size_t position) +{ + return CASC::SetFilePointer(_fileHandle, position); +} + std::size_t DB2CascFileSource::GetFileSize() const { DWORD sizeLow = 0; diff --git a/src/tools/extractor_common/DB2CascFileSource.h b/src/tools/extractor_common/DB2CascFileSource.h index 9708fcdcfe3..84376b6791d 100644 --- a/src/tools/extractor_common/DB2CascFileSource.h +++ b/src/tools/extractor_common/DB2CascFileSource.h @@ -28,6 +28,7 @@ struct DB2CascFileSource : public DB2FileSource bool IsOpen() const override; bool Read(void* buffer, std::size_t numBytes) override; std::size_t GetPosition() const override; + bool SetPosition(std::size_t position) override; std::size_t GetFileSize() const override; char const* GetFileName() const override; diff --git a/src/tools/extractor_common/ExtractorDB2LoadInfo.h b/src/tools/extractor_common/ExtractorDB2LoadInfo.h index fccfc924205..26c1fb860d7 100644 --- a/src/tools/extractor_common/ExtractorDB2LoadInfo.h +++ b/src/tools/extractor_common/ExtractorDB2LoadInfo.h @@ -26,20 +26,25 @@ struct CinematicCameraLoadInfo { static DB2FileLoadInfo const* Instance() { - static DB2FieldMeta const fields[] = + static DB2FieldMeta const loadedFields[] = { { false, FT_INT, "ID" }, - { false, FT_INT, "SoundID" }, { false, FT_FLOAT, "OriginX" }, { false, FT_FLOAT, "OriginY" }, { false, FT_FLOAT, "OriginZ" }, + { false, FT_INT, "SoundID" }, { false, FT_FLOAT, "OriginFacing" }, { false, FT_INT, "FileDataID" }, }; - static char const* types = "iffi"; - static uint8 const arraySizes[4] = { 1, 3, 1, 1 }; - static DB2Meta const meta(-1, 4, 0x0062B0F4, types, arraySizes, -1); - static DB2FileLoadInfo const loadInfo(&fields[0], std::extent::value, &meta); + static DB2MetaField const fields[4] = + { + { FT_FLOAT, 3, true }, + { FT_INT, 1, false }, + { FT_FLOAT, 1, true }, + { FT_INT, 1, false }, + }; + static DB2Meta meta(-1, 4, 0x20C5E540, fields, -1); + static DB2FileLoadInfo const loadInfo(&loadedFields[0], std::extent::value, &meta); return &loadInfo; } }; @@ -48,24 +53,30 @@ struct GameobjectDisplayInfoLoadInfo { static DB2FileLoadInfo const* Instance() { - static DB2FieldMeta const fields[] = + static DB2FieldMeta const loadedFields[] = { { false, FT_INT, "ID" }, - { true, FT_INT, "FileDataID" }, { false, FT_FLOAT, "GeoBoxMinX" }, { false, FT_FLOAT, "GeoBoxMinY" }, { false, FT_FLOAT, "GeoBoxMinZ" }, { false, FT_FLOAT, "GeoBoxMaxX" }, { false, FT_FLOAT, "GeoBoxMaxY" }, { false, FT_FLOAT, "GeoBoxMaxZ" }, + { true, FT_INT, "FileDataID" }, + { true, FT_SHORT, "ObjectEffectPackageID" }, { false, FT_FLOAT, "OverrideLootEffectScale" }, { false, FT_FLOAT, "OverrideNameScale" }, - { true, FT_SHORT, "ObjectEffectPackageID" }, }; - static char const* types = "ifffh"; - static uint8 const arraySizes[5] = { 1, 6, 1, 1, 1 }; - static DB2Meta const meta(-1, 5, 0x9F2098D1, types, arraySizes, -1); - static DB2FileLoadInfo const loadInfo(&fields[0], std::extent::value, &meta); + static DB2MetaField const fields[5] = + { + { FT_FLOAT, 6, true }, + { FT_INT, 1, true }, + { FT_SHORT, 1, true }, + { FT_FLOAT, 1, true }, + { FT_FLOAT, 1, true }, + }; + static DB2Meta meta(-1, 5, 0x7A816799, fields, -1); + static DB2FileLoadInfo const loadInfo(&loadedFields[0], std::extent::value, &meta); return &loadInfo; } }; @@ -74,16 +85,19 @@ struct LiquidMaterialLoadInfo { static DB2FileLoadInfo const* Instance() { - static DB2FieldMeta const fields[] = + static DB2FieldMeta const loadedFields[] = { { false, FT_INT, "ID" }, - { true, FT_BYTE, "LVF" }, { true, FT_BYTE, "Flags" }, + { true, FT_BYTE, "LVF" }, }; - static char const* types = "bb"; - static uint8 const arraySizes[2] = { 1, 1 }; - static DB2Meta meta(-1, 2, 0x62BE0340, types, arraySizes, -1); - static DB2FileLoadInfo const loadInfo(&fields[0], std::extent::value, &meta); + static DB2MetaField const fields[2] = + { + { FT_BYTE, 1, true }, + { FT_BYTE, 1, true }, + }; + static DB2Meta meta(-1, 2, 0x6A7287A2, fields, -1); + static DB2FileLoadInfo const loadInfo(&loadedFields[0], std::extent::value, &meta); return &loadInfo; } }; @@ -92,7 +106,7 @@ struct LiquidObjectLoadInfo { static DB2FileLoadInfo const* Instance() { - static DB2FieldMeta const fields[] = + static DB2FieldMeta const loadedFields[] = { { false, FT_INT, "ID" }, { false, FT_FLOAT, "FlowDirection" }, @@ -101,10 +115,16 @@ struct LiquidObjectLoadInfo { false, FT_BYTE, "Fishable" }, { false, FT_BYTE, "Reflection" }, }; - static char const* types = "ffhbb"; - static uint8 const arraySizes[5] = { 1, 1, 1, 1, 1 }; - static DB2Meta meta(-1, 5, 0xACC168A6, types, arraySizes, -1); - static DB2FileLoadInfo const loadInfo(&fields[0], std::extent::value, &meta); + static DB2MetaField const fields[5] = + { + { FT_FLOAT, 1, true }, + { FT_FLOAT, 1, true }, + { FT_SHORT, 1, true }, + { FT_BYTE, 1, false }, + { FT_BYTE, 1, false }, + }; + static DB2Meta meta(-1, 5, 0x7AF380AA, fields, -1); + static DB2FileLoadInfo const loadInfo(&loadedFields[0], std::extent::value, &meta); return &loadInfo; } }; @@ -113,7 +133,7 @@ struct LiquidTypeLoadInfo { static DB2FileLoadInfo const* Instance() { - static DB2FieldMeta const fields[] = + static DB2FieldMeta const loadedFields[] = { { false, FT_INT, "ID" }, { false, FT_STRING_NOT_LOCALIZED, "Name" }, @@ -123,12 +143,26 @@ struct LiquidTypeLoadInfo { false, FT_STRING_NOT_LOCALIZED, "Texture4" }, { false, FT_STRING_NOT_LOCALIZED, "Texture5" }, { false, FT_STRING_NOT_LOCALIZED, "Texture6" }, + { false, FT_SHORT, "Flags" }, + { false, FT_BYTE, "SoundBank" }, + { false, FT_INT, "SoundID" }, { false, FT_INT, "SpellID" }, { false, FT_FLOAT, "MaxDarkenDepth" }, { false, FT_FLOAT, "FogDarkenIntensity" }, { false, FT_FLOAT, "AmbDarkenIntensity" }, { false, FT_FLOAT, "DirDarkenIntensity" }, + { false, FT_SHORT, "LightID" }, { false, FT_FLOAT, "ParticleScale" }, + { false, FT_BYTE, "ParticleMovement" }, + { false, FT_BYTE, "ParticleTexSlots" }, + { false, FT_BYTE, "MaterialID" }, + { true, FT_INT, "MinimapStaticCol" }, + { false, FT_BYTE, "FrameCountTexture1" }, + { false, FT_BYTE, "FrameCountTexture2" }, + { false, FT_BYTE, "FrameCountTexture3" }, + { false, FT_BYTE, "FrameCountTexture4" }, + { false, FT_BYTE, "FrameCountTexture5" }, + { false, FT_BYTE, "FrameCountTexture6" }, { true, FT_INT, "Color1" }, { true, FT_INT, "Color2" }, { false, FT_FLOAT, "Float1" }, @@ -153,24 +187,37 @@ struct LiquidTypeLoadInfo { false, FT_INT, "Int2" }, { false, FT_INT, "Int3" }, { false, FT_INT, "Int4" }, - { false, FT_SHORT, "Flags" }, - { false, FT_SHORT, "LightID" }, - { false, FT_BYTE, "SoundBank" }, - { false, FT_BYTE, "ParticleMovement" }, - { false, FT_BYTE, "ParticleTexSlots" }, - { false, FT_BYTE, "MaterialID" }, - { false, FT_BYTE, "FrameCountTexture1" }, - { false, FT_BYTE, "FrameCountTexture2" }, - { false, FT_BYTE, "FrameCountTexture3" }, - { false, FT_BYTE, "FrameCountTexture4" }, - { false, FT_BYTE, "FrameCountTexture5" }, - { false, FT_BYTE, "FrameCountTexture6" }, - { false, FT_INT, "SoundID" }, + { false, FT_FLOAT, "Coefficient1" }, + { false, FT_FLOAT, "Coefficient2" }, + { false, FT_FLOAT, "Coefficient3" }, + { false, FT_FLOAT, "Coefficient4" }, + }; + static DB2MetaField const fields[21] = + { + { FT_STRING_NOT_LOCALIZED, 1, true }, + { FT_STRING_NOT_LOCALIZED, 6, true }, + { FT_SHORT, 1, false }, + { FT_BYTE, 1, false }, + { FT_INT, 1, false }, + { FT_INT, 1, false }, + { FT_FLOAT, 1, true }, + { FT_FLOAT, 1, true }, + { FT_FLOAT, 1, true }, + { FT_FLOAT, 1, true }, + { FT_SHORT, 1, false }, + { FT_FLOAT, 1, true }, + { FT_BYTE, 1, false }, + { FT_BYTE, 1, false }, + { FT_BYTE, 1, false }, + { FT_INT, 1, true }, + { FT_BYTE, 6, false }, + { FT_INT, 2, true }, + { FT_FLOAT, 18, true }, + { FT_INT, 4, false }, + { FT_FLOAT, 4, true }, }; - static char const* types = "ssifffffifihhbbbbbi"; - static uint8 const arraySizes[19] = { 1, 6, 1, 1, 1, 1, 1, 1, 2, 18, 4, 1, 1, 1, 1, 1, 1, 6, 1 }; - static DB2Meta const meta(-1, 19, 0x3313BBF3, types, arraySizes, -1); - static DB2FileLoadInfo const loadInfo(&fields[0], std::extent::value, &meta); + static DB2Meta meta(-1, 21, 0x29F8C65E, fields, -1); + static DB2FileLoadInfo const loadInfo(&loadedFields[0], std::extent::value, &meta); return &loadInfo; } }; @@ -179,7 +226,7 @@ struct MapLoadInfo { static DB2FileLoadInfo const* Instance() { - static DB2FieldMeta const fields[] = + static DB2FieldMeta const loadedFields[] = { { false, FT_INT, "ID" }, { false, FT_STRING_NOT_LOCALIZED, "Directory" }, @@ -188,28 +235,52 @@ struct MapLoadInfo { false, FT_STRING, "MapDescription1" }, { false, FT_STRING, "PvpShortDescription" }, { false, FT_STRING, "PvpLongDescription" }, - { true, FT_INT, "Flags1" }, - { true, FT_INT, "Flags2" }, - { false, FT_FLOAT, "MinimapIconScale" }, { false, FT_FLOAT, "CorpseX" }, { false, FT_FLOAT, "CorpseY" }, + { false, FT_BYTE, "MapType" }, + { true, FT_BYTE, "InstanceType" }, + { false, FT_BYTE, "ExpansionID" }, { false, FT_SHORT, "AreaTableID" }, { true, FT_SHORT, "LoadingScreenID" }, - { true, FT_SHORT, "CorpseMapID" }, { true, FT_SHORT, "TimeOfDayOverride" }, { true, FT_SHORT, "ParentMapID" }, { true, FT_SHORT, "CosmeticParentMapID" }, - { true, FT_SHORT, "WindSettingsID" }, - { false, FT_BYTE, "InstanceType" }, - { false, FT_BYTE, "MapType" }, - { false, FT_BYTE, "ExpansionID" }, - { false, FT_BYTE, "MaxPlayers" }, { false, FT_BYTE, "TimeOffset" }, + { false, FT_FLOAT, "MinimapIconScale" }, + { true, FT_SHORT, "CorpseMapID" }, + { false, FT_BYTE, "MaxPlayers" }, + { true, FT_SHORT, "WindSettingsID" }, + { true, FT_INT, "ZmpFileDataID" }, + { true, FT_INT, "Flags1" }, + { true, FT_INT, "Flags2" }, + }; + static DB2MetaField const fields[22] = + { + { FT_STRING_NOT_LOCALIZED, 1, true }, + { FT_STRING, 1, true }, + { FT_STRING, 1, true }, + { FT_STRING, 1, true }, + { FT_STRING, 1, true }, + { FT_STRING, 1, true }, + { FT_FLOAT, 2, true }, + { FT_BYTE, 1, false }, + { FT_BYTE, 1, true }, + { FT_BYTE, 1, false }, + { FT_SHORT, 1, false }, + { FT_SHORT, 1, true }, + { FT_SHORT, 1, true }, + { FT_SHORT, 1, true }, + { FT_SHORT, 1, true }, + { FT_BYTE, 1, false }, + { FT_FLOAT, 1, true }, + { FT_SHORT, 1, true }, + { FT_BYTE, 1, false }, + { FT_SHORT, 1, true }, + { FT_INT, 1, true }, + { FT_INT, 2, true }, }; - static char const* types = "ssssssiffhhhhhhhbbbbb"; - static uint8 const arraySizes[21] = { 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; - static DB2Meta const meta(-1, 21, 0xF568DF12, types, arraySizes, -1); - static DB2FileLoadInfo const loadInfo(&fields[0], std::extent::value, &meta); + static DB2Meta meta(-1, 22, 0x503A3E58, fields, -1); + static DB2FileLoadInfo const loadInfo(&loadedFields[0], std::extent::value, &meta); return &loadInfo; } }; -- cgit v1.2.3