diff options
| author | Shauren <shauren.trinity@gmail.com> | 2017-03-08 18:10:02 +0100 |
|---|---|---|
| committer | Shauren <shauren.trinity@gmail.com> | 2017-03-08 18:10:02 +0100 |
| commit | 7b235ce6e4da0e9c19fa9c6306bc7a71c7fb905d (patch) | |
| tree | 28611e02e44e55d28f60a9b1b3433831ed6772b8 /src/server/shared/DataStores | |
| parent | f585c831248f24c93697b0c314dd015897febe39 (diff) | |
Core/DataStores: Refactor DB2 loaders to be reusable by extractors
Diffstat (limited to 'src/server/shared/DataStores')
| -rw-r--r-- | src/server/shared/DataStores/DB2DatabaseLoader.cpp | 301 | ||||
| -rw-r--r-- | src/server/shared/DataStores/DB2DatabaseLoader.h (renamed from src/server/shared/DataStores/DB2StorageLoader.h) | 57 | ||||
| -rw-r--r-- | src/server/shared/DataStores/DB2StorageLoader.cpp | 1257 | ||||
| -rw-r--r-- | src/server/shared/DataStores/DB2Store.cpp | 189 | ||||
| -rw-r--r-- | src/server/shared/DataStores/DB2Store.h | 137 |
5 files changed, 515 insertions, 1426 deletions
diff --git a/src/server/shared/DataStores/DB2DatabaseLoader.cpp b/src/server/shared/DataStores/DB2DatabaseLoader.cpp new file mode 100644 index 00000000000..a19b8a5b389 --- /dev/null +++ b/src/server/shared/DataStores/DB2DatabaseLoader.cpp @@ -0,0 +1,301 @@ +/* + * Copyright (C) 2008-2017 TrinityCore <http://www.trinitycore.org/> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "DB2DatabaseLoader.h" +#include "Database/DatabaseEnv.h" +#include "DB2Meta.h" + +DB2LoadInfo::DB2LoadInfo() : DB2FileLoadInfo() +{ +} + +DB2LoadInfo::DB2LoadInfo(DB2FieldMeta const* fields, std::size_t fieldCount, DB2Meta const* meta, HotfixDatabaseStatements statement) + : DB2FileLoadInfo(fields, fieldCount, meta), Statement(statement) +{ +} + +static char const* nullStr = ""; + +char* DB2DatabaseLoader::Load(uint32& records, char**& indexTable, char*& stringHolders, std::vector<char*>& stringPool) +{ + // Even though this query is executed only once, prepared statement is used to send data from mysql server in binary format + PreparedQueryResult result = HotfixDatabase.Query(HotfixDatabase.GetPreparedStatement(_loadInfo->Statement)); + if (!result) + return nullptr; + + if (_loadInfo->Meta->GetDbFieldCount() != result->GetFieldCount()) + return nullptr; + + // get struct size and index pos + uint32 indexField = _loadInfo->Meta->GetDbIndexField(); + uint32 recordSize = _loadInfo->Meta->GetRecordSize(); + + // we store flat holders pool as single memory block + std::size_t stringFields = _loadInfo->GetStringFieldCount(false); + std::size_t localizedStringFields = _loadInfo->GetStringFieldCount(true); + + // each string field at load have array of string for each locale + std::size_t stringHoldersRecordPoolSize = localizedStringFields * sizeof(LocalizedString) + (stringFields - localizedStringFields) * sizeof(char*); + + if (stringFields) + { + std::size_t stringHoldersPoolSize = stringHoldersRecordPoolSize * result->GetRowCount(); + stringHolders = new char[stringHoldersPoolSize]; + + // DB2 strings expected to have at least empty string + for (std::size_t i = 0; i < stringHoldersPoolSize / sizeof(char*); ++i) + ((char const**)stringHolders)[i] = nullStr; + } + else + stringHolders = nullptr; + + // Resize index table + // database query *MUST* contain ORDER BY `index_field` DESC clause + uint32 indexTableSize = (*result)[indexField].GetUInt32() + 1; + if (indexTableSize < records) + indexTableSize = records; + + if (indexTableSize > records) + { + char** tmpIdxTable = new char*[indexTableSize]; + memset(tmpIdxTable, 0, indexTableSize * sizeof(char*)); + memcpy(tmpIdxTable, indexTable, records * sizeof(char*)); + delete[] indexTable; + indexTable = tmpIdxTable; + } + + char* tempDataTable = new char[result->GetRowCount() * recordSize]; + uint32* newIndexes = new uint32[result->GetRowCount()]; + uint32 rec = 0; + uint32 newRecords = 0; + + do + { + Field* fields = result->Fetch(); + uint32 offset = 0; + uint32 stringFieldOffset = 0; + + uint32 indexValue = fields[indexField].GetUInt32(); + + // Attempt to overwrite existing data + char* dataValue = indexTable[indexValue]; + if (!dataValue) + { + newIndexes[newRecords] = indexValue; + dataValue = &tempDataTable[newRecords++ * recordSize]; + } + + uint32 f = 0; + if (!_loadInfo->Meta->HasIndexFieldInData()) + { + *((uint32*)(&dataValue[offset])) = indexValue; + offset += 4; + ++f; + } + + for (uint32 x = 0; x < _loadInfo->Meta->FieldCount; ++x) + { + for (uint32 z = 0; z < _loadInfo->Meta->ArraySizes[x]; ++z) + { + switch (_loadInfo->TypesString[f]) + { + case FT_FLOAT: + *((float*)(&dataValue[offset])) = fields[f].GetFloat(); + offset += 4; + break; + case FT_INT: + *((int32*)(&dataValue[offset])) = fields[f].GetInt32(); + offset += 4; + break; + case FT_BYTE: + *((int8*)(&dataValue[offset])) = fields[f].GetInt8(); + offset += 1; + break; + case FT_SHORT: + *((int16*)(&dataValue[offset])) = fields[f].GetInt16(); + offset += 2; + break; + case FT_STRING: + { + LocalizedString** slot = (LocalizedString**)(&dataValue[offset]); + *slot = (LocalizedString*)(&stringHolders[stringHoldersRecordPoolSize * rec + stringFieldOffset]); + ASSERT(*slot); + + // Value in database in main table field must be for enUS locale + if (char* str = AddString(&(*slot)->Str[LOCALE_enUS], fields[f].GetString())) + stringPool.push_back(str); + + stringFieldOffset += sizeof(LocalizedString); + offset += sizeof(char*); + break; + } + case FT_STRING_NOT_LOCALIZED: + { + char const** slot = (char const**)(&dataValue[offset]); + *slot = (char*)(&stringHolders[stringHoldersRecordPoolSize * rec + stringFieldOffset]); + ASSERT(*slot); + + // Value in database in main table field must be for enUS locale + if (char* str = AddString(slot, fields[f].GetString())) + stringPool.push_back(str); + + stringFieldOffset += sizeof(char*); + offset += sizeof(char*); + break; + } + default: + ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->TypesString[x], _storageName.c_str()); + break; + } + ++f; + } + } + + ASSERT(offset == recordSize); + ++rec; + } while (result->NextRow()); + + if (!newRecords) + { + delete[] tempDataTable; + delete[] newIndexes; + return nullptr; + } + + // Compact new data table to only contain new records not previously loaded from file + char* dataTable = new char[newRecords * recordSize]; + memcpy(dataTable, tempDataTable, newRecords * recordSize); + + // insert new records to index table + for (uint32 i = 0; i < newRecords; ++i) + indexTable[newIndexes[i]] = &dataTable[i * recordSize]; + + delete[] tempDataTable; + delete[] newIndexes; + + records = indexTableSize; + + return dataTable; +} + +void DB2DatabaseLoader::LoadStrings(uint32 locale, uint32 records, char** indexTable, std::vector<char*>& stringPool) +{ + PreparedStatement* stmt = HotfixDatabase.GetPreparedStatement(HotfixDatabaseStatements(_loadInfo->Statement + 1)); + stmt->setString(0, localeNames[locale]); + PreparedQueryResult result = HotfixDatabase.Query(stmt); + if (!result) + return; + + std::size_t stringFields = _loadInfo->GetStringFieldCount(true); + if (result->GetFieldCount() != stringFields + 1 /*ID*/) + return; + + uint32 fieldCount = _loadInfo->Meta->FieldCount; + uint32 recordSize = _loadInfo->Meta->GetRecordSize(); + + do + { + Field* fields = result->Fetch(); + uint32 offset = 0; + uint32 stringFieldNumInRecord = 0; + uint32 indexValue = fields[0].GetUInt32(); + + if (indexValue >= records) + continue; + + // Attempt to overwrite existing data + if (char* dataValue = indexTable[indexValue]) + { + uint32 fieldIndex = 0; + if (!_loadInfo->Meta->HasIndexFieldInData()) + { + offset += 4; + ++fieldIndex; + } + + for (uint32 x = 0; x < fieldCount; ++x) + { + for (uint32 z = 0; z < _loadInfo->Meta->ArraySizes[x]; ++z) + { + switch (_loadInfo->TypesString[fieldIndex]) + { + case FT_FLOAT: + case FT_INT: + offset += 4; + break; + case FT_BYTE: + offset += 1; + break; + case FT_SHORT: + offset += 2; + break; + case FT_STRING: + { + // fill only not filled entries + LocalizedString* db2str = *(LocalizedString**)(&dataValue[offset]); + if (db2str->Str[locale] == nullStr) + if (char* str = AddString(&db2str->Str[locale], fields[1 + stringFieldNumInRecord].GetString())) + stringPool.push_back(str); + + ++stringFieldNumInRecord; + offset += sizeof(LocalizedString*); + break; + } + case FT_STRING_NOT_LOCALIZED: + offset += sizeof(char*); + break; + default: + ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->TypesString[x], _storageName.c_str()); + break; + } + ++fieldIndex; + } + } + + ASSERT(offset == recordSize); + } + else + TC_LOG_ERROR("sql.sql", "Hotfix locale table for storage %s references row that does not exist %u locale %s!", _storageName.c_str(), indexValue, localeNames[locale]); + + } while (result->NextRow()); + + return; +} + +char* DB2DatabaseLoader::AddString(char const** holder, std::string const& value) +{ + if (!value.empty()) + { + std::size_t existingLength = strlen(*holder); + if (existingLength >= value.length()) + { + // Reuse existing storage if there is enough space + char* str = const_cast<char*>(*holder); + memcpy(str, value.c_str(), value.length()); + str[value.length()] = '\0'; + return nullptr; + } + + char* str = new char[value.length() + 1]; + memcpy(str, value.c_str(), value.length()); + str[value.length()] = '\0'; + *holder = str; + return str; + } + + return nullptr; +} diff --git a/src/server/shared/DataStores/DB2StorageLoader.h b/src/server/shared/DataStores/DB2DatabaseLoader.h index 62361edcedd..f288b46859b 100644 --- a/src/server/shared/DataStores/DB2StorageLoader.h +++ b/src/server/shared/DataStores/DB2DatabaseLoader.h @@ -15,67 +15,18 @@ * with this program. If not, see <http://www.gnu.org/licenses/>. */ -#ifndef DB2_FILE_LOADER_H -#define DB2_FILE_LOADER_H +#ifndef DB2_DATABASE_LOADER_H +#define DB2_DATABASE_LOADER_H -#include "Define.h" +#include "DB2FileLoader.h" #include "Implementation/HotfixDatabase.h" -class DB2FileLoaderImpl; -struct DB2Meta; -struct DB2FieldMeta; - -#pragma pack(push, 1) -struct DB2Header -{ - uint32 Signature; - uint32 RecordCount; - uint32 FieldCount; - uint32 RecordSize; - uint32 StringTableSize; - uint32 TableHash; - uint32 LayoutHash; - uint32 MinId; - uint32 MaxId; - uint32 Locale; - uint32 CopyTableSize; - uint16 Flags; - int16 IndexField; -}; -#pragma pack(pop) - -struct TC_SHARED_API DB2LoadInfo +struct TC_SHARED_API DB2LoadInfo : public DB2FileLoadInfo { DB2LoadInfo(); DB2LoadInfo(DB2FieldMeta const* fields, std::size_t fieldCount, DB2Meta const* meta, HotfixDatabaseStatements statement); - uint32 GetStringFieldCount(bool localizedOnly) const; - - DB2FieldMeta const* Fields; - std::size_t FieldCount; - DB2Meta const* Meta; HotfixDatabaseStatements Statement; - std::string TypesString; -}; - -class TC_SHARED_API DB2FileLoader -{ -public: - DB2FileLoader(); - ~DB2FileLoader(); - - bool Load(char const* filename, DB2LoadInfo const* loadInfo); - char* AutoProduceData(uint32& count, char**& indexTable, std::vector<char*>& stringPool); - char* AutoProduceStrings(char* dataTable, uint32 locale); - void AutoProduceRecordCopies(uint32 records, char** indexTable, char* dataTable); - - uint32 GetCols() const { return _header.FieldCount; } - uint32 GetTableHash() const { return _header.TableHash; } - uint32 GetLayoutHash() const { return _header.LayoutHash; } - -private: - DB2FileLoaderImpl* _impl; - DB2Header _header; }; class TC_SHARED_API DB2DatabaseLoader diff --git a/src/server/shared/DataStores/DB2StorageLoader.cpp b/src/server/shared/DataStores/DB2StorageLoader.cpp deleted file mode 100644 index 11caca97afb..00000000000 --- a/src/server/shared/DataStores/DB2StorageLoader.cpp +++ /dev/null @@ -1,1257 +0,0 @@ -/* - * Copyright (C) 2008-2017 TrinityCore <http://www.trinitycore.org/> - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along - * with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -#include "DB2StorageLoader.h" -#include "Database/DatabaseEnv.h" -#include "DB2Meta.h" -#include "Utilities/ByteConverter.h" - -DB2LoadInfo::DB2LoadInfo() : Fields(nullptr), FieldCount(0), Meta(nullptr), Statement(MAX_HOTFIXDATABASE_STATEMENTS) -{ -} - -DB2LoadInfo::DB2LoadInfo(DB2FieldMeta const* fields, std::size_t fieldCount, DB2Meta const* meta, HotfixDatabaseStatements statement) - : Fields(fields), FieldCount(fieldCount), Meta(meta), Statement(statement) -{ - TypesString.reserve(FieldCount); - for (std::size_t i = 0; i < FieldCount; ++i) - TypesString += char(Fields[i].Type); -} - -uint32 DB2LoadInfo::GetStringFieldCount(bool localizedOnly) const -{ - uint32 stringFields = 0; - for (std::size_t i = 0; i < TypesString.length(); ++i) - if (TypesString[i] == FT_STRING || (TypesString[i] == FT_STRING_NOT_LOCALIZED && !localizedOnly)) - ++stringFields; - - return stringFields; -} - -class DB2FileLoaderImpl -{ -public: - virtual ~DB2FileLoaderImpl() { } - virtual bool Load(char const* filename, FILE* file, DB2LoadInfo const* loadInfo, DB2Header const* header) = 0; - virtual char* AutoProduceData(uint32& count, char**& indexTable, std::vector<char*>& stringPool) = 0; - virtual char* AutoProduceStrings(char* dataTable, uint32 locale) = 0; - virtual void AutoProduceRecordCopies(uint32 records, char** indexTable, char* dataTable) = 0; -}; - -class DB2FileLoaderRegularImpl : public DB2FileLoaderImpl -{ -public: - DB2FileLoaderRegularImpl(); - ~DB2FileLoaderRegularImpl(); - - bool Load(char const* filename, FILE* file, DB2LoadInfo const* loadInfo, DB2Header const* header) override; - char* AutoProduceData(uint32& count, char**& indexTable, std::vector<char*>& stringPool) override; - char* AutoProduceStrings(char* dataTable, uint32 locale) override; - void AutoProduceRecordCopies(uint32 records, char** indexTable, char* dataTable) override; - -private: - class Record - { - public: - float getFloat(uint32 field, uint32 arrayIndex) const; - uint32 getUInt(uint32 field, uint32 arrayIndex, bool isSigned) const; - uint8 getUInt8(uint32 field, uint32 arrayIndex) const; - uint16 getUInt16(uint32 field, uint32 arrayIndex) const; - char const* getString(uint32 field, uint32 arrayIndex) const; - - private: - uint16 GetOffset(uint32 field) const; - uint16 GetByteSize(uint32 field) const; - uint32 GetVarInt(uint32 field, uint32 arrayIndex, bool isSigned) const; - - Record(DB2FileLoaderRegularImpl &file_, unsigned char *offset_); - unsigned char *offset; - DB2FileLoaderRegularImpl &file; - - friend class DB2FileLoaderRegularImpl; - }; - - // Get record by id - Record getRecord(size_t id); - -#pragma pack(push, 1) - struct FieldEntry - { - uint16 UnusedBits; - uint16 Offset; - }; -#pragma pack(pop) - - char const* fileName; - DB2LoadInfo const* _loadInfo; - DB2Header const* _header; - - unsigned char* data; - unsigned char* stringTable; - unsigned char* idTable; - uint32 idTableSize; - unsigned char* copyTable; - FieldEntry* fields; -}; - -class DB2FileLoaderSparseImpl : public DB2FileLoaderImpl -{ -public: - DB2FileLoaderSparseImpl(); - ~DB2FileLoaderSparseImpl(); - - bool Load(char const* filename, FILE* file, DB2LoadInfo const* loadInfo, DB2Header const* header) override; - char* AutoProduceData(uint32& records, char**& indexTable, std::vector<char*>& stringPool) override; - char* AutoProduceStrings(char* dataTable, uint32 locale) override; - void AutoProduceRecordCopies(uint32 /*records*/, char** /*indexTable*/, char* /*dataTable*/) override { } - -private: -#pragma pack(push, 1) - struct OffsetTableEntry - { - uint32 FileOffset; - uint16 RecordSize; - }; - struct FieldEntry - { - uint16 UnusedBits; - uint16 Offset; - }; -#pragma pack(pop) - - char const* fileName; - DB2LoadInfo const* _loadInfo; - DB2Header const* _header; - FieldEntry* fields; - - uint32 dataStart; - unsigned char* data; - OffsetTableEntry* offsets; -}; - -float DB2FileLoaderRegularImpl::Record::getFloat(uint32 field, uint32 arrayIndex) const -{ - ASSERT(field < file._header->FieldCount); - float val = *reinterpret_cast<float*>(offset + GetOffset(field) + arrayIndex * sizeof(float)); - EndianConvert(val); - return val; -} - -uint32 DB2FileLoaderRegularImpl::Record::getUInt(uint32 field, uint32 arrayIndex, bool isSigned) const -{ - ASSERT(field < file._header->FieldCount); - return GetVarInt(field, arrayIndex, isSigned); -} - -uint8 DB2FileLoaderRegularImpl::Record::getUInt8(uint32 field, uint32 arrayIndex) const -{ - ASSERT(field < file._header->FieldCount); - ASSERT(GetByteSize(field) == 1); - return *reinterpret_cast<uint8*>(offset + GetOffset(field) + arrayIndex * sizeof(uint8)); -} - -uint16 DB2FileLoaderRegularImpl::Record::getUInt16(uint32 field, uint32 arrayIndex) const -{ - ASSERT(field < file._header->FieldCount); - ASSERT(GetByteSize(field) == 2); - uint16 val = *reinterpret_cast<uint16*>(offset + GetOffset(field) + arrayIndex * sizeof(uint16)); - EndianConvert(val); - return val; -} - -char const* DB2FileLoaderRegularImpl::Record::getString(uint32 field, uint32 arrayIndex) const -{ - ASSERT(field < file._header->FieldCount); - uint32 stringOffset = *reinterpret_cast<uint32*>(offset + GetOffset(field) + arrayIndex * sizeof(uint32)); - EndianConvert(stringOffset); - ASSERT(stringOffset < file._header->StringTableSize); - return reinterpret_cast<char*>(file.stringTable + stringOffset); -} - -uint16 DB2FileLoaderRegularImpl::Record::GetOffset(uint32 field) const -{ - ASSERT(field < file._header->FieldCount); - return file.fields[field].Offset; -} - -uint16 DB2FileLoaderRegularImpl::Record::GetByteSize(uint32 field) const -{ - ASSERT(field < file._header->FieldCount); - return 4 - file.fields[field].UnusedBits / 8; -} - -uint32 DB2FileLoaderRegularImpl::Record::GetVarInt(uint32 field, uint32 arrayIndex, bool isSigned) const -{ - ASSERT(field < file._header->FieldCount); - uint32 val = *reinterpret_cast<uint32*>(offset + GetOffset(field) + arrayIndex * (4 - file.fields[field].UnusedBits / 8)); - EndianConvert(val); - if (isSigned) - return int32(val) << file.fields[field].UnusedBits >> file.fields[field].UnusedBits; - - return val << file.fields[field].UnusedBits >> file.fields[field].UnusedBits; -} - -DB2FileLoaderRegularImpl::Record::Record(DB2FileLoaderRegularImpl &file_, unsigned char *offset_) : offset(offset_), file(file_) -{ -} - -DB2FileLoaderRegularImpl::DB2FileLoaderRegularImpl() -{ - fileName = nullptr; - _loadInfo = nullptr; - _header = nullptr; - data = nullptr; - stringTable = nullptr; - idTable = nullptr; - idTableSize = 0; - copyTable = nullptr; - fields = nullptr; -} - -bool DB2FileLoaderRegularImpl::Load(char const* filename, FILE* file, DB2LoadInfo const* loadInfo, DB2Header const* header) -{ - _loadInfo = loadInfo; - _header = header; - ASSERT(_loadInfo->Meta->IndexField == -1 || _loadInfo->Meta->IndexField == int32(header->IndexField)); - - fileName = filename; - fields = new FieldEntry[header->FieldCount]; - if (fread(fields, header->FieldCount * sizeof(FieldEntry), 1, file) != 1) - return false; - - if (!_loadInfo->Meta->HasIndexFieldInData()) - idTableSize = header->RecordCount * sizeof(uint32); - - data = new unsigned char[header->RecordSize * header->RecordCount + header->StringTableSize]; - stringTable = data + header->RecordSize * header->RecordCount; - - if (fread(data, header->RecordSize * header->RecordCount + header->StringTableSize, 1, file) != 1) - return false; - - if (idTableSize) - { - idTable = new unsigned char[idTableSize]; - if (fread(idTable, idTableSize, 1, file) != 1) - return false; - } - - if (header->CopyTableSize) - { - copyTable = new unsigned char[header->CopyTableSize]; - if (fread(copyTable, header->CopyTableSize, 1, file) != 1) - return false; - } - - return true; -} - -DB2FileLoaderRegularImpl::~DB2FileLoaderRegularImpl() -{ - delete[] data; - delete[] idTable; - delete[] copyTable; - delete[] fields; -} - -DB2FileLoaderRegularImpl::Record DB2FileLoaderRegularImpl::getRecord(size_t id) -{ - assert(data); - return Record(*this, data + id * _header->RecordSize); -} - -static char const* const nullStr = ""; - -char* DB2FileLoaderRegularImpl::AutoProduceData(uint32& records, char**& indexTable, std::vector<char*>& stringPool) -{ - if (_loadInfo->Meta->FieldCount != _header->FieldCount) - return nullptr; - - //get struct size and index pos - uint32 indexField = _loadInfo->Meta->GetIndexField(); - uint32 recordsize = _loadInfo->Meta->GetRecordSize(); - - uint32 maxi = 0; - //find max index - if (!idTableSize) - { - for (uint32 y = 0; y < _header->RecordCount; ++y) - { - uint32 ind = getRecord(y).getUInt(indexField, 0, false); - if (ind > maxi) - maxi = ind; - } - } - else - { - for (uint32 y = 0; y < _header->RecordCount; ++y) - { - uint32 ind = ((uint32*)idTable)[y]; - if (ind > maxi) - maxi = ind; - } - } - - for (uint32 y = 0; y < _header->CopyTableSize; y += 8) - { - uint32 ind = *((uint32*)(copyTable + y)); - if (ind > maxi) - maxi = ind; - } - - using index_entry_t = char*; - - ++maxi; - records = maxi; - indexTable = new index_entry_t[maxi]; - memset(indexTable, 0, maxi * sizeof(index_entry_t)); - - char* dataTable = new char[(_header->RecordCount + (_header->CopyTableSize / 8)) * recordsize]; - - // we store flat holders pool as single memory block - std::size_t stringFields = _loadInfo->GetStringFieldCount(false); - std::size_t localizedStringFields = _loadInfo->GetStringFieldCount(true); - - // each string field at load have array of string for each locale - std::size_t stringHoldersRecordPoolSize = localizedStringFields * sizeof(LocalizedString) + (stringFields - localizedStringFields) * sizeof(char*); - char* stringHoldersPool = nullptr; - if (stringFields) - { - std::size_t stringHoldersPoolSize = stringHoldersRecordPoolSize * _header->RecordCount; - - stringHoldersPool = new char[stringHoldersPoolSize]; - stringPool.push_back(stringHoldersPool); - - // DB2 strings expected to have at least empty string - for (std::size_t i = 0; i < stringHoldersPoolSize / sizeof(char*); ++i) - ((char const**)stringHoldersPool)[i] = nullStr; - } - - uint32 offset = 0; - - for (uint32 y = 0; y < _header->RecordCount; y++) - { - Record rec = getRecord(y); - uint32 indexVal = _loadInfo->Meta->HasIndexFieldInData() ? rec.getUInt(indexField, 0, false) : ((uint32*)idTable)[y]; - - indexTable[indexVal] = &dataTable[offset]; - - uint32 fieldIndex = 0; - if (!_loadInfo->Meta->HasIndexFieldInData()) - { - *((uint32*)(&dataTable[offset])) = indexVal; - offset += 4; - ++fieldIndex; - } - - uint32 stringFieldOffset = 0; - - for (uint32 x = 0; x < _header->FieldCount; ++x) - { - for (uint32 z = 0; z < _loadInfo->Meta->ArraySizes[x]; ++z) - { - switch (_loadInfo->TypesString[fieldIndex]) - { - case FT_FLOAT: - *((float*)(&dataTable[offset])) = rec.getFloat(x, z); - offset += 4; - break; - case FT_INT: - *((uint32*)(&dataTable[offset])) = rec.getUInt(x, z, _loadInfo->Fields[fieldIndex].IsSigned); - offset += 4; - break; - case FT_BYTE: - *((uint8*)(&dataTable[offset])) = rec.getUInt8(x, z); - offset += 1; - break; - case FT_SHORT: - *((uint16*)(&dataTable[offset])) = rec.getUInt16(x, z); - offset += 2; - break; - case FT_STRING: - case FT_STRING_NOT_LOCALIZED: - { - // init db2 string field slots by pointers to string holders - char const*** slot = (char const***)(&dataTable[offset]); - *slot = (char const**)(&stringHoldersPool[stringHoldersRecordPoolSize * y + stringFieldOffset]); - if (_loadInfo->TypesString[fieldIndex] == FT_STRING) - stringFieldOffset += sizeof(LocalizedString); - else - stringFieldOffset += sizeof(char*); - - offset += sizeof(char*); - break; - } - default: - ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->TypesString[x], fileName); - break; - } - ++fieldIndex; - } - } - } - - return dataTable; -} - -char* DB2FileLoaderRegularImpl::AutoProduceStrings(char* dataTable, uint32 locale) -{ - if (_loadInfo->Meta->FieldCount != _header->FieldCount) - return nullptr; - - if (!(_header->Locale & (1 << locale))) - { - char const* sep = ""; - std::ostringstream str; - for (uint32 i = 0; i < TOTAL_LOCALES; ++i) - { - if (_header->Locale & (1 << i)) - { - str << sep << localeNames[i]; - sep = ", "; - } - } - - TC_LOG_ERROR("", "Attempted to load %s which has locales %s as %s. Check if you placed your localized db2 files in correct directory.", fileName, str.str().c_str(), localeNames[locale]); - return nullptr; - } - - char* stringPool = new char[_header->StringTableSize]; - memcpy(stringPool, stringTable, _header->StringTableSize); - - uint32 offset = 0; - - for (uint32 y = 0; y < _header->RecordCount; y++) - { - uint32 fieldIndex = 0; - if (!_loadInfo->Meta->HasIndexFieldInData()) - { - offset += 4; - ++fieldIndex; - } - - for (uint32 x = 0; x < _header->FieldCount; ++x) - { - for (uint32 z = 0; z < _loadInfo->Meta->ArraySizes[x]; ++z) - { - switch (_loadInfo->TypesString[fieldIndex]) - { - case FT_FLOAT: - case FT_INT: - offset += 4; - break; - case FT_BYTE: - offset += 1; - break; - case FT_SHORT: - offset += 2; - break; - case FT_STRING: - { - // fill only not filled entries - LocalizedString* db2str = *(LocalizedString**)(&dataTable[offset]); - if (db2str->Str[locale] == nullStr) - { - char const* st = getRecord(y).getString(x, z); - db2str->Str[locale] = stringPool + (st - (char const*)stringTable); - } - - offset += sizeof(char*); - break; - } - case FT_STRING_NOT_LOCALIZED: - { - char** db2str = (char**)(&dataTable[offset]); - char const* st = getRecord(y).getString(x, z); - *db2str = stringPool + (st - (char const*)stringTable); - offset += sizeof(char*); - break; - } - default: - ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->TypesString[x], fileName); - break; - } - ++fieldIndex; - } - } - } - - return stringPool; -} - -void DB2FileLoaderRegularImpl::AutoProduceRecordCopies(uint32 records, char** indexTable, char* dataTable) -{ - uint32 recordsize = _loadInfo->Meta->GetRecordSize(); - uint32 offset = _header->RecordCount * recordsize; - uint32* copyIds = (uint32*)copyTable; - for (uint32 c = 0; c < _header->CopyTableSize / 4; c += 2) - { - uint32 to = copyIds[c]; - uint32 from = copyIds[c + 1]; - - if (from && from < records && to < records && indexTable[from]) - { - indexTable[to] = &dataTable[offset]; - memcpy(indexTable[to], indexTable[from], recordsize); - - if (_loadInfo->Meta->HasIndexFieldInData()) - *((uint32*)(&dataTable[offset + fields[_loadInfo->Meta->GetIndexField()].Offset])) = to; - else - *((uint32*)(&dataTable[offset])) = to; - - offset += recordsize; - } - } -} - -DB2FileLoaderSparseImpl::DB2FileLoaderSparseImpl() -{ - fileName = nullptr; - _loadInfo = nullptr; - _header = nullptr; - fields = nullptr; - dataStart = 0; - data = nullptr; - offsets = nullptr; -} - -bool DB2FileLoaderSparseImpl::Load(char const* filename, FILE* file, DB2LoadInfo const* loadInfo, DB2Header const* header) -{ - _loadInfo = loadInfo; - _header = header; - fileName = filename; - - fields = new FieldEntry[header->FieldCount]; - if (fread(fields, header->FieldCount * sizeof(FieldEntry), 1, file) != 1) - return false; - - dataStart = ftell(file); - - data = new unsigned char[header->StringTableSize - dataStart]; - - if (fread(data, header->StringTableSize - dataStart, 1, file) != 1) - return false; - - offsets = new OffsetTableEntry[header->MaxId - header->MinId + 1]; - if (fread(offsets, (header->MaxId - header->MinId + 1) * 6, 1, file) != 1) - return false; - - return true; -} - -DB2FileLoaderSparseImpl::~DB2FileLoaderSparseImpl() -{ - delete[] data; - delete[] offsets; - delete[] fields; -} - -char* DB2FileLoaderSparseImpl::AutoProduceData(uint32& maxId, char**& indexTable, std::vector<char*>& stringPool) -{ - if (_loadInfo->Meta->FieldCount != _header->FieldCount) - return NULL; - - //get struct size and index pos - uint32 recordsize = _loadInfo->Meta->GetRecordSize(); - - uint32 offsetCount = _header->MaxId - _header->MinId + 1; - uint32 records = 0; - uint32 expandedDataSize = 0; - for (uint32 i = 0; i < offsetCount; ++i) - { - if (offsets[i].FileOffset && offsets[i].RecordSize) - { - ++records; - expandedDataSize += offsets[i].RecordSize; - } - } - - using index_entry_t = char*; - - maxId = _header->MaxId + 1; - indexTable = new index_entry_t[maxId]; - memset(indexTable, 0, maxId * sizeof(index_entry_t)); - - char* dataTable = new char[records * recordsize]; - - // we store flat holders pool as single memory block - std::size_t stringFields = _loadInfo->GetStringFieldCount(false); - std::size_t localizedStringFields = _loadInfo->GetStringFieldCount(true); - - // each string field at load have array of string for each locale - std::size_t stringHoldersRecordPoolSize = localizedStringFields * sizeof(LocalizedString) + (stringFields - localizedStringFields) * sizeof(char*); - std::size_t stringHoldersPoolSize = stringHoldersRecordPoolSize * records; - - char* stringHoldersPool = new char[stringHoldersPoolSize]; - stringPool.push_back(stringHoldersPool); - - // DB2 strings expected to have at least empty string - for (std::size_t i = 0; i < stringHoldersPoolSize / sizeof(char*); ++i) - ((char const**)stringHoldersPool)[i] = nullStr; - - char* stringTable = new char[expandedDataSize - records * ((recordsize - (!_loadInfo->Meta->HasIndexFieldInData() ? 4 : 0)) - stringFields * sizeof(char*))]; - memset(stringTable, 0, expandedDataSize - records * ((recordsize - (!_loadInfo->Meta->HasIndexFieldInData() ? 4 : 0)) - stringFields * sizeof(char*))); - stringPool.push_back(stringTable); - char* stringPtr = stringTable; - - uint32 offset = 0; - uint32 recordNum = 0; - for (uint32 y = 0; y < offsetCount; ++y) - { - if (!offsets[y].FileOffset || !offsets[y].RecordSize) - continue; - - indexTable[y + _header->MinId] = &dataTable[offset]; - ASSERT(indexTable[y + _header->MinId]); - std::size_t fieldOffset = 0; - uint32 stringFieldOffset = 0; - - uint32 fieldIndex = 0; - if (!_loadInfo->Meta->HasIndexFieldInData()) - { - *((uint32*)(&dataTable[offset])) = y + _header->MinId; - offset += 4; - ++fieldIndex; - } - - for (uint32 x = 0; x < _header->FieldCount; ++x) - { - uint16 fieldBytes = 4 - fields[x].UnusedBits / 8; - for (uint32 z = 0; z < _loadInfo->Meta->ArraySizes[x]; ++z) - { - switch (_loadInfo->TypesString[fieldIndex]) - { - case FT_FLOAT: - { - float val = *reinterpret_cast<float*>(&data[offsets[y].FileOffset - dataStart + fieldOffset]); - EndianConvert(val); - *((float*)(&dataTable[offset])) = val; - offset += 4; - fieldOffset += 4; - break; - } - case FT_INT: - { - ASSERT(fieldBytes && fieldBytes <= 4); - uint32 val = 0; - switch (fieldBytes) - { - case 1: - val = *reinterpret_cast<uint8*>(&data[offsets[y].FileOffset - dataStart + fieldOffset]); - break; - case 2: - { - uint16 val16 = *reinterpret_cast<uint16*>(&data[offsets[y].FileOffset - dataStart + fieldOffset]); - EndianConvert(val16); - val = val16; - break; - } - case 3: - { -#pragma pack(push, 1) - struct dbcint24 { uint8 v[3]; }; -#pragma pack(pop) - dbcint24 i24v = *reinterpret_cast<dbcint24*>(&data[offsets[y].FileOffset - dataStart + fieldOffset]); - EndianConvert(i24v); - val = uint32(i24v.v[0]) | (uint32(i24v.v[1]) << 8) | (uint32(i24v.v[2]) << 16); - break; - } - case 4: - val = *reinterpret_cast<uint32*>(&data[offsets[y].FileOffset - dataStart + fieldOffset]); - EndianConvert(val); - break; - default: - break; - } - *((uint32*)(&dataTable[offset])) = val; - offset += 4; - fieldOffset += fieldBytes; - break; - } - case FT_BYTE: - { - ASSERT(fieldBytes == 1); - *((uint8*)(&dataTable[offset])) = *reinterpret_cast<uint8*>(&data[offsets[y].FileOffset - dataStart + fieldOffset]); - offset += 1; - fieldOffset += 1; - break; - } - case FT_SHORT: - { - ASSERT(fieldBytes == 2); - uint16 val = *reinterpret_cast<uint16*>(&data[offsets[y].FileOffset - dataStart + fieldOffset]); - EndianConvert(val); - *((uint16*)(&dataTable[offset])) = val; - offset += 2; - fieldOffset += 2; - break; - } - case FT_STRING: - { - LocalizedString** slot = (LocalizedString**)(&dataTable[offset]); - *slot = (LocalizedString*)(&stringHoldersPool[stringHoldersRecordPoolSize * recordNum + stringFieldOffset]); - for (uint32 locale = 0; locale < TOTAL_LOCALES; ++locale) - if (_header->Locale & (1 << locale)) - (*slot)->Str[locale] = stringPtr; - strcpy(stringPtr, (char*)&data[offsets[y].FileOffset - dataStart + fieldOffset]); - fieldOffset += strlen(stringPtr) + 1; - stringPtr += strlen(stringPtr) + 1; - stringFieldOffset += sizeof(LocalizedString); - offset += sizeof(LocalizedString*); - break; - } - case FT_STRING_NOT_LOCALIZED: - { - char const*** slot = (char const***)(&dataTable[offset]); - *slot = (char const**)(&stringHoldersPool[stringHoldersRecordPoolSize * recordNum + stringFieldOffset]); - **slot = stringPtr; - strcpy(stringPtr, (char*)&data[offsets[y].FileOffset - dataStart + fieldOffset]); - fieldOffset += strlen(stringPtr) + 1; - stringPtr += strlen(stringPtr) + 1; - stringFieldOffset += sizeof(char*); - offset += sizeof(char*); - break; - } - default: - ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->TypesString[x], fileName); - break; - } - ++fieldIndex; - } - } - - ++recordNum; - } - - return dataTable; -} - -char* DB2FileLoaderSparseImpl::AutoProduceStrings(char* dataTable, uint32 locale) -{ - if (_loadInfo->Meta->FieldCount != _header->FieldCount) - return nullptr; - - if (!(_header->Locale & (1 << locale))) - { - char const* sep = ""; - std::ostringstream str; - for (uint32 i = 0; i < TOTAL_LOCALES; ++i) - { - if (_header->Locale & (1 << i)) - { - str << sep << localeNames[i]; - sep = ", "; - } - } - - TC_LOG_ERROR("", "Attempted to load %s which has locales %s as %s. Check if you placed your localized db2 files in correct directory.", fileName, str.str().c_str(), localeNames[locale]); - return nullptr; - } - - uint32 offsetCount = _header->MaxId - _header->MinId + 1; - uint32 records = 0; - for (uint32 i = 0; i < offsetCount; ++i) - if (offsets[i].FileOffset && offsets[i].RecordSize) - ++records; - - uint32 recordsize = _loadInfo->Meta->GetRecordSize(); - std::size_t stringFields = _loadInfo->GetStringFieldCount(true); - char* stringTable = new char[_header->StringTableSize - dataStart - records * ((recordsize - (!_loadInfo->Meta->HasIndexFieldInData() ? 4 : 0)) - stringFields * sizeof(char*))]; - memset(stringTable, 0, _header->StringTableSize - dataStart - records * ((recordsize - (!_loadInfo->Meta->HasIndexFieldInData() ? 4 : 0)) - stringFields * sizeof(char*))); - char* stringPtr = stringTable; - - uint32 offset = 0; - - for (uint32 y = 0; y < offsetCount; y++) - { - if (!offsets[y].FileOffset || !offsets[y].RecordSize) - continue; - - uint32 fieldIndex = 0; - if (!_loadInfo->Meta->HasIndexFieldInData()) - { - offset += 4; - ++fieldIndex; - } - - std::size_t fieldOffset = 0; - for (uint32 x = 0; x < _header->FieldCount; ++x) - { - for (uint32 z = 0; z < _loadInfo->Meta->ArraySizes[x]; ++z) - { - switch (_loadInfo->TypesString[fieldIndex]) - { - case FT_FLOAT: - offset += 4; - fieldOffset += 4; - break; - case FT_INT: - offset += 4; - fieldOffset += 4 - fields[x].UnusedBits / 8; - break; - case FT_BYTE: - offset += 1; - fieldOffset += 1; - break; - case FT_SHORT: - offset += 2; - fieldOffset += 2; - break; - case FT_STRING: - { - LocalizedString* db2str = *(LocalizedString**)(&dataTable[offset]); - db2str->Str[locale] = stringPtr; - strcpy(stringPtr, (char*)&data[offsets[y].FileOffset - dataStart + fieldOffset]); - fieldOffset += strlen(stringPtr) + 1; - stringPtr += strlen(stringPtr) + 1; - offset += sizeof(char*); - break; - } - case FT_STRING_NOT_LOCALIZED: - { - fieldOffset += strlen((char*)&data[offsets[y].FileOffset - dataStart + fieldOffset]) + 1; - offset += sizeof(char*); - break; - } - default: - ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->TypesString[x], fileName); - break; - } - ++fieldIndex; - } - } - } - - return stringTable; -} - -char* DB2DatabaseLoader::Load(uint32& records, char**& indexTable, char*& stringHolders, std::vector<char*>& stringPool) -{ - // Even though this query is executed only once, prepared statement is used to send data from mysql server in binary format - PreparedQueryResult result = HotfixDatabase.Query(HotfixDatabase.GetPreparedStatement(_loadInfo->Statement)); - if (!result) - return nullptr; - - if (_loadInfo->Meta->GetDbFieldCount() != result->GetFieldCount()) - return nullptr; - - // get struct size and index pos - uint32 indexField = _loadInfo->Meta->GetDbIndexField(); - uint32 recordSize = _loadInfo->Meta->GetRecordSize(); - - // we store flat holders pool as single memory block - std::size_t stringFields = _loadInfo->GetStringFieldCount(false); - std::size_t localizedStringFields = _loadInfo->GetStringFieldCount(true); - - // each string field at load have array of string for each locale - std::size_t stringHoldersRecordPoolSize = localizedStringFields * sizeof(LocalizedString) + (stringFields - localizedStringFields) * sizeof(char*); - - if (stringFields) - { - std::size_t stringHoldersPoolSize = stringHoldersRecordPoolSize * result->GetRowCount(); - stringHolders = new char[stringHoldersPoolSize]; - - // DB2 strings expected to have at least empty string - for (std::size_t i = 0; i < stringHoldersPoolSize / sizeof(char*); ++i) - ((char const**)stringHolders)[i] = nullStr; - } - else - stringHolders = nullptr; - - // Resize index table - // database query *MUST* contain ORDER BY `index_field` DESC clause - uint32 indexTableSize = (*result)[indexField].GetUInt32() + 1; - if (indexTableSize < records) - indexTableSize = records; - - if (indexTableSize > records) - { - char** tmpIdxTable = new char*[indexTableSize]; - memset(tmpIdxTable, 0, indexTableSize * sizeof(char*)); - memcpy(tmpIdxTable, indexTable, records * sizeof(char*)); - delete[] indexTable; - indexTable = tmpIdxTable; - } - - char* tempDataTable = new char[result->GetRowCount() * recordSize]; - uint32* newIndexes = new uint32[result->GetRowCount()]; - uint32 rec = 0; - uint32 newRecords = 0; - - do - { - Field* fields = result->Fetch(); - uint32 offset = 0; - uint32 stringFieldOffset = 0; - - uint32 indexValue = fields[indexField].GetUInt32(); - - // Attempt to overwrite existing data - char* dataValue = indexTable[indexValue]; - if (!dataValue) - { - newIndexes[newRecords] = indexValue; - dataValue = &tempDataTable[newRecords++ * recordSize]; - } - - uint32 f = 0; - if (!_loadInfo->Meta->HasIndexFieldInData()) - { - *((uint32*)(&dataValue[offset])) = indexValue; - offset += 4; - ++f; - } - - for (uint32 x = 0; x < _loadInfo->Meta->FieldCount; ++x) - { - for (uint32 z = 0; z < _loadInfo->Meta->ArraySizes[x]; ++z) - { - switch (_loadInfo->TypesString[f]) - { - case FT_FLOAT: - *((float*)(&dataValue[offset])) = fields[f].GetFloat(); - offset += 4; - break; - case FT_INT: - *((int32*)(&dataValue[offset])) = fields[f].GetInt32(); - offset += 4; - break; - case FT_BYTE: - *((int8*)(&dataValue[offset])) = fields[f].GetInt8(); - offset += 1; - break; - case FT_SHORT: - *((int16*)(&dataValue[offset])) = fields[f].GetInt16(); - offset += 2; - break; - case FT_STRING: - { - LocalizedString** slot = (LocalizedString**)(&dataValue[offset]); - *slot = (LocalizedString*)(&stringHolders[stringHoldersRecordPoolSize * rec + stringFieldOffset]); - ASSERT(*slot); - - // Value in database in main table field must be for enUS locale - if (char* str = AddString(&(*slot)->Str[LOCALE_enUS], fields[f].GetString())) - stringPool.push_back(str); - - stringFieldOffset += sizeof(LocalizedString); - offset += sizeof(char*); - break; - } - case FT_STRING_NOT_LOCALIZED: - { - char const** slot = (char const**)(&dataValue[offset]); - *slot = (char*)(&stringHolders[stringHoldersRecordPoolSize * rec + stringFieldOffset]); - ASSERT(*slot); - - // Value in database in main table field must be for enUS locale - if (char* str = AddString(slot, fields[f].GetString())) - stringPool.push_back(str); - - stringFieldOffset += sizeof(char*); - offset += sizeof(char*); - break; - } - default: - ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->TypesString[x], _storageName.c_str()); - break; - } - ++f; - } - } - - ASSERT(offset == recordSize); - ++rec; - } while (result->NextRow()); - - if (!newRecords) - { - delete[] tempDataTable; - delete[] newIndexes; - return nullptr; - } - - // Compact new data table to only contain new records not previously loaded from file - char* dataTable = new char[newRecords * recordSize]; - memcpy(dataTable, tempDataTable, newRecords * recordSize); - - // insert new records to index table - for (uint32 i = 0; i < newRecords; ++i) - indexTable[newIndexes[i]] = &dataTable[i * recordSize]; - - delete[] tempDataTable; - delete[] newIndexes; - - records = indexTableSize; - - return dataTable; -} - -void DB2DatabaseLoader::LoadStrings(uint32 locale, uint32 records, char** indexTable, std::vector<char*>& stringPool) -{ - PreparedStatement* stmt = HotfixDatabase.GetPreparedStatement(HotfixDatabaseStatements(_loadInfo->Statement + 1)); - stmt->setString(0, localeNames[locale]); - PreparedQueryResult result = HotfixDatabase.Query(stmt); - if (!result) - return; - - std::size_t stringFields = _loadInfo->GetStringFieldCount(true); - if (result->GetFieldCount() != stringFields + 1 /*ID*/) - return; - - uint32 fieldCount = _loadInfo->Meta->FieldCount; - uint32 recordSize = _loadInfo->Meta->GetRecordSize(); - - do - { - Field* fields = result->Fetch(); - uint32 offset = 0; - uint32 stringFieldNumInRecord = 0; - uint32 indexValue = fields[0].GetUInt32(); - - if (indexValue >= records) - continue; - - // Attempt to overwrite existing data - if (char* dataValue = indexTable[indexValue]) - { - uint32 fieldIndex = 0; - if (!_loadInfo->Meta->HasIndexFieldInData()) - { - offset += 4; - ++fieldIndex; - } - - for (uint32 x = 0; x < fieldCount; ++x) - { - for (uint32 z = 0; z < _loadInfo->Meta->ArraySizes[x]; ++z) - { - switch (_loadInfo->TypesString[fieldIndex]) - { - case FT_FLOAT: - case FT_INT: - offset += 4; - break; - case FT_BYTE: - offset += 1; - break; - case FT_SHORT: - offset += 2; - break; - case FT_STRING: - { - // fill only not filled entries - LocalizedString* db2str = *(LocalizedString**)(&dataValue[offset]); - if (db2str->Str[locale] == nullStr) - if (char* str = AddString(&db2str->Str[locale], fields[1 + stringFieldNumInRecord].GetString())) - stringPool.push_back(str); - - ++stringFieldNumInRecord; - offset += sizeof(LocalizedString*); - break; - } - case FT_STRING_NOT_LOCALIZED: - offset += sizeof(char*); - break; - default: - ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->TypesString[x], _storageName.c_str()); - break; - } - ++fieldIndex; - } - } - - ASSERT(offset == recordSize); - } - else - TC_LOG_ERROR("sql.sql", "Hotfix locale table for storage %s references row that does not exist %u locale %s!", _storageName.c_str(), indexValue, localeNames[locale]); - - } while (result->NextRow()); - - return; -} - -char* DB2DatabaseLoader::AddString(char const** holder, std::string const& value) -{ - if (!value.empty()) - { - std::size_t existingLength = strlen(*holder); - if (existingLength >= value.length()) - { - // Reuse existing storage if there is enough space - char* str = const_cast<char*>(*holder); - memcpy(str, value.c_str(), value.length()); - str[value.length()] = '\0'; - return nullptr; - } - - char* str = new char[value.length() + 1]; - memcpy(str, value.c_str(), value.length()); - str[value.length()] = '\0'; - *holder = str; - return str; - } - - return nullptr; -} - -DB2FileLoader::DB2FileLoader() : _impl(nullptr) -{ -} - -DB2FileLoader::~DB2FileLoader() -{ - delete _impl; -} - -bool DB2FileLoader::Load(char const* filename, DB2LoadInfo const* loadInfo) -{ - FILE* f = fopen(filename, "rb"); - if (!f) - return false; - - if (fread(&_header.Signature, 4, 1, f) != 1) // Signature - { - fclose(f); - return false; - } - - EndianConvert(_header.Signature); - - if (_header.Signature != 0x35424457) - { - fclose(f); - return false; //'WDB5' - } - - if (fread(&_header.RecordCount, 4, 1, f) != 1) // Number of records - { - fclose(f); - return false; - } - - EndianConvert(_header.RecordCount); - - if (fread(&_header.FieldCount, 4, 1, f) != 1) // Number of fields - { - fclose(f); - return false; - } - - EndianConvert(_header.FieldCount); - - if (fread(&_header.RecordSize, 4, 1, f) != 1) // Size of a record - { - fclose(f); - return false; - } - - EndianConvert(_header.RecordSize); - - if (fread(&_header.StringTableSize, 4, 1, f) != 1) // String size - { - fclose(f); - return false; - } - - EndianConvert(_header.StringTableSize); - - if (fread(&_header.TableHash, 4, 1, f) != 1) // Table hash - { - fclose(f); - return false; - } - - EndianConvert(_header.TableHash); - - if (fread(&_header.LayoutHash, 4, 1, f) != 1) // Layout hash - { - fclose(f); - return false; - } - - EndianConvert(_header.LayoutHash); - - if (_header.LayoutHash != loadInfo->Meta->LayoutHash) - { - fclose(f); - return false; - } - - if (fread(&_header.MinId, 4, 1, f) != 1) // MinIndex WDB2 - { - fclose(f); - return false; - } - - EndianConvert(_header.MinId); - - if (fread(&_header.MaxId, 4, 1, f) != 1) // MaxIndex WDB2 - { - fclose(f); - return false; - } - - EndianConvert(_header.MaxId); - - if (fread(&_header.Locale, 4, 1, f) != 1) // Locales - { - fclose(f); - return false; - } - - EndianConvert(_header.Locale); - - if (fread(&_header.CopyTableSize, 4, 1, f) != 1) - { - fclose(f); - return false; - } - - EndianConvert(_header.CopyTableSize); - - if (fread(&_header.Flags, 4, 1, f) != 1) - { - fclose(f); - return false; - } - - EndianConvert(_header.Flags); - - if (!(_header.Flags & 0x1)) - _impl = new DB2FileLoaderRegularImpl(); - else - _impl = new DB2FileLoaderSparseImpl(); - - bool result = _impl->Load(filename, f, loadInfo, &_header); - fclose(f); - return result; -} - -char* DB2FileLoader::AutoProduceData(uint32& count, char**& indexTable, std::vector<char*>& stringPool) -{ - return _impl->AutoProduceData(count, indexTable, stringPool); -} - -char* DB2FileLoader::AutoProduceStrings(char* dataTable, uint32 locale) -{ - return _impl->AutoProduceStrings(dataTable, locale); -} - -void DB2FileLoader::AutoProduceRecordCopies(uint32 records, char** indexTable, char* dataTable) -{ - _impl->AutoProduceRecordCopies(records, indexTable, dataTable); -} diff --git a/src/server/shared/DataStores/DB2Store.cpp b/src/server/shared/DataStores/DB2Store.cpp new file mode 100644 index 00000000000..bb36ed6b61b --- /dev/null +++ b/src/server/shared/DataStores/DB2Store.cpp @@ -0,0 +1,189 @@ +/* + * Copyright (C) 2008-2017 TrinityCore <http://www.trinitycore.org/> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "DB2Store.h" +#include "ByteBuffer.h" +#include "DB2DatabaseLoader.h" +#include "DB2Meta.h" + +struct DB2FileSystemSource : public DB2FileSource +{ + DB2FileSystemSource(std::string const& fileName) + { + _fileName = fileName; + _file = fopen(_fileName.c_str(), "rb"); + } + + ~DB2FileSystemSource() + { + if (_file) + fclose(_file); + } + + bool IsOpen() const override + { + return _file != nullptr; + } + + bool Read(void* buffer, std::size_t numBytes) override + { + return fread(buffer, numBytes, 1, _file) == 1; + } + + std::size_t GetPosition() const override + { + return ftell(_file); + } + + char const* GetFileName() const override + { + return _fileName.c_str(); + } + +private: + std::string _fileName; + FILE* _file; +}; + +DB2StorageBase::DB2StorageBase(char const* fileName, DB2LoadInfo const* loadInfo) + : _tableHash(0), _layoutHash(0), _fileName(fileName), _fieldCount(0), _loadInfo(loadInfo), _dataTable(nullptr), _dataTableEx(nullptr), _indexTableSize(0) +{ +} + +DB2StorageBase::~DB2StorageBase() +{ + delete[] _dataTable; + delete[] _dataTableEx; + for (char* strings : _stringPool) + delete[] strings; +} + +void DB2StorageBase::WriteRecordData(char const* entry, uint32 locale, ByteBuffer& buffer) const +{ + std::size_t i = 0; + if (!_loadInfo->Meta->HasIndexFieldInData()) + { + entry += 4; + ++i; + } + + for (; i < _loadInfo->FieldCount; ++i) + { + switch (_loadInfo->TypesString[i]) + { + case FT_INT: + buffer << *(uint32*)entry; + entry += 4; + break; + case FT_FLOAT: + buffer << *(float*)entry; + entry += 4; + break; + case FT_BYTE: + buffer << *(uint8*)entry; + entry += 1; + break; + case FT_SHORT: + buffer << *(uint16*)entry; + entry += 2; + break; + case FT_STRING: + { + LocalizedString* locStr = *(LocalizedString**)entry; + if (locStr->Str[locale][0] == '\0') + locale = 0; + + buffer << locStr->Str[locale]; + entry += sizeof(LocalizedString*); + break; + } + case FT_STRING_NOT_LOCALIZED: + { + buffer << *(char const**)entry; + entry += sizeof(char const*); + break; + } + } + } +} + +bool DB2StorageBase::Load(std::string const& path, uint32 locale, char**& indexTable) +{ + indexTable = nullptr; + DB2FileLoader db2; + { + DB2FileSystemSource source(path + _fileName); + // Check if load was successful, only then continue + if (!db2.Load(&source, _loadInfo)) + return false; + } + + _fieldCount = db2.GetCols(); + _tableHash = db2.GetTableHash(); + _layoutHash = db2.GetLayoutHash(); + + // load raw non-string data + _dataTable = db2.AutoProduceData(_indexTableSize, indexTable, _stringPool); + + // load strings from db2 data + if (!_stringPool.empty()) + if (char* stringBlock = db2.AutoProduceStrings(_dataTable, locale)) + _stringPool.push_back(stringBlock); + + db2.AutoProduceRecordCopies(_indexTableSize, indexTable, _dataTable); + + // error in db2 file at loading if NULL + return indexTable != nullptr; +} + +bool DB2StorageBase::LoadStringsFrom(std::string const& path, uint32 locale, char** indexTable) +{ + // DB2 must be already loaded using Load + if (!indexTable) + return false; + + DB2FileLoader db2; + { + DB2FileSystemSource source(path + _fileName); + // Check if load was successful, only then continue + if (!db2.Load(&source, _loadInfo)) + return false; + } + + // load strings from another locale db2 data + if (_loadInfo->GetStringFieldCount(true)) + if (char* stringBlock = db2.AutoProduceStrings(_dataTable, locale)) + _stringPool.push_back(stringBlock); + + return true; +} + +void DB2StorageBase::LoadFromDB(char**& indexTable) +{ + char* extraStringHolders = nullptr; + _dataTableEx = DB2DatabaseLoader(_fileName, _loadInfo).Load(_indexTableSize, indexTable, extraStringHolders, _stringPool); + if (extraStringHolders) + _stringPool.push_back(extraStringHolders); +} + +void DB2StorageBase::LoadStringsFromDB(uint32 locale, char** indexTable) +{ + if (!_loadInfo->GetStringFieldCount(true)) + return; + + DB2DatabaseLoader(_fileName, _loadInfo).LoadStrings(locale, _indexTableSize, indexTable, _stringPool); +} diff --git a/src/server/shared/DataStores/DB2Store.h b/src/server/shared/DataStores/DB2Store.h index 7011a3b6e61..68cd888cd05 100644 --- a/src/server/shared/DataStores/DB2Store.h +++ b/src/server/shared/DataStores/DB2Store.h @@ -19,41 +19,29 @@ #define DB2STORE_H #include "Common.h" -#include "DB2Meta.h" -#include "DB2StorageLoader.h" #include "DBStorageIterator.h" -#include "ByteBuffer.h" +#include <vector> + +class ByteBuffer; +struct DB2LoadInfo; /// Interface class for common access -class DB2StorageBase +class TC_SHARED_API DB2StorageBase { public: - DB2StorageBase(char const* fileName, DB2LoadInfo&& loadInfo) - : _tableHash(0), _layoutHash(0), _fileName(fileName), _fieldCount(0), _loadInfo(std::move(loadInfo)), _dataTable(nullptr), _dataTableEx(nullptr) { } - - virtual ~DB2StorageBase() - { - delete[] _dataTable; - delete[] _dataTableEx; - for (char* strings : _stringPool) - delete[] strings; - } + DB2StorageBase(char const* fileName, DB2LoadInfo const* loadInfo); + virtual ~DB2StorageBase(); uint32 GetTableHash() const { return _tableHash; } - uint32 GetLayoutHash() const { return _layoutHash; } virtual bool HasRecord(uint32 id) const = 0; - virtual void WriteRecord(uint32 id, uint32 locale, ByteBuffer& buffer) const = 0; - virtual void EraseRecord(uint32 id) = 0; std::string const& GetFileName() const { return _fileName; } - uint32 GetFieldCount() const { return _fieldCount; } - - DB2LoadInfo const* GetLoadInfo() const { return &_loadInfo; } + DB2LoadInfo const* GetLoadInfo() const { return _loadInfo; } virtual bool Load(std::string const& path, uint32 locale) = 0; virtual bool LoadStringsFrom(std::string const& path, uint32 locale) = 0; @@ -61,63 +49,21 @@ public: virtual void LoadStringsFromDB(uint32 locale) = 0; protected: - void WriteRecordData(char const* entry, uint32 locale, ByteBuffer& buffer) const - { - std::size_t i = 0; - if (!_loadInfo.Meta->HasIndexFieldInData()) - { - entry += 4; - ++i; - } - - for (; i < _loadInfo.FieldCount; ++i) - { - switch (_loadInfo.TypesString[i]) - { - case FT_INT: - buffer << *(uint32*)entry; - entry += 4; - break; - case FT_FLOAT: - buffer << *(float*)entry; - entry += 4; - break; - case FT_BYTE: - buffer << *(uint8*)entry; - entry += 1; - break; - case FT_SHORT: - buffer << *(uint16*)entry; - entry += 2; - break; - case FT_STRING: - { - LocalizedString* locStr = *(LocalizedString**)entry; - if (locStr->Str[locale][0] == '\0') - locale = 0; - - buffer << locStr->Str[locale]; - entry += sizeof(LocalizedString*); - break; - } - case FT_STRING_NOT_LOCALIZED: - { - buffer << *(char const**)entry; - entry += sizeof(char const*); - break; - } - } - } - } + void WriteRecordData(char const* entry, uint32 locale, ByteBuffer& buffer) const; + bool Load(std::string const& path, uint32 locale, char**& indexTable); + bool LoadStringsFrom(std::string const& path, uint32 locale, char** indexTable); + void LoadFromDB(char**& indexTable); + void LoadStringsFromDB(uint32 locale, char** indexTable); uint32 _tableHash; uint32 _layoutHash; std::string _fileName; uint32 _fieldCount; - DB2LoadInfo _loadInfo; + DB2LoadInfo const* _loadInfo; char* _dataTable; char* _dataTableEx; std::vector<char*> _stringPool; + uint32 _indexTableSize; }; template<class T> @@ -128,10 +74,9 @@ class DB2Storage : public DB2StorageBase public: typedef DBStorageIterator<T> iterator; - DB2Storage(char const* fileName, DB2LoadInfo&& loadInfo) : DB2StorageBase(fileName, std::move(loadInfo)), - _indexTableSize(0) + DB2Storage(char const* fileName, DB2LoadInfo const* loadInfo) : DB2StorageBase(fileName, loadInfo) { - _indexTable.AsT = NULL; + _indexTable.AsT = nullptr; } ~DB2Storage() @@ -153,61 +98,22 @@ public: uint32 GetNumRows() const { return _indexTableSize; } bool Load(std::string const& path, uint32 locale) override { - DB2FileLoader db2; - // Check if load was successful, only then continue - if (!db2.Load((path + _fileName).c_str(), &_loadInfo)) - return false; - - _fieldCount = db2.GetCols(); - _tableHash = db2.GetTableHash(); - _layoutHash = db2.GetLayoutHash(); - - // load raw non-string data - _dataTable = db2.AutoProduceData(_indexTableSize, _indexTable.AsChar, _stringPool); - - // load strings from db2 data - if (!_stringPool.empty()) - if (char* stringBlock = db2.AutoProduceStrings(_dataTable, locale)) - _stringPool.push_back(stringBlock); - - db2.AutoProduceRecordCopies(_indexTableSize, _indexTable.AsChar, _dataTable); - - // error in db2 file at loading if NULL - return _indexTable.AsT != NULL; + return DB2StorageBase::Load(path, locale, _indexTable.AsChar); } bool LoadStringsFrom(std::string const& path, uint32 locale) override { - // DB2 must be already loaded using Load - if (!_indexTable.AsT) - return false; - - DB2FileLoader db2; - // Check if load was successful, only then continue - if (!db2.Load((path + _fileName).c_str(), &_loadInfo)) - return false; - - // load strings from another locale db2 data - if (_loadInfo.GetStringFieldCount(true)) - if (char* stringBlock = db2.AutoProduceStrings(_dataTable, locale)) - _stringPool.push_back(stringBlock); - return true; + return DB2StorageBase::LoadStringsFrom(path, locale, _indexTable.AsChar); } void LoadFromDB() override { - char* extraStringHolders = nullptr; - _dataTableEx = DB2DatabaseLoader(_fileName, &_loadInfo).Load(_indexTableSize, _indexTable.AsChar, extraStringHolders, _stringPool); - if (extraStringHolders) - _stringPool.push_back(extraStringHolders); + DB2StorageBase::LoadFromDB(_indexTable.AsChar); } void LoadStringsFromDB(uint32 locale) override { - if (!_loadInfo.GetStringFieldCount(true)) - return; - - DB2DatabaseLoader(_fileName, &_loadInfo).LoadStrings(locale, _indexTableSize, _indexTable.AsChar, _stringPool); + DB2StorageBase::LoadStringsFromDB(locale, _indexTable.AsChar); } iterator begin() { return iterator(_indexTable.AsT, _indexTableSize); } @@ -219,7 +125,6 @@ private: T** AsT; char** AsChar; } _indexTable; - uint32 _indexTableSize; }; #endif |
