aboutsummaryrefslogtreecommitdiff
path: root/src/server/shared/DataStores
diff options
context:
space:
mode:
authorShauren <shauren.trinity@gmail.com>2017-03-08 18:10:02 +0100
committerShauren <shauren.trinity@gmail.com>2017-03-08 18:10:02 +0100
commit7b235ce6e4da0e9c19fa9c6306bc7a71c7fb905d (patch)
tree28611e02e44e55d28f60a9b1b3433831ed6772b8 /src/server/shared/DataStores
parentf585c831248f24c93697b0c314dd015897febe39 (diff)
Core/DataStores: Refactor DB2 loaders to be reusable by extractors
Diffstat (limited to 'src/server/shared/DataStores')
-rw-r--r--src/server/shared/DataStores/DB2DatabaseLoader.cpp301
-rw-r--r--src/server/shared/DataStores/DB2DatabaseLoader.h (renamed from src/server/shared/DataStores/DB2StorageLoader.h)57
-rw-r--r--src/server/shared/DataStores/DB2StorageLoader.cpp1257
-rw-r--r--src/server/shared/DataStores/DB2Store.cpp189
-rw-r--r--src/server/shared/DataStores/DB2Store.h137
5 files changed, 515 insertions, 1426 deletions
diff --git a/src/server/shared/DataStores/DB2DatabaseLoader.cpp b/src/server/shared/DataStores/DB2DatabaseLoader.cpp
new file mode 100644
index 00000000000..a19b8a5b389
--- /dev/null
+++ b/src/server/shared/DataStores/DB2DatabaseLoader.cpp
@@ -0,0 +1,301 @@
+/*
+ * Copyright (C) 2008-2017 TrinityCore <http://www.trinitycore.org/>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "DB2DatabaseLoader.h"
+#include "Database/DatabaseEnv.h"
+#include "DB2Meta.h"
+
+DB2LoadInfo::DB2LoadInfo() : DB2FileLoadInfo()
+{
+}
+
+DB2LoadInfo::DB2LoadInfo(DB2FieldMeta const* fields, std::size_t fieldCount, DB2Meta const* meta, HotfixDatabaseStatements statement)
+ : DB2FileLoadInfo(fields, fieldCount, meta), Statement(statement)
+{
+}
+
+static char const* nullStr = "";
+
+char* DB2DatabaseLoader::Load(uint32& records, char**& indexTable, char*& stringHolders, std::vector<char*>& stringPool)
+{
+ // Even though this query is executed only once, prepared statement is used to send data from mysql server in binary format
+ PreparedQueryResult result = HotfixDatabase.Query(HotfixDatabase.GetPreparedStatement(_loadInfo->Statement));
+ if (!result)
+ return nullptr;
+
+ if (_loadInfo->Meta->GetDbFieldCount() != result->GetFieldCount())
+ return nullptr;
+
+ // get struct size and index pos
+ uint32 indexField = _loadInfo->Meta->GetDbIndexField();
+ uint32 recordSize = _loadInfo->Meta->GetRecordSize();
+
+ // we store flat holders pool as single memory block
+ std::size_t stringFields = _loadInfo->GetStringFieldCount(false);
+ std::size_t localizedStringFields = _loadInfo->GetStringFieldCount(true);
+
+ // each string field at load have array of string for each locale
+ std::size_t stringHoldersRecordPoolSize = localizedStringFields * sizeof(LocalizedString) + (stringFields - localizedStringFields) * sizeof(char*);
+
+ if (stringFields)
+ {
+ std::size_t stringHoldersPoolSize = stringHoldersRecordPoolSize * result->GetRowCount();
+ stringHolders = new char[stringHoldersPoolSize];
+
+ // DB2 strings expected to have at least empty string
+ for (std::size_t i = 0; i < stringHoldersPoolSize / sizeof(char*); ++i)
+ ((char const**)stringHolders)[i] = nullStr;
+ }
+ else
+ stringHolders = nullptr;
+
+ // Resize index table
+ // database query *MUST* contain ORDER BY `index_field` DESC clause
+ uint32 indexTableSize = (*result)[indexField].GetUInt32() + 1;
+ if (indexTableSize < records)
+ indexTableSize = records;
+
+ if (indexTableSize > records)
+ {
+ char** tmpIdxTable = new char*[indexTableSize];
+ memset(tmpIdxTable, 0, indexTableSize * sizeof(char*));
+ memcpy(tmpIdxTable, indexTable, records * sizeof(char*));
+ delete[] indexTable;
+ indexTable = tmpIdxTable;
+ }
+
+ char* tempDataTable = new char[result->GetRowCount() * recordSize];
+ uint32* newIndexes = new uint32[result->GetRowCount()];
+ uint32 rec = 0;
+ uint32 newRecords = 0;
+
+ do
+ {
+ Field* fields = result->Fetch();
+ uint32 offset = 0;
+ uint32 stringFieldOffset = 0;
+
+ uint32 indexValue = fields[indexField].GetUInt32();
+
+ // Attempt to overwrite existing data
+ char* dataValue = indexTable[indexValue];
+ if (!dataValue)
+ {
+ newIndexes[newRecords] = indexValue;
+ dataValue = &tempDataTable[newRecords++ * recordSize];
+ }
+
+ uint32 f = 0;
+ if (!_loadInfo->Meta->HasIndexFieldInData())
+ {
+ *((uint32*)(&dataValue[offset])) = indexValue;
+ offset += 4;
+ ++f;
+ }
+
+ for (uint32 x = 0; x < _loadInfo->Meta->FieldCount; ++x)
+ {
+ for (uint32 z = 0; z < _loadInfo->Meta->ArraySizes[x]; ++z)
+ {
+ switch (_loadInfo->TypesString[f])
+ {
+ case FT_FLOAT:
+ *((float*)(&dataValue[offset])) = fields[f].GetFloat();
+ offset += 4;
+ break;
+ case FT_INT:
+ *((int32*)(&dataValue[offset])) = fields[f].GetInt32();
+ offset += 4;
+ break;
+ case FT_BYTE:
+ *((int8*)(&dataValue[offset])) = fields[f].GetInt8();
+ offset += 1;
+ break;
+ case FT_SHORT:
+ *((int16*)(&dataValue[offset])) = fields[f].GetInt16();
+ offset += 2;
+ break;
+ case FT_STRING:
+ {
+ LocalizedString** slot = (LocalizedString**)(&dataValue[offset]);
+ *slot = (LocalizedString*)(&stringHolders[stringHoldersRecordPoolSize * rec + stringFieldOffset]);
+ ASSERT(*slot);
+
+ // Value in database in main table field must be for enUS locale
+ if (char* str = AddString(&(*slot)->Str[LOCALE_enUS], fields[f].GetString()))
+ stringPool.push_back(str);
+
+ stringFieldOffset += sizeof(LocalizedString);
+ offset += sizeof(char*);
+ break;
+ }
+ case FT_STRING_NOT_LOCALIZED:
+ {
+ char const** slot = (char const**)(&dataValue[offset]);
+ *slot = (char*)(&stringHolders[stringHoldersRecordPoolSize * rec + stringFieldOffset]);
+ ASSERT(*slot);
+
+ // Value in database in main table field must be for enUS locale
+ if (char* str = AddString(slot, fields[f].GetString()))
+ stringPool.push_back(str);
+
+ stringFieldOffset += sizeof(char*);
+ offset += sizeof(char*);
+ break;
+ }
+ default:
+ ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->TypesString[x], _storageName.c_str());
+ break;
+ }
+ ++f;
+ }
+ }
+
+ ASSERT(offset == recordSize);
+ ++rec;
+ } while (result->NextRow());
+
+ if (!newRecords)
+ {
+ delete[] tempDataTable;
+ delete[] newIndexes;
+ return nullptr;
+ }
+
+ // Compact new data table to only contain new records not previously loaded from file
+ char* dataTable = new char[newRecords * recordSize];
+ memcpy(dataTable, tempDataTable, newRecords * recordSize);
+
+ // insert new records to index table
+ for (uint32 i = 0; i < newRecords; ++i)
+ indexTable[newIndexes[i]] = &dataTable[i * recordSize];
+
+ delete[] tempDataTable;
+ delete[] newIndexes;
+
+ records = indexTableSize;
+
+ return dataTable;
+}
+
+void DB2DatabaseLoader::LoadStrings(uint32 locale, uint32 records, char** indexTable, std::vector<char*>& stringPool)
+{
+ PreparedStatement* stmt = HotfixDatabase.GetPreparedStatement(HotfixDatabaseStatements(_loadInfo->Statement + 1));
+ stmt->setString(0, localeNames[locale]);
+ PreparedQueryResult result = HotfixDatabase.Query(stmt);
+ if (!result)
+ return;
+
+ std::size_t stringFields = _loadInfo->GetStringFieldCount(true);
+ if (result->GetFieldCount() != stringFields + 1 /*ID*/)
+ return;
+
+ uint32 fieldCount = _loadInfo->Meta->FieldCount;
+ uint32 recordSize = _loadInfo->Meta->GetRecordSize();
+
+ do
+ {
+ Field* fields = result->Fetch();
+ uint32 offset = 0;
+ uint32 stringFieldNumInRecord = 0;
+ uint32 indexValue = fields[0].GetUInt32();
+
+ if (indexValue >= records)
+ continue;
+
+ // Attempt to overwrite existing data
+ if (char* dataValue = indexTable[indexValue])
+ {
+ uint32 fieldIndex = 0;
+ if (!_loadInfo->Meta->HasIndexFieldInData())
+ {
+ offset += 4;
+ ++fieldIndex;
+ }
+
+ for (uint32 x = 0; x < fieldCount; ++x)
+ {
+ for (uint32 z = 0; z < _loadInfo->Meta->ArraySizes[x]; ++z)
+ {
+ switch (_loadInfo->TypesString[fieldIndex])
+ {
+ case FT_FLOAT:
+ case FT_INT:
+ offset += 4;
+ break;
+ case FT_BYTE:
+ offset += 1;
+ break;
+ case FT_SHORT:
+ offset += 2;
+ break;
+ case FT_STRING:
+ {
+ // fill only not filled entries
+ LocalizedString* db2str = *(LocalizedString**)(&dataValue[offset]);
+ if (db2str->Str[locale] == nullStr)
+ if (char* str = AddString(&db2str->Str[locale], fields[1 + stringFieldNumInRecord].GetString()))
+ stringPool.push_back(str);
+
+ ++stringFieldNumInRecord;
+ offset += sizeof(LocalizedString*);
+ break;
+ }
+ case FT_STRING_NOT_LOCALIZED:
+ offset += sizeof(char*);
+ break;
+ default:
+ ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->TypesString[x], _storageName.c_str());
+ break;
+ }
+ ++fieldIndex;
+ }
+ }
+
+ ASSERT(offset == recordSize);
+ }
+ else
+ TC_LOG_ERROR("sql.sql", "Hotfix locale table for storage %s references row that does not exist %u locale %s!", _storageName.c_str(), indexValue, localeNames[locale]);
+
+ } while (result->NextRow());
+
+ return;
+}
+
+char* DB2DatabaseLoader::AddString(char const** holder, std::string const& value)
+{
+ if (!value.empty())
+ {
+ std::size_t existingLength = strlen(*holder);
+ if (existingLength >= value.length())
+ {
+ // Reuse existing storage if there is enough space
+ char* str = const_cast<char*>(*holder);
+ memcpy(str, value.c_str(), value.length());
+ str[value.length()] = '\0';
+ return nullptr;
+ }
+
+ char* str = new char[value.length() + 1];
+ memcpy(str, value.c_str(), value.length());
+ str[value.length()] = '\0';
+ *holder = str;
+ return str;
+ }
+
+ return nullptr;
+}
diff --git a/src/server/shared/DataStores/DB2StorageLoader.h b/src/server/shared/DataStores/DB2DatabaseLoader.h
index 62361edcedd..f288b46859b 100644
--- a/src/server/shared/DataStores/DB2StorageLoader.h
+++ b/src/server/shared/DataStores/DB2DatabaseLoader.h
@@ -15,67 +15,18 @@
* with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-#ifndef DB2_FILE_LOADER_H
-#define DB2_FILE_LOADER_H
+#ifndef DB2_DATABASE_LOADER_H
+#define DB2_DATABASE_LOADER_H
-#include "Define.h"
+#include "DB2FileLoader.h"
#include "Implementation/HotfixDatabase.h"
-class DB2FileLoaderImpl;
-struct DB2Meta;
-struct DB2FieldMeta;
-
-#pragma pack(push, 1)
-struct DB2Header
-{
- uint32 Signature;
- uint32 RecordCount;
- uint32 FieldCount;
- uint32 RecordSize;
- uint32 StringTableSize;
- uint32 TableHash;
- uint32 LayoutHash;
- uint32 MinId;
- uint32 MaxId;
- uint32 Locale;
- uint32 CopyTableSize;
- uint16 Flags;
- int16 IndexField;
-};
-#pragma pack(pop)
-
-struct TC_SHARED_API DB2LoadInfo
+struct TC_SHARED_API DB2LoadInfo : public DB2FileLoadInfo
{
DB2LoadInfo();
DB2LoadInfo(DB2FieldMeta const* fields, std::size_t fieldCount, DB2Meta const* meta, HotfixDatabaseStatements statement);
- uint32 GetStringFieldCount(bool localizedOnly) const;
-
- DB2FieldMeta const* Fields;
- std::size_t FieldCount;
- DB2Meta const* Meta;
HotfixDatabaseStatements Statement;
- std::string TypesString;
-};
-
-class TC_SHARED_API DB2FileLoader
-{
-public:
- DB2FileLoader();
- ~DB2FileLoader();
-
- bool Load(char const* filename, DB2LoadInfo const* loadInfo);
- char* AutoProduceData(uint32& count, char**& indexTable, std::vector<char*>& stringPool);
- char* AutoProduceStrings(char* dataTable, uint32 locale);
- void AutoProduceRecordCopies(uint32 records, char** indexTable, char* dataTable);
-
- uint32 GetCols() const { return _header.FieldCount; }
- uint32 GetTableHash() const { return _header.TableHash; }
- uint32 GetLayoutHash() const { return _header.LayoutHash; }
-
-private:
- DB2FileLoaderImpl* _impl;
- DB2Header _header;
};
class TC_SHARED_API DB2DatabaseLoader
diff --git a/src/server/shared/DataStores/DB2StorageLoader.cpp b/src/server/shared/DataStores/DB2StorageLoader.cpp
deleted file mode 100644
index 11caca97afb..00000000000
--- a/src/server/shared/DataStores/DB2StorageLoader.cpp
+++ /dev/null
@@ -1,1257 +0,0 @@
-/*
- * Copyright (C) 2008-2017 TrinityCore <http://www.trinitycore.org/>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation; either version 2 of the License, or (at your
- * option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "DB2StorageLoader.h"
-#include "Database/DatabaseEnv.h"
-#include "DB2Meta.h"
-#include "Utilities/ByteConverter.h"
-
-DB2LoadInfo::DB2LoadInfo() : Fields(nullptr), FieldCount(0), Meta(nullptr), Statement(MAX_HOTFIXDATABASE_STATEMENTS)
-{
-}
-
-DB2LoadInfo::DB2LoadInfo(DB2FieldMeta const* fields, std::size_t fieldCount, DB2Meta const* meta, HotfixDatabaseStatements statement)
- : Fields(fields), FieldCount(fieldCount), Meta(meta), Statement(statement)
-{
- TypesString.reserve(FieldCount);
- for (std::size_t i = 0; i < FieldCount; ++i)
- TypesString += char(Fields[i].Type);
-}
-
-uint32 DB2LoadInfo::GetStringFieldCount(bool localizedOnly) const
-{
- uint32 stringFields = 0;
- for (std::size_t i = 0; i < TypesString.length(); ++i)
- if (TypesString[i] == FT_STRING || (TypesString[i] == FT_STRING_NOT_LOCALIZED && !localizedOnly))
- ++stringFields;
-
- return stringFields;
-}
-
-class DB2FileLoaderImpl
-{
-public:
- virtual ~DB2FileLoaderImpl() { }
- virtual bool Load(char const* filename, FILE* file, DB2LoadInfo const* loadInfo, DB2Header const* header) = 0;
- virtual char* AutoProduceData(uint32& count, char**& indexTable, std::vector<char*>& stringPool) = 0;
- virtual char* AutoProduceStrings(char* dataTable, uint32 locale) = 0;
- virtual void AutoProduceRecordCopies(uint32 records, char** indexTable, char* dataTable) = 0;
-};
-
-class DB2FileLoaderRegularImpl : public DB2FileLoaderImpl
-{
-public:
- DB2FileLoaderRegularImpl();
- ~DB2FileLoaderRegularImpl();
-
- bool Load(char const* filename, FILE* file, DB2LoadInfo const* loadInfo, DB2Header const* header) override;
- char* AutoProduceData(uint32& count, char**& indexTable, std::vector<char*>& stringPool) override;
- char* AutoProduceStrings(char* dataTable, uint32 locale) override;
- void AutoProduceRecordCopies(uint32 records, char** indexTable, char* dataTable) override;
-
-private:
- class Record
- {
- public:
- float getFloat(uint32 field, uint32 arrayIndex) const;
- uint32 getUInt(uint32 field, uint32 arrayIndex, bool isSigned) const;
- uint8 getUInt8(uint32 field, uint32 arrayIndex) const;
- uint16 getUInt16(uint32 field, uint32 arrayIndex) const;
- char const* getString(uint32 field, uint32 arrayIndex) const;
-
- private:
- uint16 GetOffset(uint32 field) const;
- uint16 GetByteSize(uint32 field) const;
- uint32 GetVarInt(uint32 field, uint32 arrayIndex, bool isSigned) const;
-
- Record(DB2FileLoaderRegularImpl &file_, unsigned char *offset_);
- unsigned char *offset;
- DB2FileLoaderRegularImpl &file;
-
- friend class DB2FileLoaderRegularImpl;
- };
-
- // Get record by id
- Record getRecord(size_t id);
-
-#pragma pack(push, 1)
- struct FieldEntry
- {
- uint16 UnusedBits;
- uint16 Offset;
- };
-#pragma pack(pop)
-
- char const* fileName;
- DB2LoadInfo const* _loadInfo;
- DB2Header const* _header;
-
- unsigned char* data;
- unsigned char* stringTable;
- unsigned char* idTable;
- uint32 idTableSize;
- unsigned char* copyTable;
- FieldEntry* fields;
-};
-
-class DB2FileLoaderSparseImpl : public DB2FileLoaderImpl
-{
-public:
- DB2FileLoaderSparseImpl();
- ~DB2FileLoaderSparseImpl();
-
- bool Load(char const* filename, FILE* file, DB2LoadInfo const* loadInfo, DB2Header const* header) override;
- char* AutoProduceData(uint32& records, char**& indexTable, std::vector<char*>& stringPool) override;
- char* AutoProduceStrings(char* dataTable, uint32 locale) override;
- void AutoProduceRecordCopies(uint32 /*records*/, char** /*indexTable*/, char* /*dataTable*/) override { }
-
-private:
-#pragma pack(push, 1)
- struct OffsetTableEntry
- {
- uint32 FileOffset;
- uint16 RecordSize;
- };
- struct FieldEntry
- {
- uint16 UnusedBits;
- uint16 Offset;
- };
-#pragma pack(pop)
-
- char const* fileName;
- DB2LoadInfo const* _loadInfo;
- DB2Header const* _header;
- FieldEntry* fields;
-
- uint32 dataStart;
- unsigned char* data;
- OffsetTableEntry* offsets;
-};
-
-float DB2FileLoaderRegularImpl::Record::getFloat(uint32 field, uint32 arrayIndex) const
-{
- ASSERT(field < file._header->FieldCount);
- float val = *reinterpret_cast<float*>(offset + GetOffset(field) + arrayIndex * sizeof(float));
- EndianConvert(val);
- return val;
-}
-
-uint32 DB2FileLoaderRegularImpl::Record::getUInt(uint32 field, uint32 arrayIndex, bool isSigned) const
-{
- ASSERT(field < file._header->FieldCount);
- return GetVarInt(field, arrayIndex, isSigned);
-}
-
-uint8 DB2FileLoaderRegularImpl::Record::getUInt8(uint32 field, uint32 arrayIndex) const
-{
- ASSERT(field < file._header->FieldCount);
- ASSERT(GetByteSize(field) == 1);
- return *reinterpret_cast<uint8*>(offset + GetOffset(field) + arrayIndex * sizeof(uint8));
-}
-
-uint16 DB2FileLoaderRegularImpl::Record::getUInt16(uint32 field, uint32 arrayIndex) const
-{
- ASSERT(field < file._header->FieldCount);
- ASSERT(GetByteSize(field) == 2);
- uint16 val = *reinterpret_cast<uint16*>(offset + GetOffset(field) + arrayIndex * sizeof(uint16));
- EndianConvert(val);
- return val;
-}
-
-char const* DB2FileLoaderRegularImpl::Record::getString(uint32 field, uint32 arrayIndex) const
-{
- ASSERT(field < file._header->FieldCount);
- uint32 stringOffset = *reinterpret_cast<uint32*>(offset + GetOffset(field) + arrayIndex * sizeof(uint32));
- EndianConvert(stringOffset);
- ASSERT(stringOffset < file._header->StringTableSize);
- return reinterpret_cast<char*>(file.stringTable + stringOffset);
-}
-
-uint16 DB2FileLoaderRegularImpl::Record::GetOffset(uint32 field) const
-{
- ASSERT(field < file._header->FieldCount);
- return file.fields[field].Offset;
-}
-
-uint16 DB2FileLoaderRegularImpl::Record::GetByteSize(uint32 field) const
-{
- ASSERT(field < file._header->FieldCount);
- return 4 - file.fields[field].UnusedBits / 8;
-}
-
-uint32 DB2FileLoaderRegularImpl::Record::GetVarInt(uint32 field, uint32 arrayIndex, bool isSigned) const
-{
- ASSERT(field < file._header->FieldCount);
- uint32 val = *reinterpret_cast<uint32*>(offset + GetOffset(field) + arrayIndex * (4 - file.fields[field].UnusedBits / 8));
- EndianConvert(val);
- if (isSigned)
- return int32(val) << file.fields[field].UnusedBits >> file.fields[field].UnusedBits;
-
- return val << file.fields[field].UnusedBits >> file.fields[field].UnusedBits;
-}
-
-DB2FileLoaderRegularImpl::Record::Record(DB2FileLoaderRegularImpl &file_, unsigned char *offset_) : offset(offset_), file(file_)
-{
-}
-
-DB2FileLoaderRegularImpl::DB2FileLoaderRegularImpl()
-{
- fileName = nullptr;
- _loadInfo = nullptr;
- _header = nullptr;
- data = nullptr;
- stringTable = nullptr;
- idTable = nullptr;
- idTableSize = 0;
- copyTable = nullptr;
- fields = nullptr;
-}
-
-bool DB2FileLoaderRegularImpl::Load(char const* filename, FILE* file, DB2LoadInfo const* loadInfo, DB2Header const* header)
-{
- _loadInfo = loadInfo;
- _header = header;
- ASSERT(_loadInfo->Meta->IndexField == -1 || _loadInfo->Meta->IndexField == int32(header->IndexField));
-
- fileName = filename;
- fields = new FieldEntry[header->FieldCount];
- if (fread(fields, header->FieldCount * sizeof(FieldEntry), 1, file) != 1)
- return false;
-
- if (!_loadInfo->Meta->HasIndexFieldInData())
- idTableSize = header->RecordCount * sizeof(uint32);
-
- data = new unsigned char[header->RecordSize * header->RecordCount + header->StringTableSize];
- stringTable = data + header->RecordSize * header->RecordCount;
-
- if (fread(data, header->RecordSize * header->RecordCount + header->StringTableSize, 1, file) != 1)
- return false;
-
- if (idTableSize)
- {
- idTable = new unsigned char[idTableSize];
- if (fread(idTable, idTableSize, 1, file) != 1)
- return false;
- }
-
- if (header->CopyTableSize)
- {
- copyTable = new unsigned char[header->CopyTableSize];
- if (fread(copyTable, header->CopyTableSize, 1, file) != 1)
- return false;
- }
-
- return true;
-}
-
-DB2FileLoaderRegularImpl::~DB2FileLoaderRegularImpl()
-{
- delete[] data;
- delete[] idTable;
- delete[] copyTable;
- delete[] fields;
-}
-
-DB2FileLoaderRegularImpl::Record DB2FileLoaderRegularImpl::getRecord(size_t id)
-{
- assert(data);
- return Record(*this, data + id * _header->RecordSize);
-}
-
-static char const* const nullStr = "";
-
-char* DB2FileLoaderRegularImpl::AutoProduceData(uint32& records, char**& indexTable, std::vector<char*>& stringPool)
-{
- if (_loadInfo->Meta->FieldCount != _header->FieldCount)
- return nullptr;
-
- //get struct size and index pos
- uint32 indexField = _loadInfo->Meta->GetIndexField();
- uint32 recordsize = _loadInfo->Meta->GetRecordSize();
-
- uint32 maxi = 0;
- //find max index
- if (!idTableSize)
- {
- for (uint32 y = 0; y < _header->RecordCount; ++y)
- {
- uint32 ind = getRecord(y).getUInt(indexField, 0, false);
- if (ind > maxi)
- maxi = ind;
- }
- }
- else
- {
- for (uint32 y = 0; y < _header->RecordCount; ++y)
- {
- uint32 ind = ((uint32*)idTable)[y];
- if (ind > maxi)
- maxi = ind;
- }
- }
-
- for (uint32 y = 0; y < _header->CopyTableSize; y += 8)
- {
- uint32 ind = *((uint32*)(copyTable + y));
- if (ind > maxi)
- maxi = ind;
- }
-
- using index_entry_t = char*;
-
- ++maxi;
- records = maxi;
- indexTable = new index_entry_t[maxi];
- memset(indexTable, 0, maxi * sizeof(index_entry_t));
-
- char* dataTable = new char[(_header->RecordCount + (_header->CopyTableSize / 8)) * recordsize];
-
- // we store flat holders pool as single memory block
- std::size_t stringFields = _loadInfo->GetStringFieldCount(false);
- std::size_t localizedStringFields = _loadInfo->GetStringFieldCount(true);
-
- // each string field at load have array of string for each locale
- std::size_t stringHoldersRecordPoolSize = localizedStringFields * sizeof(LocalizedString) + (stringFields - localizedStringFields) * sizeof(char*);
- char* stringHoldersPool = nullptr;
- if (stringFields)
- {
- std::size_t stringHoldersPoolSize = stringHoldersRecordPoolSize * _header->RecordCount;
-
- stringHoldersPool = new char[stringHoldersPoolSize];
- stringPool.push_back(stringHoldersPool);
-
- // DB2 strings expected to have at least empty string
- for (std::size_t i = 0; i < stringHoldersPoolSize / sizeof(char*); ++i)
- ((char const**)stringHoldersPool)[i] = nullStr;
- }
-
- uint32 offset = 0;
-
- for (uint32 y = 0; y < _header->RecordCount; y++)
- {
- Record rec = getRecord(y);
- uint32 indexVal = _loadInfo->Meta->HasIndexFieldInData() ? rec.getUInt(indexField, 0, false) : ((uint32*)idTable)[y];
-
- indexTable[indexVal] = &dataTable[offset];
-
- uint32 fieldIndex = 0;
- if (!_loadInfo->Meta->HasIndexFieldInData())
- {
- *((uint32*)(&dataTable[offset])) = indexVal;
- offset += 4;
- ++fieldIndex;
- }
-
- uint32 stringFieldOffset = 0;
-
- for (uint32 x = 0; x < _header->FieldCount; ++x)
- {
- for (uint32 z = 0; z < _loadInfo->Meta->ArraySizes[x]; ++z)
- {
- switch (_loadInfo->TypesString[fieldIndex])
- {
- case FT_FLOAT:
- *((float*)(&dataTable[offset])) = rec.getFloat(x, z);
- offset += 4;
- break;
- case FT_INT:
- *((uint32*)(&dataTable[offset])) = rec.getUInt(x, z, _loadInfo->Fields[fieldIndex].IsSigned);
- offset += 4;
- break;
- case FT_BYTE:
- *((uint8*)(&dataTable[offset])) = rec.getUInt8(x, z);
- offset += 1;
- break;
- case FT_SHORT:
- *((uint16*)(&dataTable[offset])) = rec.getUInt16(x, z);
- offset += 2;
- break;
- case FT_STRING:
- case FT_STRING_NOT_LOCALIZED:
- {
- // init db2 string field slots by pointers to string holders
- char const*** slot = (char const***)(&dataTable[offset]);
- *slot = (char const**)(&stringHoldersPool[stringHoldersRecordPoolSize * y + stringFieldOffset]);
- if (_loadInfo->TypesString[fieldIndex] == FT_STRING)
- stringFieldOffset += sizeof(LocalizedString);
- else
- stringFieldOffset += sizeof(char*);
-
- offset += sizeof(char*);
- break;
- }
- default:
- ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->TypesString[x], fileName);
- break;
- }
- ++fieldIndex;
- }
- }
- }
-
- return dataTable;
-}
-
-char* DB2FileLoaderRegularImpl::AutoProduceStrings(char* dataTable, uint32 locale)
-{
- if (_loadInfo->Meta->FieldCount != _header->FieldCount)
- return nullptr;
-
- if (!(_header->Locale & (1 << locale)))
- {
- char const* sep = "";
- std::ostringstream str;
- for (uint32 i = 0; i < TOTAL_LOCALES; ++i)
- {
- if (_header->Locale & (1 << i))
- {
- str << sep << localeNames[i];
- sep = ", ";
- }
- }
-
- TC_LOG_ERROR("", "Attempted to load %s which has locales %s as %s. Check if you placed your localized db2 files in correct directory.", fileName, str.str().c_str(), localeNames[locale]);
- return nullptr;
- }
-
- char* stringPool = new char[_header->StringTableSize];
- memcpy(stringPool, stringTable, _header->StringTableSize);
-
- uint32 offset = 0;
-
- for (uint32 y = 0; y < _header->RecordCount; y++)
- {
- uint32 fieldIndex = 0;
- if (!_loadInfo->Meta->HasIndexFieldInData())
- {
- offset += 4;
- ++fieldIndex;
- }
-
- for (uint32 x = 0; x < _header->FieldCount; ++x)
- {
- for (uint32 z = 0; z < _loadInfo->Meta->ArraySizes[x]; ++z)
- {
- switch (_loadInfo->TypesString[fieldIndex])
- {
- case FT_FLOAT:
- case FT_INT:
- offset += 4;
- break;
- case FT_BYTE:
- offset += 1;
- break;
- case FT_SHORT:
- offset += 2;
- break;
- case FT_STRING:
- {
- // fill only not filled entries
- LocalizedString* db2str = *(LocalizedString**)(&dataTable[offset]);
- if (db2str->Str[locale] == nullStr)
- {
- char const* st = getRecord(y).getString(x, z);
- db2str->Str[locale] = stringPool + (st - (char const*)stringTable);
- }
-
- offset += sizeof(char*);
- break;
- }
- case FT_STRING_NOT_LOCALIZED:
- {
- char** db2str = (char**)(&dataTable[offset]);
- char const* st = getRecord(y).getString(x, z);
- *db2str = stringPool + (st - (char const*)stringTable);
- offset += sizeof(char*);
- break;
- }
- default:
- ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->TypesString[x], fileName);
- break;
- }
- ++fieldIndex;
- }
- }
- }
-
- return stringPool;
-}
-
-void DB2FileLoaderRegularImpl::AutoProduceRecordCopies(uint32 records, char** indexTable, char* dataTable)
-{
- uint32 recordsize = _loadInfo->Meta->GetRecordSize();
- uint32 offset = _header->RecordCount * recordsize;
- uint32* copyIds = (uint32*)copyTable;
- for (uint32 c = 0; c < _header->CopyTableSize / 4; c += 2)
- {
- uint32 to = copyIds[c];
- uint32 from = copyIds[c + 1];
-
- if (from && from < records && to < records && indexTable[from])
- {
- indexTable[to] = &dataTable[offset];
- memcpy(indexTable[to], indexTable[from], recordsize);
-
- if (_loadInfo->Meta->HasIndexFieldInData())
- *((uint32*)(&dataTable[offset + fields[_loadInfo->Meta->GetIndexField()].Offset])) = to;
- else
- *((uint32*)(&dataTable[offset])) = to;
-
- offset += recordsize;
- }
- }
-}
-
-DB2FileLoaderSparseImpl::DB2FileLoaderSparseImpl()
-{
- fileName = nullptr;
- _loadInfo = nullptr;
- _header = nullptr;
- fields = nullptr;
- dataStart = 0;
- data = nullptr;
- offsets = nullptr;
-}
-
-bool DB2FileLoaderSparseImpl::Load(char const* filename, FILE* file, DB2LoadInfo const* loadInfo, DB2Header const* header)
-{
- _loadInfo = loadInfo;
- _header = header;
- fileName = filename;
-
- fields = new FieldEntry[header->FieldCount];
- if (fread(fields, header->FieldCount * sizeof(FieldEntry), 1, file) != 1)
- return false;
-
- dataStart = ftell(file);
-
- data = new unsigned char[header->StringTableSize - dataStart];
-
- if (fread(data, header->StringTableSize - dataStart, 1, file) != 1)
- return false;
-
- offsets = new OffsetTableEntry[header->MaxId - header->MinId + 1];
- if (fread(offsets, (header->MaxId - header->MinId + 1) * 6, 1, file) != 1)
- return false;
-
- return true;
-}
-
-DB2FileLoaderSparseImpl::~DB2FileLoaderSparseImpl()
-{
- delete[] data;
- delete[] offsets;
- delete[] fields;
-}
-
-char* DB2FileLoaderSparseImpl::AutoProduceData(uint32& maxId, char**& indexTable, std::vector<char*>& stringPool)
-{
- if (_loadInfo->Meta->FieldCount != _header->FieldCount)
- return NULL;
-
- //get struct size and index pos
- uint32 recordsize = _loadInfo->Meta->GetRecordSize();
-
- uint32 offsetCount = _header->MaxId - _header->MinId + 1;
- uint32 records = 0;
- uint32 expandedDataSize = 0;
- for (uint32 i = 0; i < offsetCount; ++i)
- {
- if (offsets[i].FileOffset && offsets[i].RecordSize)
- {
- ++records;
- expandedDataSize += offsets[i].RecordSize;
- }
- }
-
- using index_entry_t = char*;
-
- maxId = _header->MaxId + 1;
- indexTable = new index_entry_t[maxId];
- memset(indexTable, 0, maxId * sizeof(index_entry_t));
-
- char* dataTable = new char[records * recordsize];
-
- // we store flat holders pool as single memory block
- std::size_t stringFields = _loadInfo->GetStringFieldCount(false);
- std::size_t localizedStringFields = _loadInfo->GetStringFieldCount(true);
-
- // each string field at load have array of string for each locale
- std::size_t stringHoldersRecordPoolSize = localizedStringFields * sizeof(LocalizedString) + (stringFields - localizedStringFields) * sizeof(char*);
- std::size_t stringHoldersPoolSize = stringHoldersRecordPoolSize * records;
-
- char* stringHoldersPool = new char[stringHoldersPoolSize];
- stringPool.push_back(stringHoldersPool);
-
- // DB2 strings expected to have at least empty string
- for (std::size_t i = 0; i < stringHoldersPoolSize / sizeof(char*); ++i)
- ((char const**)stringHoldersPool)[i] = nullStr;
-
- char* stringTable = new char[expandedDataSize - records * ((recordsize - (!_loadInfo->Meta->HasIndexFieldInData() ? 4 : 0)) - stringFields * sizeof(char*))];
- memset(stringTable, 0, expandedDataSize - records * ((recordsize - (!_loadInfo->Meta->HasIndexFieldInData() ? 4 : 0)) - stringFields * sizeof(char*)));
- stringPool.push_back(stringTable);
- char* stringPtr = stringTable;
-
- uint32 offset = 0;
- uint32 recordNum = 0;
- for (uint32 y = 0; y < offsetCount; ++y)
- {
- if (!offsets[y].FileOffset || !offsets[y].RecordSize)
- continue;
-
- indexTable[y + _header->MinId] = &dataTable[offset];
- ASSERT(indexTable[y + _header->MinId]);
- std::size_t fieldOffset = 0;
- uint32 stringFieldOffset = 0;
-
- uint32 fieldIndex = 0;
- if (!_loadInfo->Meta->HasIndexFieldInData())
- {
- *((uint32*)(&dataTable[offset])) = y + _header->MinId;
- offset += 4;
- ++fieldIndex;
- }
-
- for (uint32 x = 0; x < _header->FieldCount; ++x)
- {
- uint16 fieldBytes = 4 - fields[x].UnusedBits / 8;
- for (uint32 z = 0; z < _loadInfo->Meta->ArraySizes[x]; ++z)
- {
- switch (_loadInfo->TypesString[fieldIndex])
- {
- case FT_FLOAT:
- {
- float val = *reinterpret_cast<float*>(&data[offsets[y].FileOffset - dataStart + fieldOffset]);
- EndianConvert(val);
- *((float*)(&dataTable[offset])) = val;
- offset += 4;
- fieldOffset += 4;
- break;
- }
- case FT_INT:
- {
- ASSERT(fieldBytes && fieldBytes <= 4);
- uint32 val = 0;
- switch (fieldBytes)
- {
- case 1:
- val = *reinterpret_cast<uint8*>(&data[offsets[y].FileOffset - dataStart + fieldOffset]);
- break;
- case 2:
- {
- uint16 val16 = *reinterpret_cast<uint16*>(&data[offsets[y].FileOffset - dataStart + fieldOffset]);
- EndianConvert(val16);
- val = val16;
- break;
- }
- case 3:
- {
-#pragma pack(push, 1)
- struct dbcint24 { uint8 v[3]; };
-#pragma pack(pop)
- dbcint24 i24v = *reinterpret_cast<dbcint24*>(&data[offsets[y].FileOffset - dataStart + fieldOffset]);
- EndianConvert(i24v);
- val = uint32(i24v.v[0]) | (uint32(i24v.v[1]) << 8) | (uint32(i24v.v[2]) << 16);
- break;
- }
- case 4:
- val = *reinterpret_cast<uint32*>(&data[offsets[y].FileOffset - dataStart + fieldOffset]);
- EndianConvert(val);
- break;
- default:
- break;
- }
- *((uint32*)(&dataTable[offset])) = val;
- offset += 4;
- fieldOffset += fieldBytes;
- break;
- }
- case FT_BYTE:
- {
- ASSERT(fieldBytes == 1);
- *((uint8*)(&dataTable[offset])) = *reinterpret_cast<uint8*>(&data[offsets[y].FileOffset - dataStart + fieldOffset]);
- offset += 1;
- fieldOffset += 1;
- break;
- }
- case FT_SHORT:
- {
- ASSERT(fieldBytes == 2);
- uint16 val = *reinterpret_cast<uint16*>(&data[offsets[y].FileOffset - dataStart + fieldOffset]);
- EndianConvert(val);
- *((uint16*)(&dataTable[offset])) = val;
- offset += 2;
- fieldOffset += 2;
- break;
- }
- case FT_STRING:
- {
- LocalizedString** slot = (LocalizedString**)(&dataTable[offset]);
- *slot = (LocalizedString*)(&stringHoldersPool[stringHoldersRecordPoolSize * recordNum + stringFieldOffset]);
- for (uint32 locale = 0; locale < TOTAL_LOCALES; ++locale)
- if (_header->Locale & (1 << locale))
- (*slot)->Str[locale] = stringPtr;
- strcpy(stringPtr, (char*)&data[offsets[y].FileOffset - dataStart + fieldOffset]);
- fieldOffset += strlen(stringPtr) + 1;
- stringPtr += strlen(stringPtr) + 1;
- stringFieldOffset += sizeof(LocalizedString);
- offset += sizeof(LocalizedString*);
- break;
- }
- case FT_STRING_NOT_LOCALIZED:
- {
- char const*** slot = (char const***)(&dataTable[offset]);
- *slot = (char const**)(&stringHoldersPool[stringHoldersRecordPoolSize * recordNum + stringFieldOffset]);
- **slot = stringPtr;
- strcpy(stringPtr, (char*)&data[offsets[y].FileOffset - dataStart + fieldOffset]);
- fieldOffset += strlen(stringPtr) + 1;
- stringPtr += strlen(stringPtr) + 1;
- stringFieldOffset += sizeof(char*);
- offset += sizeof(char*);
- break;
- }
- default:
- ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->TypesString[x], fileName);
- break;
- }
- ++fieldIndex;
- }
- }
-
- ++recordNum;
- }
-
- return dataTable;
-}
-
-char* DB2FileLoaderSparseImpl::AutoProduceStrings(char* dataTable, uint32 locale)
-{
- if (_loadInfo->Meta->FieldCount != _header->FieldCount)
- return nullptr;
-
- if (!(_header->Locale & (1 << locale)))
- {
- char const* sep = "";
- std::ostringstream str;
- for (uint32 i = 0; i < TOTAL_LOCALES; ++i)
- {
- if (_header->Locale & (1 << i))
- {
- str << sep << localeNames[i];
- sep = ", ";
- }
- }
-
- TC_LOG_ERROR("", "Attempted to load %s which has locales %s as %s. Check if you placed your localized db2 files in correct directory.", fileName, str.str().c_str(), localeNames[locale]);
- return nullptr;
- }
-
- uint32 offsetCount = _header->MaxId - _header->MinId + 1;
- uint32 records = 0;
- for (uint32 i = 0; i < offsetCount; ++i)
- if (offsets[i].FileOffset && offsets[i].RecordSize)
- ++records;
-
- uint32 recordsize = _loadInfo->Meta->GetRecordSize();
- std::size_t stringFields = _loadInfo->GetStringFieldCount(true);
- char* stringTable = new char[_header->StringTableSize - dataStart - records * ((recordsize - (!_loadInfo->Meta->HasIndexFieldInData() ? 4 : 0)) - stringFields * sizeof(char*))];
- memset(stringTable, 0, _header->StringTableSize - dataStart - records * ((recordsize - (!_loadInfo->Meta->HasIndexFieldInData() ? 4 : 0)) - stringFields * sizeof(char*)));
- char* stringPtr = stringTable;
-
- uint32 offset = 0;
-
- for (uint32 y = 0; y < offsetCount; y++)
- {
- if (!offsets[y].FileOffset || !offsets[y].RecordSize)
- continue;
-
- uint32 fieldIndex = 0;
- if (!_loadInfo->Meta->HasIndexFieldInData())
- {
- offset += 4;
- ++fieldIndex;
- }
-
- std::size_t fieldOffset = 0;
- for (uint32 x = 0; x < _header->FieldCount; ++x)
- {
- for (uint32 z = 0; z < _loadInfo->Meta->ArraySizes[x]; ++z)
- {
- switch (_loadInfo->TypesString[fieldIndex])
- {
- case FT_FLOAT:
- offset += 4;
- fieldOffset += 4;
- break;
- case FT_INT:
- offset += 4;
- fieldOffset += 4 - fields[x].UnusedBits / 8;
- break;
- case FT_BYTE:
- offset += 1;
- fieldOffset += 1;
- break;
- case FT_SHORT:
- offset += 2;
- fieldOffset += 2;
- break;
- case FT_STRING:
- {
- LocalizedString* db2str = *(LocalizedString**)(&dataTable[offset]);
- db2str->Str[locale] = stringPtr;
- strcpy(stringPtr, (char*)&data[offsets[y].FileOffset - dataStart + fieldOffset]);
- fieldOffset += strlen(stringPtr) + 1;
- stringPtr += strlen(stringPtr) + 1;
- offset += sizeof(char*);
- break;
- }
- case FT_STRING_NOT_LOCALIZED:
- {
- fieldOffset += strlen((char*)&data[offsets[y].FileOffset - dataStart + fieldOffset]) + 1;
- offset += sizeof(char*);
- break;
- }
- default:
- ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->TypesString[x], fileName);
- break;
- }
- ++fieldIndex;
- }
- }
- }
-
- return stringTable;
-}
-
-char* DB2DatabaseLoader::Load(uint32& records, char**& indexTable, char*& stringHolders, std::vector<char*>& stringPool)
-{
- // Even though this query is executed only once, prepared statement is used to send data from mysql server in binary format
- PreparedQueryResult result = HotfixDatabase.Query(HotfixDatabase.GetPreparedStatement(_loadInfo->Statement));
- if (!result)
- return nullptr;
-
- if (_loadInfo->Meta->GetDbFieldCount() != result->GetFieldCount())
- return nullptr;
-
- // get struct size and index pos
- uint32 indexField = _loadInfo->Meta->GetDbIndexField();
- uint32 recordSize = _loadInfo->Meta->GetRecordSize();
-
- // we store flat holders pool as single memory block
- std::size_t stringFields = _loadInfo->GetStringFieldCount(false);
- std::size_t localizedStringFields = _loadInfo->GetStringFieldCount(true);
-
- // each string field at load have array of string for each locale
- std::size_t stringHoldersRecordPoolSize = localizedStringFields * sizeof(LocalizedString) + (stringFields - localizedStringFields) * sizeof(char*);
-
- if (stringFields)
- {
- std::size_t stringHoldersPoolSize = stringHoldersRecordPoolSize * result->GetRowCount();
- stringHolders = new char[stringHoldersPoolSize];
-
- // DB2 strings expected to have at least empty string
- for (std::size_t i = 0; i < stringHoldersPoolSize / sizeof(char*); ++i)
- ((char const**)stringHolders)[i] = nullStr;
- }
- else
- stringHolders = nullptr;
-
- // Resize index table
- // database query *MUST* contain ORDER BY `index_field` DESC clause
- uint32 indexTableSize = (*result)[indexField].GetUInt32() + 1;
- if (indexTableSize < records)
- indexTableSize = records;
-
- if (indexTableSize > records)
- {
- char** tmpIdxTable = new char*[indexTableSize];
- memset(tmpIdxTable, 0, indexTableSize * sizeof(char*));
- memcpy(tmpIdxTable, indexTable, records * sizeof(char*));
- delete[] indexTable;
- indexTable = tmpIdxTable;
- }
-
- char* tempDataTable = new char[result->GetRowCount() * recordSize];
- uint32* newIndexes = new uint32[result->GetRowCount()];
- uint32 rec = 0;
- uint32 newRecords = 0;
-
- do
- {
- Field* fields = result->Fetch();
- uint32 offset = 0;
- uint32 stringFieldOffset = 0;
-
- uint32 indexValue = fields[indexField].GetUInt32();
-
- // Attempt to overwrite existing data
- char* dataValue = indexTable[indexValue];
- if (!dataValue)
- {
- newIndexes[newRecords] = indexValue;
- dataValue = &tempDataTable[newRecords++ * recordSize];
- }
-
- uint32 f = 0;
- if (!_loadInfo->Meta->HasIndexFieldInData())
- {
- *((uint32*)(&dataValue[offset])) = indexValue;
- offset += 4;
- ++f;
- }
-
- for (uint32 x = 0; x < _loadInfo->Meta->FieldCount; ++x)
- {
- for (uint32 z = 0; z < _loadInfo->Meta->ArraySizes[x]; ++z)
- {
- switch (_loadInfo->TypesString[f])
- {
- case FT_FLOAT:
- *((float*)(&dataValue[offset])) = fields[f].GetFloat();
- offset += 4;
- break;
- case FT_INT:
- *((int32*)(&dataValue[offset])) = fields[f].GetInt32();
- offset += 4;
- break;
- case FT_BYTE:
- *((int8*)(&dataValue[offset])) = fields[f].GetInt8();
- offset += 1;
- break;
- case FT_SHORT:
- *((int16*)(&dataValue[offset])) = fields[f].GetInt16();
- offset += 2;
- break;
- case FT_STRING:
- {
- LocalizedString** slot = (LocalizedString**)(&dataValue[offset]);
- *slot = (LocalizedString*)(&stringHolders[stringHoldersRecordPoolSize * rec + stringFieldOffset]);
- ASSERT(*slot);
-
- // Value in database in main table field must be for enUS locale
- if (char* str = AddString(&(*slot)->Str[LOCALE_enUS], fields[f].GetString()))
- stringPool.push_back(str);
-
- stringFieldOffset += sizeof(LocalizedString);
- offset += sizeof(char*);
- break;
- }
- case FT_STRING_NOT_LOCALIZED:
- {
- char const** slot = (char const**)(&dataValue[offset]);
- *slot = (char*)(&stringHolders[stringHoldersRecordPoolSize * rec + stringFieldOffset]);
- ASSERT(*slot);
-
- // Value in database in main table field must be for enUS locale
- if (char* str = AddString(slot, fields[f].GetString()))
- stringPool.push_back(str);
-
- stringFieldOffset += sizeof(char*);
- offset += sizeof(char*);
- break;
- }
- default:
- ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->TypesString[x], _storageName.c_str());
- break;
- }
- ++f;
- }
- }
-
- ASSERT(offset == recordSize);
- ++rec;
- } while (result->NextRow());
-
- if (!newRecords)
- {
- delete[] tempDataTable;
- delete[] newIndexes;
- return nullptr;
- }
-
- // Compact new data table to only contain new records not previously loaded from file
- char* dataTable = new char[newRecords * recordSize];
- memcpy(dataTable, tempDataTable, newRecords * recordSize);
-
- // insert new records to index table
- for (uint32 i = 0; i < newRecords; ++i)
- indexTable[newIndexes[i]] = &dataTable[i * recordSize];
-
- delete[] tempDataTable;
- delete[] newIndexes;
-
- records = indexTableSize;
-
- return dataTable;
-}
-
-void DB2DatabaseLoader::LoadStrings(uint32 locale, uint32 records, char** indexTable, std::vector<char*>& stringPool)
-{
- PreparedStatement* stmt = HotfixDatabase.GetPreparedStatement(HotfixDatabaseStatements(_loadInfo->Statement + 1));
- stmt->setString(0, localeNames[locale]);
- PreparedQueryResult result = HotfixDatabase.Query(stmt);
- if (!result)
- return;
-
- std::size_t stringFields = _loadInfo->GetStringFieldCount(true);
- if (result->GetFieldCount() != stringFields + 1 /*ID*/)
- return;
-
- uint32 fieldCount = _loadInfo->Meta->FieldCount;
- uint32 recordSize = _loadInfo->Meta->GetRecordSize();
-
- do
- {
- Field* fields = result->Fetch();
- uint32 offset = 0;
- uint32 stringFieldNumInRecord = 0;
- uint32 indexValue = fields[0].GetUInt32();
-
- if (indexValue >= records)
- continue;
-
- // Attempt to overwrite existing data
- if (char* dataValue = indexTable[indexValue])
- {
- uint32 fieldIndex = 0;
- if (!_loadInfo->Meta->HasIndexFieldInData())
- {
- offset += 4;
- ++fieldIndex;
- }
-
- for (uint32 x = 0; x < fieldCount; ++x)
- {
- for (uint32 z = 0; z < _loadInfo->Meta->ArraySizes[x]; ++z)
- {
- switch (_loadInfo->TypesString[fieldIndex])
- {
- case FT_FLOAT:
- case FT_INT:
- offset += 4;
- break;
- case FT_BYTE:
- offset += 1;
- break;
- case FT_SHORT:
- offset += 2;
- break;
- case FT_STRING:
- {
- // fill only not filled entries
- LocalizedString* db2str = *(LocalizedString**)(&dataValue[offset]);
- if (db2str->Str[locale] == nullStr)
- if (char* str = AddString(&db2str->Str[locale], fields[1 + stringFieldNumInRecord].GetString()))
- stringPool.push_back(str);
-
- ++stringFieldNumInRecord;
- offset += sizeof(LocalizedString*);
- break;
- }
- case FT_STRING_NOT_LOCALIZED:
- offset += sizeof(char*);
- break;
- default:
- ASSERT(false, "Unknown format character '%c' found in %s meta", _loadInfo->TypesString[x], _storageName.c_str());
- break;
- }
- ++fieldIndex;
- }
- }
-
- ASSERT(offset == recordSize);
- }
- else
- TC_LOG_ERROR("sql.sql", "Hotfix locale table for storage %s references row that does not exist %u locale %s!", _storageName.c_str(), indexValue, localeNames[locale]);
-
- } while (result->NextRow());
-
- return;
-}
-
-char* DB2DatabaseLoader::AddString(char const** holder, std::string const& value)
-{
- if (!value.empty())
- {
- std::size_t existingLength = strlen(*holder);
- if (existingLength >= value.length())
- {
- // Reuse existing storage if there is enough space
- char* str = const_cast<char*>(*holder);
- memcpy(str, value.c_str(), value.length());
- str[value.length()] = '\0';
- return nullptr;
- }
-
- char* str = new char[value.length() + 1];
- memcpy(str, value.c_str(), value.length());
- str[value.length()] = '\0';
- *holder = str;
- return str;
- }
-
- return nullptr;
-}
-
-DB2FileLoader::DB2FileLoader() : _impl(nullptr)
-{
-}
-
-DB2FileLoader::~DB2FileLoader()
-{
- delete _impl;
-}
-
-bool DB2FileLoader::Load(char const* filename, DB2LoadInfo const* loadInfo)
-{
- FILE* f = fopen(filename, "rb");
- if (!f)
- return false;
-
- if (fread(&_header.Signature, 4, 1, f) != 1) // Signature
- {
- fclose(f);
- return false;
- }
-
- EndianConvert(_header.Signature);
-
- if (_header.Signature != 0x35424457)
- {
- fclose(f);
- return false; //'WDB5'
- }
-
- if (fread(&_header.RecordCount, 4, 1, f) != 1) // Number of records
- {
- fclose(f);
- return false;
- }
-
- EndianConvert(_header.RecordCount);
-
- if (fread(&_header.FieldCount, 4, 1, f) != 1) // Number of fields
- {
- fclose(f);
- return false;
- }
-
- EndianConvert(_header.FieldCount);
-
- if (fread(&_header.RecordSize, 4, 1, f) != 1) // Size of a record
- {
- fclose(f);
- return false;
- }
-
- EndianConvert(_header.RecordSize);
-
- if (fread(&_header.StringTableSize, 4, 1, f) != 1) // String size
- {
- fclose(f);
- return false;
- }
-
- EndianConvert(_header.StringTableSize);
-
- if (fread(&_header.TableHash, 4, 1, f) != 1) // Table hash
- {
- fclose(f);
- return false;
- }
-
- EndianConvert(_header.TableHash);
-
- if (fread(&_header.LayoutHash, 4, 1, f) != 1) // Layout hash
- {
- fclose(f);
- return false;
- }
-
- EndianConvert(_header.LayoutHash);
-
- if (_header.LayoutHash != loadInfo->Meta->LayoutHash)
- {
- fclose(f);
- return false;
- }
-
- if (fread(&_header.MinId, 4, 1, f) != 1) // MinIndex WDB2
- {
- fclose(f);
- return false;
- }
-
- EndianConvert(_header.MinId);
-
- if (fread(&_header.MaxId, 4, 1, f) != 1) // MaxIndex WDB2
- {
- fclose(f);
- return false;
- }
-
- EndianConvert(_header.MaxId);
-
- if (fread(&_header.Locale, 4, 1, f) != 1) // Locales
- {
- fclose(f);
- return false;
- }
-
- EndianConvert(_header.Locale);
-
- if (fread(&_header.CopyTableSize, 4, 1, f) != 1)
- {
- fclose(f);
- return false;
- }
-
- EndianConvert(_header.CopyTableSize);
-
- if (fread(&_header.Flags, 4, 1, f) != 1)
- {
- fclose(f);
- return false;
- }
-
- EndianConvert(_header.Flags);
-
- if (!(_header.Flags & 0x1))
- _impl = new DB2FileLoaderRegularImpl();
- else
- _impl = new DB2FileLoaderSparseImpl();
-
- bool result = _impl->Load(filename, f, loadInfo, &_header);
- fclose(f);
- return result;
-}
-
-char* DB2FileLoader::AutoProduceData(uint32& count, char**& indexTable, std::vector<char*>& stringPool)
-{
- return _impl->AutoProduceData(count, indexTable, stringPool);
-}
-
-char* DB2FileLoader::AutoProduceStrings(char* dataTable, uint32 locale)
-{
- return _impl->AutoProduceStrings(dataTable, locale);
-}
-
-void DB2FileLoader::AutoProduceRecordCopies(uint32 records, char** indexTable, char* dataTable)
-{
- _impl->AutoProduceRecordCopies(records, indexTable, dataTable);
-}
diff --git a/src/server/shared/DataStores/DB2Store.cpp b/src/server/shared/DataStores/DB2Store.cpp
new file mode 100644
index 00000000000..bb36ed6b61b
--- /dev/null
+++ b/src/server/shared/DataStores/DB2Store.cpp
@@ -0,0 +1,189 @@
+/*
+ * Copyright (C) 2008-2017 TrinityCore <http://www.trinitycore.org/>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "DB2Store.h"
+#include "ByteBuffer.h"
+#include "DB2DatabaseLoader.h"
+#include "DB2Meta.h"
+
+struct DB2FileSystemSource : public DB2FileSource
+{
+ DB2FileSystemSource(std::string const& fileName)
+ {
+ _fileName = fileName;
+ _file = fopen(_fileName.c_str(), "rb");
+ }
+
+ ~DB2FileSystemSource()
+ {
+ if (_file)
+ fclose(_file);
+ }
+
+ bool IsOpen() const override
+ {
+ return _file != nullptr;
+ }
+
+ bool Read(void* buffer, std::size_t numBytes) override
+ {
+ return fread(buffer, numBytes, 1, _file) == 1;
+ }
+
+ std::size_t GetPosition() const override
+ {
+ return ftell(_file);
+ }
+
+ char const* GetFileName() const override
+ {
+ return _fileName.c_str();
+ }
+
+private:
+ std::string _fileName;
+ FILE* _file;
+};
+
+DB2StorageBase::DB2StorageBase(char const* fileName, DB2LoadInfo const* loadInfo)
+ : _tableHash(0), _layoutHash(0), _fileName(fileName), _fieldCount(0), _loadInfo(loadInfo), _dataTable(nullptr), _dataTableEx(nullptr), _indexTableSize(0)
+{
+}
+
+DB2StorageBase::~DB2StorageBase()
+{
+ delete[] _dataTable;
+ delete[] _dataTableEx;
+ for (char* strings : _stringPool)
+ delete[] strings;
+}
+
+void DB2StorageBase::WriteRecordData(char const* entry, uint32 locale, ByteBuffer& buffer) const
+{
+ std::size_t i = 0;
+ if (!_loadInfo->Meta->HasIndexFieldInData())
+ {
+ entry += 4;
+ ++i;
+ }
+
+ for (; i < _loadInfo->FieldCount; ++i)
+ {
+ switch (_loadInfo->TypesString[i])
+ {
+ case FT_INT:
+ buffer << *(uint32*)entry;
+ entry += 4;
+ break;
+ case FT_FLOAT:
+ buffer << *(float*)entry;
+ entry += 4;
+ break;
+ case FT_BYTE:
+ buffer << *(uint8*)entry;
+ entry += 1;
+ break;
+ case FT_SHORT:
+ buffer << *(uint16*)entry;
+ entry += 2;
+ break;
+ case FT_STRING:
+ {
+ LocalizedString* locStr = *(LocalizedString**)entry;
+ if (locStr->Str[locale][0] == '\0')
+ locale = 0;
+
+ buffer << locStr->Str[locale];
+ entry += sizeof(LocalizedString*);
+ break;
+ }
+ case FT_STRING_NOT_LOCALIZED:
+ {
+ buffer << *(char const**)entry;
+ entry += sizeof(char const*);
+ break;
+ }
+ }
+ }
+}
+
+bool DB2StorageBase::Load(std::string const& path, uint32 locale, char**& indexTable)
+{
+ indexTable = nullptr;
+ DB2FileLoader db2;
+ {
+ DB2FileSystemSource source(path + _fileName);
+ // Check if load was successful, only then continue
+ if (!db2.Load(&source, _loadInfo))
+ return false;
+ }
+
+ _fieldCount = db2.GetCols();
+ _tableHash = db2.GetTableHash();
+ _layoutHash = db2.GetLayoutHash();
+
+ // load raw non-string data
+ _dataTable = db2.AutoProduceData(_indexTableSize, indexTable, _stringPool);
+
+ // load strings from db2 data
+ if (!_stringPool.empty())
+ if (char* stringBlock = db2.AutoProduceStrings(_dataTable, locale))
+ _stringPool.push_back(stringBlock);
+
+ db2.AutoProduceRecordCopies(_indexTableSize, indexTable, _dataTable);
+
+ // error in db2 file at loading if NULL
+ return indexTable != nullptr;
+}
+
+bool DB2StorageBase::LoadStringsFrom(std::string const& path, uint32 locale, char** indexTable)
+{
+ // DB2 must be already loaded using Load
+ if (!indexTable)
+ return false;
+
+ DB2FileLoader db2;
+ {
+ DB2FileSystemSource source(path + _fileName);
+ // Check if load was successful, only then continue
+ if (!db2.Load(&source, _loadInfo))
+ return false;
+ }
+
+ // load strings from another locale db2 data
+ if (_loadInfo->GetStringFieldCount(true))
+ if (char* stringBlock = db2.AutoProduceStrings(_dataTable, locale))
+ _stringPool.push_back(stringBlock);
+
+ return true;
+}
+
+void DB2StorageBase::LoadFromDB(char**& indexTable)
+{
+ char* extraStringHolders = nullptr;
+ _dataTableEx = DB2DatabaseLoader(_fileName, _loadInfo).Load(_indexTableSize, indexTable, extraStringHolders, _stringPool);
+ if (extraStringHolders)
+ _stringPool.push_back(extraStringHolders);
+}
+
+void DB2StorageBase::LoadStringsFromDB(uint32 locale, char** indexTable)
+{
+ if (!_loadInfo->GetStringFieldCount(true))
+ return;
+
+ DB2DatabaseLoader(_fileName, _loadInfo).LoadStrings(locale, _indexTableSize, indexTable, _stringPool);
+}
diff --git a/src/server/shared/DataStores/DB2Store.h b/src/server/shared/DataStores/DB2Store.h
index 7011a3b6e61..68cd888cd05 100644
--- a/src/server/shared/DataStores/DB2Store.h
+++ b/src/server/shared/DataStores/DB2Store.h
@@ -19,41 +19,29 @@
#define DB2STORE_H
#include "Common.h"
-#include "DB2Meta.h"
-#include "DB2StorageLoader.h"
#include "DBStorageIterator.h"
-#include "ByteBuffer.h"
+#include <vector>
+
+class ByteBuffer;
+struct DB2LoadInfo;
/// Interface class for common access
-class DB2StorageBase
+class TC_SHARED_API DB2StorageBase
{
public:
- DB2StorageBase(char const* fileName, DB2LoadInfo&& loadInfo)
- : _tableHash(0), _layoutHash(0), _fileName(fileName), _fieldCount(0), _loadInfo(std::move(loadInfo)), _dataTable(nullptr), _dataTableEx(nullptr) { }
-
- virtual ~DB2StorageBase()
- {
- delete[] _dataTable;
- delete[] _dataTableEx;
- for (char* strings : _stringPool)
- delete[] strings;
- }
+ DB2StorageBase(char const* fileName, DB2LoadInfo const* loadInfo);
+ virtual ~DB2StorageBase();
uint32 GetTableHash() const { return _tableHash; }
-
uint32 GetLayoutHash() const { return _layoutHash; }
virtual bool HasRecord(uint32 id) const = 0;
-
virtual void WriteRecord(uint32 id, uint32 locale, ByteBuffer& buffer) const = 0;
-
virtual void EraseRecord(uint32 id) = 0;
std::string const& GetFileName() const { return _fileName; }
-
uint32 GetFieldCount() const { return _fieldCount; }
-
- DB2LoadInfo const* GetLoadInfo() const { return &_loadInfo; }
+ DB2LoadInfo const* GetLoadInfo() const { return _loadInfo; }
virtual bool Load(std::string const& path, uint32 locale) = 0;
virtual bool LoadStringsFrom(std::string const& path, uint32 locale) = 0;
@@ -61,63 +49,21 @@ public:
virtual void LoadStringsFromDB(uint32 locale) = 0;
protected:
- void WriteRecordData(char const* entry, uint32 locale, ByteBuffer& buffer) const
- {
- std::size_t i = 0;
- if (!_loadInfo.Meta->HasIndexFieldInData())
- {
- entry += 4;
- ++i;
- }
-
- for (; i < _loadInfo.FieldCount; ++i)
- {
- switch (_loadInfo.TypesString[i])
- {
- case FT_INT:
- buffer << *(uint32*)entry;
- entry += 4;
- break;
- case FT_FLOAT:
- buffer << *(float*)entry;
- entry += 4;
- break;
- case FT_BYTE:
- buffer << *(uint8*)entry;
- entry += 1;
- break;
- case FT_SHORT:
- buffer << *(uint16*)entry;
- entry += 2;
- break;
- case FT_STRING:
- {
- LocalizedString* locStr = *(LocalizedString**)entry;
- if (locStr->Str[locale][0] == '\0')
- locale = 0;
-
- buffer << locStr->Str[locale];
- entry += sizeof(LocalizedString*);
- break;
- }
- case FT_STRING_NOT_LOCALIZED:
- {
- buffer << *(char const**)entry;
- entry += sizeof(char const*);
- break;
- }
- }
- }
- }
+ void WriteRecordData(char const* entry, uint32 locale, ByteBuffer& buffer) const;
+ bool Load(std::string const& path, uint32 locale, char**& indexTable);
+ bool LoadStringsFrom(std::string const& path, uint32 locale, char** indexTable);
+ void LoadFromDB(char**& indexTable);
+ void LoadStringsFromDB(uint32 locale, char** indexTable);
uint32 _tableHash;
uint32 _layoutHash;
std::string _fileName;
uint32 _fieldCount;
- DB2LoadInfo _loadInfo;
+ DB2LoadInfo const* _loadInfo;
char* _dataTable;
char* _dataTableEx;
std::vector<char*> _stringPool;
+ uint32 _indexTableSize;
};
template<class T>
@@ -128,10 +74,9 @@ class DB2Storage : public DB2StorageBase
public:
typedef DBStorageIterator<T> iterator;
- DB2Storage(char const* fileName, DB2LoadInfo&& loadInfo) : DB2StorageBase(fileName, std::move(loadInfo)),
- _indexTableSize(0)
+ DB2Storage(char const* fileName, DB2LoadInfo const* loadInfo) : DB2StorageBase(fileName, loadInfo)
{
- _indexTable.AsT = NULL;
+ _indexTable.AsT = nullptr;
}
~DB2Storage()
@@ -153,61 +98,22 @@ public:
uint32 GetNumRows() const { return _indexTableSize; }
bool Load(std::string const& path, uint32 locale) override
{
- DB2FileLoader db2;
- // Check if load was successful, only then continue
- if (!db2.Load((path + _fileName).c_str(), &_loadInfo))
- return false;
-
- _fieldCount = db2.GetCols();
- _tableHash = db2.GetTableHash();
- _layoutHash = db2.GetLayoutHash();
-
- // load raw non-string data
- _dataTable = db2.AutoProduceData(_indexTableSize, _indexTable.AsChar, _stringPool);
-
- // load strings from db2 data
- if (!_stringPool.empty())
- if (char* stringBlock = db2.AutoProduceStrings(_dataTable, locale))
- _stringPool.push_back(stringBlock);
-
- db2.AutoProduceRecordCopies(_indexTableSize, _indexTable.AsChar, _dataTable);
-
- // error in db2 file at loading if NULL
- return _indexTable.AsT != NULL;
+ return DB2StorageBase::Load(path, locale, _indexTable.AsChar);
}
bool LoadStringsFrom(std::string const& path, uint32 locale) override
{
- // DB2 must be already loaded using Load
- if (!_indexTable.AsT)
- return false;
-
- DB2FileLoader db2;
- // Check if load was successful, only then continue
- if (!db2.Load((path + _fileName).c_str(), &_loadInfo))
- return false;
-
- // load strings from another locale db2 data
- if (_loadInfo.GetStringFieldCount(true))
- if (char* stringBlock = db2.AutoProduceStrings(_dataTable, locale))
- _stringPool.push_back(stringBlock);
- return true;
+ return DB2StorageBase::LoadStringsFrom(path, locale, _indexTable.AsChar);
}
void LoadFromDB() override
{
- char* extraStringHolders = nullptr;
- _dataTableEx = DB2DatabaseLoader(_fileName, &_loadInfo).Load(_indexTableSize, _indexTable.AsChar, extraStringHolders, _stringPool);
- if (extraStringHolders)
- _stringPool.push_back(extraStringHolders);
+ DB2StorageBase::LoadFromDB(_indexTable.AsChar);
}
void LoadStringsFromDB(uint32 locale) override
{
- if (!_loadInfo.GetStringFieldCount(true))
- return;
-
- DB2DatabaseLoader(_fileName, &_loadInfo).LoadStrings(locale, _indexTableSize, _indexTable.AsChar, _stringPool);
+ DB2StorageBase::LoadStringsFromDB(locale, _indexTable.AsChar);
}
iterator begin() { return iterator(_indexTable.AsT, _indexTableSize); }
@@ -219,7 +125,6 @@ private:
T** AsT;
char** AsChar;
} _indexTable;
- uint32 _indexTableSize;
};
#endif