Mirror/TrinityCore
1
0
Fork 0
mirror of https://github.com/TrinityCore/TrinityCore.git synced 2026-01-15 23:20:36 +01:00
Code Issues Packages Projects Releases Wiki Activity

Core/DataStores: Simplified string memory allocation in db2 files, dropped unneccessary level of indirection

Browse Source
This commit is contained in:
Shauren
2020-06-27 13:33:17 +02:00
parent eccc015ce7
commit deca201f77
38 changed files with 333 additions and 380 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
src/common/DataStores/DB2FileLoader.cpp
View File

@@ -17,7 +17,6 @@
#include "DB2FileLoader.h"
#include "ByteConverter.h"
#include "Common.h"
#include "DB2Meta.h"
#include "Errors.h"
#include "Log.h"
@@ -112,10 +111,6 @@ DB2FieldMeta::DB2FieldMeta(bool isSigned, DBCFormer type, char const* name)
{
}
DB2FileLoadInfo::DB2FileLoadInfo() : Fields(nullptr), FieldCount(0), Meta(nullptr)
{
}
DB2FileLoadInfo::DB2FileLoadInfo(DB2FieldMeta const* fields, std::size_t fieldCount, DB2Meta const* meta)
: Fields(fields), FieldCount(fieldCount), Meta(meta)
{
@@ -382,7 +377,7 @@ DB2FileLoaderRegularImpl::~DB2FileLoaderRegularImpl()
static char const* const nullStr = "";
char* DB2FileLoaderRegularImpl::AutoProduceData(uint32& records, char**& indexTable, std::vector<char*>& stringPool)
char* DB2FileLoaderRegularImpl::AutoProduceData(uint32& records, char**& indexTable, std::vector<char*>& /*stringPool*/)
{
//get struct size and index pos
uint32 recordsize = _loadInfo->Meta->GetRecordSize();
@@ -397,27 +392,8 @@ char* DB2FileLoaderRegularImpl::AutoProduceData(uint32& records, char**& indexTa
char* dataTable = new char[(_header->RecordCount + _copyTable.size()) * recordsize];
// we store flat holders pool as single memory block
std::size_t stringFields = _loadInfo->GetStringFieldCount(false);
std::size_t localizedStringFields = _loadInfo->GetStringFieldCount(true);
// each string field at load have array of string for each locale
std::size_t stringHoldersRecordPoolSize = localizedStringFields * sizeof(LocalizedString) + (stringFields - localizedStringFields) * sizeof(char*);
char* stringHoldersPool = nullptr;
if (stringFields)
{
std::size_t stringHoldersPoolSize = stringHoldersRecordPoolSize * _header->RecordCount;
stringHoldersPool = new char[stringHoldersPoolSize];
stringPool.push_back(stringHoldersPool);
// DB2 strings expected to have at least empty string
for (std::size_t i = 0; i < stringHoldersPoolSize / sizeof(char*); ++i)
((char const**)stringHoldersPool)[i] = nullStr;
}
uint32 offset = 0;
uint32 y = 0;
uint32 recordIndex = 0;
for (uint32 section = 0; section < _header->SectionCount; ++section)
{
@@ -425,17 +401,17 @@ char* DB2FileLoaderRegularImpl::AutoProduceData(uint32& records, char**& indexTa
if (sectionHeader.TactId)
{
offset += recordsize * sectionHeader.RecordCount;
y += sectionHeader.RecordCount;
recordIndex += sectionHeader.RecordCount;
continue;
}
for (uint32 sr = 0; sr < sectionHeader.RecordCount; ++sr, ++y)
for (uint32 sr = 0; sr < sectionHeader.RecordCount; ++sr, ++recordIndex)
{
unsigned char const* rawRecord = GetRawRecordData(y, &section);
unsigned char const* rawRecord = GetRawRecordData(recordIndex, &section);
if (!rawRecord)
continue;
uint32 indexVal = RecordGetId(rawRecord, y);
uint32 indexVal = RecordGetId(rawRecord, recordIndex);
indexTable[indexVal] = &dataTable[offset];
@@ -447,8 +423,6 @@ char* DB2FileLoaderRegularImpl::AutoProduceData(uint32& records, char**& indexTa
++fieldIndex;
}
uint32 stringFieldOffset = 0;
for (uint32 x = 0; x < _header->FieldCount; ++x)
{
for (uint32 z = 0; z < _loadInfo->Meta->Fields[x].ArraySize; ++z)
@@ -476,19 +450,15 @@ char* DB2FileLoaderRegularImpl::AutoProduceData(uint32& records, char**& indexTa
offset += 8;
break;
case FT_STRING:
case FT_STRING_NOT_LOCALIZED:
{
// init db2 string field slots by pointers to string holders
char const*** slot = (char const***)(&dataTable[offset]);
*slot = (char const**)(&stringHoldersPool[stringHoldersRecordPoolSize * y + stringFieldOffset]);
if (_loadInfo->TypesString[fieldIndex] == FT_STRING)
stringFieldOffset += sizeof(LocalizedString);
else
stringFieldOffset += sizeof(char*);
for (char const*& localeStr : ((LocalizedString*)(&dataTable[offset]))->Str)
localeStr = nullStr;
offset += sizeof(LocalizedString);
break;
case FT_STRING_NOT_LOCALIZED:
*(char const**)(&dataTable[offset]) = nullStr;
offset += sizeof(char*);
break;
}
default:
ASSERT(false, "Unknown format character '%c' found in %s meta for field %s",
_loadInfo->TypesString[fieldIndex], _fileName, _loadInfo->Fields[fieldIndex].Name);
@@ -607,22 +577,13 @@ char* DB2FileLoaderRegularImpl::AutoProduceStrings(char** indexTable, uint32 ind
break;
case FT_STRING:
{
// fill only not filled entries
LocalizedString* db2str = *(LocalizedString * *)(&recordData[offset]);
if (db2str->Str[locale] == nullStr)
{
char const* st = RecordGetString(rawRecord, x, z);
db2str->Str[locale] = stringPool + (st - (char const*)_stringTable);
}
offset += sizeof(char*);
((LocalizedString*)(&recordData[offset]))->Str[locale] = stringPool + (RecordGetString(rawRecord, x, z) - (char const*)_stringTable);
offset += sizeof(LocalizedString);
break;
}
case FT_STRING_NOT_LOCALIZED:
{
char** db2str = (char**)(&recordData[offset]);
char const* st = RecordGetString(rawRecord, x, z);
*db2str = stringPool + (st - (char const*)_stringTable);
*((char**)(&recordData[offset])) = stringPool + (RecordGetString(rawRecord, x, z) - (char const*)_stringTable);
offset += sizeof(char*);
break;
}
@@ -685,7 +646,7 @@ void DB2FileLoaderRegularImpl::FillParentLookup(char* dataTable)
else
{
// in data block, must fit
ASSERT(parentId <= 0xFFFF, "ParentId value %u does not fit into uint16 field (%s in %s)",
ASSERT(parentId <= std::numeric_limits<uint16>::max(), "ParentId value %u does not fit into uint16 field (%s in %s)",
parentId, _loadInfo->Fields[_loadInfo->GetFieldIndexByMetaIndex(_loadInfo->Meta->ParentIndexField)].Name, _fileName);
*reinterpret_cast<uint16*>(&recordData[parentIdOffset]) = parentId;
}
@@ -701,7 +662,7 @@ void DB2FileLoaderRegularImpl::FillParentLookup(char* dataTable)
else
{
// in data block, must fit
ASSERT(parentId <= 0xFF, "ParentId value %u does not fit into uint8 field (%s in %s)",
ASSERT(parentId <= std::numeric_limits<uint8>::max(), "ParentId value %u does not fit into uint8 field (%s in %s)",
parentId, _loadInfo->Fields[_loadInfo->GetFieldIndexByMetaIndex(_loadInfo->Meta->ParentIndexField)].Name, _fileName);
*reinterpret_cast<uint8*>(&recordData[parentIdOffset]) = parentId;
}
@@ -1076,24 +1037,19 @@ char* DB2FileLoaderSparseImpl::AutoProduceData(uint32& maxId, char**& indexTable
memset(indexTable, 0, maxId * sizeof(index_entry_t));
char* dataTable = new char[(records + _copyTable.size()) * recordsize];
memset(dataTable, 0, (records + _copyTable.size()) * recordsize);
// we store flat holders pool as single memory block
std::size_t stringFields = _loadInfo->GetStringFieldCount(false);
std::size_t localizedStringFields = _loadInfo->GetStringFieldCount(true);
// each string field at load have array of string for each locale
std::size_t stringHoldersRecordPoolSize = localizedStringFields * sizeof(LocalizedString) + (stringFields - localizedStringFields) * sizeof(char*);
std::size_t stringHoldersPoolSize = stringHoldersRecordPoolSize * records;
std::size_t stringsInRecordSize = (stringFields - localizedStringFields) * sizeof(char*);
std::size_t localizedStringsInRecordSize = localizedStringFields * sizeof(LocalizedString);
char* stringHoldersPool = new char[stringHoldersPoolSize];
stringPool.push_back(stringHoldersPool);
// string table size is "total size of all records" - RecordCount * "size of record without strings"
std::size_t stringTableSize = _totalRecordSize - (records * ((recordsize - (!_loadInfo->Meta->HasIndexFieldInData() ? 4 : 0)) - stringsInRecordSize - localizedStringsInRecordSize));
// DB2 strings expected to have at least empty string
for (std::size_t i = 0; i < stringHoldersPoolSize / sizeof(char*); ++i)
((char const**)stringHoldersPool)[i] = nullStr;
char* stringTable = new char[_totalRecordSize - records * ((recordsize - (!_loadInfo->Meta->HasIndexFieldInData() ? 4 : 0)) - stringFields * sizeof(char*))];
memset(stringTable, 0, _totalRecordSize - records * ((recordsize - (!_loadInfo->Meta->HasIndexFieldInData() ? 4 : 0)) - stringFields * sizeof(char*)));
char* stringTable = new char[stringTableSize];
memset(stringTable, 0, stringTableSize);
stringPool.push_back(stringTable);
char* stringPtr = stringTable;
@@ -1128,7 +1084,6 @@ char* DB2FileLoaderSparseImpl::AutoProduceData(uint32& maxId, char**& indexTable
++fieldIndex;
}
uint32 stringFieldOffset = 0;
for (uint32 x = 0; x < _header->FieldCount; ++x)
{
for (uint32 z = 0; z < _loadInfo->Meta->Fields[x].ArraySize; ++z)
@@ -1157,25 +1112,25 @@ char* DB2FileLoaderSparseImpl::AutoProduceData(uint32& maxId, char**& indexTable
break;
case FT_STRING:
{
LocalizedString** slot = (LocalizedString * *)(&dataTable[offset]);
*slot = (LocalizedString*)(&stringHoldersPool[stringHoldersRecordPoolSize * recordNum + stringFieldOffset]);
LocalizedString* slot = (LocalizedString*)(&dataTable[offset]);
for (uint32 locale = 0; locale < TOTAL_LOCALES; ++locale)
{
slot->Str[locale] = nullStr;
if (_header->Locale & (1 << locale))
(*slot)->Str[locale] = stringPtr;
slot->Str[locale] = stringPtr;
}
strcpy(stringPtr, RecordGetString(rawRecord, x, z));
stringPtr += strlen(stringPtr) + 1;
stringFieldOffset += sizeof(LocalizedString);
offset += sizeof(LocalizedString*);
offset += sizeof(LocalizedString);
break;
}
case FT_STRING_NOT_LOCALIZED:
{
char const*** slot = (char const***)(&dataTable[offset]);
*slot = (char const**)(&stringHoldersPool[stringHoldersRecordPoolSize * recordNum + stringFieldOffset]);
**slot = stringPtr;
char const** slot = (char const**)(&dataTable[offset]);
*slot = stringPtr;
strcpy(stringPtr, RecordGetString(rawRecord, x, z));
stringPtr += strlen(stringPtr) + 1;
stringFieldOffset += sizeof(char*);
offset += sizeof(char*);
break;
}
@@ -1246,9 +1201,17 @@ char* DB2FileLoaderSparseImpl::AutoProduceStrings(char** indexTable, uint32 inde
uint32 records = _catalog.size();
uint32 recordsize = _loadInfo->Meta->GetRecordSize();
std::size_t stringFields = _loadInfo->GetStringFieldCount(true);
char* stringTable = new char[_totalRecordSize - records * ((recordsize - (!_loadInfo->Meta->HasIndexFieldInData() ? 4 : 0)) - stringFields * sizeof(char*))];
memset(stringTable, 0, _totalRecordSize - records * ((recordsize - (!_loadInfo->Meta->HasIndexFieldInData() ? 4 : 0)) - stringFields * sizeof(char*)));
std::size_t stringFields = _loadInfo->GetStringFieldCount(false);
std::size_t localizedStringFields = _loadInfo->GetStringFieldCount(true);
std::size_t stringsInRecordSize = (stringFields - localizedStringFields) * sizeof(char*);
std::size_t localizedStringsInRecordSize = localizedStringFields * sizeof(LocalizedString);
// string table size is "total size of all records" - RecordCount * "size of record without strings"
std::size_t stringTableSize = _totalRecordSize - (records * ((recordsize - (!_loadInfo->Meta->HasIndexFieldInData() ? 4 : 0)) - stringsInRecordSize - localizedStringsInRecordSize));
char* stringTable = new char[stringTableSize];
memset(stringTable, 0, stringTableSize);
char* stringPtr = stringTable;
uint32 y = 0;
@@ -1305,11 +1268,11 @@ char* DB2FileLoaderSparseImpl::AutoProduceStrings(char** indexTable, uint32 inde
break;
case FT_STRING:
{
LocalizedString* db2str = *(LocalizedString * *)(&recordData[offset]);
LocalizedString* db2str = (LocalizedString*)(&recordData[offset]);
db2str->Str[locale] = stringPtr;
strcpy(stringPtr, RecordGetString(rawRecord, x, z));
stringPtr += strlen(stringPtr) + 1;
offset += sizeof(char*);
offset += sizeof(LocalizedString);
break;
}
case FT_STRING_NOT_LOCALIZED:
@@ -1999,7 +1962,7 @@ char* DB2FileLoader::AutoProduceData(uint32& count, char**& indexTable, std::vec
return _impl->AutoProduceData(count, indexTable, stringPool);
}
char* DB2FileLoader::AutoProduceStrings(char** indexTable, uint32 indexTableSize, uint32 locale)
char* DB2FileLoader::AutoProduceStrings(char** indexTable, uint32 indexTableSize, LocaleConstant locale)
{
return _impl->AutoProduceStrings(indexTable, indexTableSize, locale);
}