Mirror/TrinityCore
1
0
Fork 0
mirror of https://github.com/TrinityCore/TrinityCore.git synced 2026-01-25 03:12:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

Tools/vmaps_extractor: Use multiple threads to extract data

Browse Source
This commit is contained in:
Shauren
2025-11-01 14:26:48 +01:00
parent 4b27db87b7
commit b705b169ef
8 changed files with 233 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
src/tools/vmap4_extractor/adtfile.cpp
View File

@@ -15,12 +15,11 @@
* with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vmapexport.h"
#include "adtfile.h"
#include "StringFormat.h"
#include <cstdio>
#include "Errors.h"
#include "Memory.h"
#include "StringFormat.h"
#include "vmapexport.h"
#include <cstdio>
char const* GetPlainName(char const* FileName)
{
@@ -183,16 +182,15 @@ bool ADTFile::init(uint32 map_num, uint32 originalMapId)
{
ADT::MDDF doodadDef;
_file.read(&doodadDef, sizeof(ADT::MDDF));
if (!(doodadDef.Flags & 0x40))
{
Doodad::Extract(doodadDef, ModelInstanceNames[doodadDef.Id].c_str(), map_num, originalMapId, dirfile.get(), dirfileCache);
}
std::string fileName;
if (doodadDef.Flags & 0x40)
fileName = Trinity::StringFormat("FILE{:08X}.xxx", doodadDef.Id);
else
{
std::string fileName = Trinity::StringFormat("FILE{:08X}.xxx", doodadDef.Id);
ExtractSingleModel(fileName);
fileName = ModelInstanceNames[doodadDef.Id];
if (ExtractSingleModel(fileName))
Doodad::Extract(doodadDef, fileName.c_str(), map_num, originalMapId, dirfile.get(), dirfileCache);
}
}
ModelInstanceNames.clear();
@@ -210,15 +208,18 @@ bool ADTFile::init(uint32 map_num, uint32 originalMapId)
std::string fileName;
if (mapObjDef.Flags & 0x8)
{
fileName = Trinity::StringFormat("FILE{:08X}.xxx", mapObjDef.Id);
ExtractSingleWmo(fileName);
}
else
fileName = WmoInstanceNames[mapObjDef.Id];
MapObject::Extract(mapObjDef, fileName.c_str(), false, map_num, originalMapId, dirfile.get(), dirfileCache);
Doodad::ExtractSet(WmoDoodads[fileName], mapObjDef, false, map_num, originalMapId, dirfile.get(), dirfileCache);
if (ExtractedModelData const* extracted = ExtractSingleWmo(fileName))
{
if (extracted->HasCollision())
MapObject::Extract(mapObjDef, fileName.c_str(), false, map_num, originalMapId, dirfile.get(), dirfileCache);
if (extracted->Doodads)
Doodad::ExtractSet(*extracted->Doodads, mapObjDef, false, map_num, originalMapId, dirfile.get(), dirfileCache);
}
}
WmoInstanceNames.clear();

36
src/tools/vmap4_extractor/gameobject_extract.cpp
View File

@@ -19,6 +19,7 @@
#include "DB2CascFileSource.h"
#include "Errors.h"
#include "ExtractorDB2LoadInfo.h"
#include "Memory.h"
#include "model.h"
#include "StringFormat.h"
#include "vmapexport.h"
@@ -28,10 +29,10 @@
#include <cstdio>
#include "advstd.h"
bool ExtractSingleModel(std::string& fname)
ExtractedModelData const* ExtractSingleModel(std::string& fname)
{
if (fname.length() < 4)
return false;
return nullptr;
std::string extension = fname.substr(fname.length() - 4, 4);
if (extension == ".mdx" || extension == ".MDX" || extension == ".mdl" || extension == ".MDL")
@@ -45,18 +46,28 @@ bool ExtractSingleModel(std::string& fname)
char* name = GetPlainName((char*)fname.c_str());
NormalizeFileName(name, strlen(name));
auto [model, shouldExtract] = BeginModelExtraction(name);
if (!shouldExtract)
{
model->Wait();
return model->State.load(std::memory_order::relaxed) == ExtractedModelData::Ok ? model : nullptr;
}
auto stateGuard = Trinity::make_unique_ptr_with_deleter<&ExtractedModelData::Fail>(model);
Model mdl(originalName);
if (!mdl.open())
return nullptr;
std::string output(szWorkDirWmo);
output += "/";
output += name;
if (FileExists(output.c_str()))
return true;
if (!mdl.ConvertToVMAPModel(output.c_str()))
return nullptr;
Model mdl(originalName);
if (!mdl.open())
return false;
return mdl.ConvertToVMAPModel(output.c_str());
stateGuard->Complete();
return stateGuard.release();
}
extern std::shared_ptr<CASC::Storage> CascStorage;
@@ -123,9 +134,12 @@ void ExtractGameobjectModels()
std::string_view header(headerRaw.data(), headerRaw.size());
if (header == "REVM")
result = ExtractSingleWmo(fileName);
{
ExtractedModelData const* wmo = ExtractSingleWmo(fileName);
result = wmo && wmo->HasCollision();
}
else if (header == "MD20" || header == "MD21")
result = ExtractSingleModel(fileName);
result = ExtractSingleModel(fileName) != nullptr;
else if (header == "BLP2")
continue; // broken db2 data
else

28
src/tools/vmap4_extractor/model.cpp
View File

@@ -159,21 +159,7 @@ Vec3D fixCoordSystem(Vec3D const& v)
void Doodad::Extract(ADT::MDDF const& doodadDef, char const* ModelInstName, uint32 mapID, uint32 originalMapId, FILE* pDirfile, std::vector<ADTOutputCache>* dirfileCache)
{
std::string tempname = Trinity::StringFormat("{}/{}", szWorkDirWmo, ModelInstName);
FILE* input = fopen(tempname.c_str(), "r+b");
if (!input)
return;
fseek(input, 8, SEEK_SET); // get the correct no of vertices
int nVertices;
int count = fread(&nVertices, sizeof(int), 1, input);
fclose(input);
if (count != 1 || nVertices == 0)
return;
// scale factor - divide by 1024. blizzard devs must be on crack, why not just use a float?
// scale factor - divide by 1024
float sc = doodadDef.Scale / 1024.0f;
Vec3D position = fixCoords(doodadDef.Position);
@@ -261,17 +247,7 @@ void Doodad::ExtractSet(WMODoodadData const& doodadData, ADT::MODF const& wmo, b
nlen = ModelInstName.length();
}
std::string tempname = Trinity::StringFormat("{}/{}", szWorkDirWmo, ModelInstName);
FILE* input = fopen(tempname.c_str(), "r+b");
if (!input)
continue;
fseek(input, 8, SEEK_SET); // get the correct no of vertices
int nVertices;
int count = fread(&nVertices, sizeof(int), 1, input);
fclose(input);
if (count != 1 || nVertices == 0)
if (!ExtractSingleModel(ModelInstName))
continue;
ASSERT(doodadId < std::numeric_limits<uint16>::max());

212
src/tools/vmap4_extractor/vmapexport.cpp
View File

@@ -24,7 +24,10 @@
#include "Locales.h"
#include "MapDefines.h"
#include "MapUtils.h"
#include "Memory.h"
#include "StringConvert.h"
#include "StringFormat.h"
#include "ThreadPool.h"
#include "Util.h"
#include "VMapDefinitions.h"
#include "wdtfile.h"
@@ -33,7 +36,6 @@
#include <boost/filesystem/directory.hpp>
#include <boost/filesystem/operations.hpp>
#include <algorithm>
#include <fstream>
#include <list>
#include <map>
#include <unordered_map>
@@ -62,19 +64,21 @@ struct MapEntry
int16 ParentMapID = 0;
std::string Name;
std::string Directory;
uint32 ChildDepth = 0;
bool IsParent = false;
};
std::unordered_map<uint32, LiquidMaterialEntry> LiquidMaterials;
std::unordered_map<uint32, LiquidTypeEntry> LiquidTypes;
std::vector<MapEntry> map_ids; // partitioned by parent maps first
std::unordered_set<uint32> maps_that_are_parents;
std::vector<MapEntry> map_ids;
boost::filesystem::path input_path;
bool preciseVectorData = false;
char const* CascProduct = "wow";
char const* CascRegion = "eu";
bool UseRemoteCasc = false;
uint32 DbcLocale = 0;
std::unordered_map<std::string, WMODoodadData> WmoDoodads;
uint32 Threads = std::thread::hardware_concurrency();
// Constants
@@ -171,38 +175,51 @@ uint32 GetInstalledLocalesMask()
return 0;
}
uint32 uniqueObjectIdGenerator = std::numeric_limits<uint32>::max() - 1;
std::map<std::pair<uint32, uint16>, uint32> uniqueObjectIds;
static std::atomic<uint32> UniqueObjectIdGenerator = std::numeric_limits<uint32>::max() - 1;
static std::mutex UniqueObjectIdsMutex;
static std::map<std::pair<uint32, uint16>, uint32> UniqueObjectIds;
uint32 GenerateUniqueObjectId(uint32 clientId, uint16 clientDoodadId, bool isWmo)
{
// WMO client ids must be preserved, they are used in DB2 files
uint32 newId = isWmo ? clientId : uniqueObjectIdGenerator--;
return uniqueObjectIds.emplace(std::make_pair(clientId, clientDoodadId), newId).first->second;
uint32 newId = isWmo ? clientId : UniqueObjectIdGenerator--;
std::lock_guard lock(UniqueObjectIdsMutex);
return UniqueObjectIds.emplace(std::make_pair(clientId, clientDoodadId), newId).first->second;
}
// Local testing functions
bool FileExists(char const* file)
static std::mutex ExtractedModelsMutex;
std::unordered_map<std::string, ExtractedModelData> ExtractedModels;
std::pair<ExtractedModelData*, bool> BeginModelExtraction(std::string const& outputName)
{
if (FILE* n = fopen(file, "rb"))
{
fclose(n);
return true;
}
return false;
std::lock_guard lock(ExtractedModelsMutex);
auto [itr, isNew] = ExtractedModels.try_emplace(outputName);
return { &itr->second, isNew };
}
bool ExtractSingleWmo(std::string& fname)
ExtractedModelData const* ExtractSingleWmo(std::string& fname)
{
// Copy files from archive
std::string originalName = fname;
char* plain_name = GetPlainName(&fname[0]);
NormalizeFileName(plain_name, strlen(plain_name));
std::string szLocalFile = Trinity::StringFormat("{}/{}", szWorkDirWmo, plain_name);
if (FileExists(szLocalFile.c_str()))
return true;
auto [model, shouldExtract] = BeginModelExtraction(plain_name);
if (!shouldExtract)
{
model->Wait();
switch (model->State.load(std::memory_order::relaxed))
{
case ExtractedModelData::Ok:
case ExtractedModelData::OkNoCollision:
return model;
default:
return nullptr;
}
}
auto stateGuard = Trinity::make_unique_ptr_with_deleter<&ExtractedModelData::Fail>(model);
int p = 0;
// Select root wmo files
@@ -213,23 +230,24 @@ bool ExtractSingleWmo(std::string& fname)
p++;
if (p == 3)
return true;
return nullptr;
bool file_ok = true;
WMORoot froot(originalName);
if (!froot.open())
{
printf("Couldn't open RootWmo!!!\n");
return true;
return nullptr;
}
FILE *output = fopen(szLocalFile.c_str(),"wb");
std::string szLocalFile = Trinity::StringFormat("{}/{}", szWorkDirWmo, plain_name);
FILE* output = fopen(szLocalFile.c_str(), "wb");
if(!output)
{
printf("couldn't open %s for writing!\n", szLocalFile.c_str());
return false;
return nullptr;
}
froot.ConvertToVMAPRootWmo(output);
WMODoodadData& doodads = WmoDoodads[plain_name];
WMODoodadData& doodads = *(model->Doodads = std::make_unique<WMODoodadData>());
std::swap(doodads, froot.DoodadData);
int Wmo_nVertices = 0;
uint32 groupCount = 0;
@@ -266,7 +284,7 @@ bool ExtractSingleWmo(std::string& fname)
continue;
uint32 doodadNameIndex = doodads.Spawns[groupReference].NameIndex;
if (froot.ValidDoodadNames.find(doodadNameIndex) == froot.ValidDoodadNames.end())
if (!froot.ValidDoodadNames.contains(doodadNameIndex))
continue;
doodads.References.insert(groupReference);
@@ -279,10 +297,18 @@ bool ExtractSingleWmo(std::string& fname)
fwrite(&groupCount, sizeof(uint32), 1, output);
fclose(output);
// Delete the extracted file in the case of an error
if (!file_ok)
if (!Wmo_nVertices && (doodads.Sets.empty() || doodads.References.empty()))
file_ok = false;
// Delete the extracted file in the case of an error or no collision
if (!file_ok || !Wmo_nVertices)
remove(szLocalFile.c_str());
return true;
if (!file_ok)
return nullptr;
stateGuard->Complete(Wmo_nVertices ? ExtractedModelData::Ok : ExtractedModelData::OkNoCollision);
return stateGuard.release();
}
bool IsLiquidIgnored(uint32 liquidTypeId)
@@ -298,62 +324,75 @@ bool IsLiquidIgnored(uint32 liquidTypeId)
void ParsMapFiles()
{
std::unordered_map<uint32, WDTFile> wdts;
auto getWDT = [&wdts](uint32 mapId) -> WDTFile*
{
auto itr = wdts.find(mapId);
if (itr == wdts.end())
{
auto mapEntryItr = std::ranges::find(map_ids, mapId, &MapEntry::Id);
if (mapEntryItr == map_ids.end())
return nullptr;
uint32 fileDataId = mapEntryItr->WdtFileDataId;
if (!fileDataId)
return nullptr;
std::string description = Trinity::StringFormat("WDT for map {} - {} (FileDataID {})", mapId, mapEntryItr->Name, fileDataId);
std::string directory = mapEntryItr->Directory;
itr = wdts.emplace(std::piecewise_construct, std::forward_as_tuple(mapId), std::forward_as_tuple(fileDataId, description, std::move(directory), maps_that_are_parents.count(mapId) > 0)).first;
if (!itr->second.init(mapId))
{
wdts.erase(itr);
return nullptr;
}
}
return &itr->second;
};
std::map<uint32, std::vector<MapEntry const*>> steps;
for (MapEntry const& mapEntry : map_ids)
{
if (WDTFile* WDT = getWDT(mapEntry.Id))
steps[mapEntry.ChildDepth].push_back(&mapEntry);
// preload WDTs
std::string description = Trinity::StringFormat("WDT for map {} - {} (FileDataID {})", mapEntry.Id, mapEntry.Name, mapEntry.WdtFileDataId);
auto itr = wdts.try_emplace(mapEntry.Id, mapEntry.WdtFileDataId, description, mapEntry.Directory, mapEntry.IsParent).first;
if (!itr->second.init(mapEntry.Id))
wdts.erase(itr);
}
for (auto const& [_, maps] : steps)
{
Trinity::ThreadPool threadPool(Threads);
for (MapEntry const* mapEntry : maps)
{
WDTFile* parentWDT = mapEntry.ParentMapID >= 0 ? getWDT(mapEntry.ParentMapID) : nullptr;
printf("Processing Map %u\n[", mapEntry.Id);
for (int32 x = 0; x < 64; ++x)
threadPool.PostWork([mapEntry, &wdts]
{
for (int32 y = 0; y < 64; ++y)
if (WDTFile* WDT = Trinity::Containers::MapGetValuePtr(wdts, mapEntry->Id))
{
bool success = false;
if (ADTFile* ADT = WDT->GetMap(x, y))
int16 parentMapId = mapEntry->ParentMapID;
std::vector<WDTFile*> parentWDTs;
while (parentMapId >= 0)
{
success = ADT->init(mapEntry.Id, mapEntry.Id);
WDT->FreeADT(ADT);
parentWDTs.push_back(Trinity::Containers::MapGetValuePtr(wdts, mapEntry->ParentMapID));
auto parentMapItr = std::ranges::find(map_ids, uint32(parentMapId), &MapEntry::Id);
if (parentMapItr == map_ids.end())
break;
parentMapId = parentMapItr->ParentMapID;
}
if (!success && parentWDT)
printf("Processing Map %u\n", mapEntry->Id);
for (int32 x = 0; x < 64; ++x)
{
if (ADTFile* ADT = parentWDT->GetMap(x, y))
for (int32 y = 0; y < 64; ++y)
{
ADT->init(mapEntry.Id, mapEntry.ParentMapID);
parentWDT->FreeADT(ADT);
bool success = false;
if (ADTFile* ADT = WDT->GetMap(x, y, true))
{
success = ADT->init(mapEntry->Id, mapEntry->Id);
WDT->FreeADT(ADT);
}
if (!success)
{
for (WDTFile* parentWDT : parentWDTs)
{
if (ADTFile* ADT = parentWDT->GetMap(x, y, false))
{
success = ADT->init(mapEntry->Id, mapEntry->ParentMapID);
parentWDT->FreeADT(ADT);
}
if (success)
break;
}
}
}
}
printf("Processing Map %u Done\n", mapEntry->Id);
}
printf("#");
fflush(stdout);
}
printf("]\n");
});
}
threadPool.Join();
}
}
@@ -396,9 +435,6 @@ void ReadMapTable()
if (map.ParentMapID < 0)
map.ParentMapID = int16(record.GetUInt16("CosmeticParentMapID"));
if (map.ParentMapID >= 0)
maps_that_are_parents.insert(map.ParentMapID);
idToIndex[map.Id] = map_ids.size() - 1;
}
@@ -413,10 +449,22 @@ void ReadMapTable()
}
}
std::erase_if(map_ids, [](MapEntry const& map) { return !map.WdtFileDataId; });
// force parent maps to be extracted first
std::stable_partition(map_ids.begin(), map_ids.end(), [](MapEntry const& map) { return maps_that_are_parents.contains(map.Id); });
for (MapEntry& map : map_ids)
{
int16 parentMapId = map.ParentMapID;
while (parentMapId >= 0)
{
++map.ChildDepth;
MapEntry& parent = map_ids[idToIndex[parentMapId]];
parent.IsParent = true;
parentMapId = parent.ParentMapID;
}
}
std::erase_if(map_ids, [](MapEntry const& map) { return !map.WdtFileDataId; });
printf("Done! (" SZFMTD " maps loaded)\n", map_ids.size());
}
@@ -530,6 +578,13 @@ bool processArgv(int argc, char ** argv, const char *versionString)
else
result = false;
}
else if (strcmp("--threads", argv[i]) == 0)
{
if (i + 1 < argc && strlen(argv[i + 1]))
Threads = Trinity::StringTo<uint32>(argv[++i]).value_or(std::thread::hardware_concurrency());
else
result = false;
}
else
{
result = false;
@@ -548,6 +603,7 @@ bool processArgv(int argc, char ** argv, const char *versionString)
printf(" -c use remote casc\n");
printf(" -r set remote casc region - standard: eu\n");
printf(" -dl dbc locale\n");
printf(" --threads <N> number of threads to use, default: all cpu cores\n");
printf(" -? : This message.\n");
}

42
src/tools/vmap4_extractor/vmapexport.h
View File

@@ -19,6 +19,8 @@
#define VMAPEXPORT_H
#include "Define.h"
#include <atomic>
#include <memory>
#include <string>
#include <unordered_map>
@@ -40,14 +42,46 @@ enum class ModelFlags : uint32
struct WMODoodadData;
extern const char * szWorkDirWmo;
extern std::unordered_map<std::string, WMODoodadData> WmoDoodads;
uint32 GenerateUniqueObjectId(uint32 clientId, uint16 clientDoodadId, bool isWmo);
bool FileExists(const char * file);
struct ExtractedModelData
{
enum ExtractionState : uint8
{
InProgress,
Ok,
OkNoCollision, // has no data by itself but its WMO doodads do
Failed
};
bool ExtractSingleWmo(std::string& fname);
bool ExtractSingleModel(std::string& fname);
std::atomic<ExtractionState> State;
std::unique_ptr<WMODoodadData> Doodads;
void Wait()
{
State.wait(InProgress);
}
void Fail()
{
State.store(Failed);
State.notify_all();
}
void Complete(ExtractionState state = Ok)
{
State.store(state);
State.notify_all();
}
bool HasCollision() const { return State.load(std::memory_order::relaxed) == Ok; }
};
std::pair<ExtractedModelData*, bool> BeginModelExtraction(std::string const& outputName);
ExtractedModelData const* ExtractSingleWmo(std::string& fname);
ExtractedModelData const* ExtractSingleModel(std::string& fname);
void ExtractGameobjectModels();

31
src/tools/vmap4_extractor/wdtfile.cpp
View File

@@ -27,17 +27,9 @@
extern std::shared_ptr<CASC::Storage> CascStorage;
WDTFile::WDTFile(uint32 fileDataId, std::string const& description, std::string mapName, bool cache)
: _file(CascStorage, fileDataId, description), _mapName(std::move(mapName))
: _file(CascStorage, fileDataId, description), _header(), _adtInfo(), _mapName(std::move(mapName)),
_adtCache(cache ? std::make_unique<ADTCache>() : nullptr)
{
memset(&_header, 0, sizeof(WDT::MPHD));
memset(&_adtInfo, 0, sizeof(WDT::MAIN));
if (cache)
{
_adtCache = std::make_unique<ADTCache>();
memset(_adtCache->file, 0, sizeof(_adtCache->file));
}
else
_adtCache = nullptr;
}
WDTFile::~WDTFile() = default;
@@ -116,17 +108,21 @@ bool WDTFile::init(uint32 mapId)
{
ADT::MODF mapObjDef;
_file.read(&mapObjDef, sizeof(ADT::MODF));
std::string fileName;
if (mapObjDef.Flags & 0x8)
{
fileName = Trinity::StringFormat("FILE{:08X}.xxx", mapObjDef.Id);
ExtractSingleWmo(fileName);
}
else
fileName = _wmoNames[mapObjDef.Id];
MapObject::Extract(mapObjDef, fileName.c_str(), true, mapId, mapId, dirfile.get(), nullptr);
Doodad::ExtractSet(WmoDoodads[fileName], mapObjDef, true, mapId, mapId, dirfile.get(), nullptr);
if (ExtractedModelData const* extracted = ExtractSingleWmo(fileName))
{
if (extracted->HasCollision())
MapObject::Extract(mapObjDef, fileName.c_str(), true, mapId, mapId, dirfile.get(), nullptr);
if (extracted->Doodads)
Doodad::ExtractSet(*extracted->Doodads, mapObjDef, true, mapId, mapId, dirfile.get(), nullptr);
}
}
}
}
@@ -137,7 +133,7 @@ bool WDTFile::init(uint32 mapId)
return true;
}
ADTFile* WDTFile::GetMap(int32 x, int32 y)
ADTFile* WDTFile::GetMap(int32 x, int32 y, bool createIfMissing)
{
if (!(x >= 0 && y >= 0 && x < 64 && y < 64))
return nullptr;
@@ -148,6 +144,9 @@ ADTFile* WDTFile::GetMap(int32 x, int32 y)
if (!(_adtInfo.Data[y][x].Flag & 1))
return nullptr;
if (!createIfMissing)
return nullptr;
ADTFile* adt;
std::string name = Trinity::StringFormat(R"(World\Maps\{}\{}_{}_{}_obj0.adt)", _mapName, _mapName, x, y);
if (_header.Flags & 0x200)

2
src/tools/vmap4_extractor/wdtfile.h
View File

@@ -74,7 +74,7 @@ public:
~WDTFile();
bool init(uint32 mapId);
ADTFile* GetMap(int32 x, int32 y);
ADTFile* GetMap(int32 x, int32 y, bool createIfMissing);
void FreeADT(ADTFile* adt);
private:
CASCFile _file;

17
src/tools/vmap4_extractor/wmo.cpp
View File

@@ -589,23 +589,6 @@ void MapObject::Extract(ADT::MODF const& mapObjDef, char const* WmoInstName, boo
{
//-----------add_in _dir_file----------------
std::string tempname = Trinity::StringFormat("{}/{}", szWorkDirWmo, WmoInstName);
FILE* input = fopen(tempname.c_str(), "r+b");
if (!input)
{
printf("WMOInstance::WMOInstance: couldn't open %s\n", tempname.c_str());
return;
}
fseek(input, 8, SEEK_SET); // get the correct no of vertices
int nVertices;
int count = fread(&nVertices, sizeof(int), 1, input);
fclose(input);
if (count != 1 || nVertices == 0)
return;
Vec3D position = fixCoords(mapObjDef.Position);
AaBox3D bounds;
bounds.min = fixCoords(mapObjDef.Bounds.min);