aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorNaios <naios-dev@live.de>2015-03-21 20:11:23 +0100
committerNayd <dnpd.dd@gmail.com>2015-03-21 20:54:54 +0000
commit2bd1c8541d638f5041a92f491f02919270981bc2 (patch)
treec2472304f260f8f3553cc86b4de1e19042c01807 /src
parent236e43443c644a9a50cb48ff7300950c928d336c (diff)
Core/Updates: Add an automatic database update system. Automatically detects new and edited sql updates through file lists and hashing. Detects renames, deletes and is able to create and auto import full databases. * cleanups in main.cpp of world & bnetserver * refactoring in DatabaseWorkerPool.h & MySQLConnection.cpp
Make sure you re-run cmake, because boost::iostreams was added as dependency. Maybe you need to install libboost-iostreams1.55-dev on unix as well. Import every update manual until (included) those INSERT IGNORE updates for each database. Thanks DDuarte and Shauren for your amazing ideas, help and advises. In hope that nobody gets a "Your database structure is not up to date..." anymore ,-) (cherry picked from commit 352012e53173372ebc82898e1b6854c983b01b25) (cherry picked from commit 1f7f9feafc3f5017d617634ae993d2cdf4430920) For 3.3.5: * Synchronized ConfigureBoost.cmake with 6.x, libboost-filesystem1.55-dev also added as dependency! Signed-off-by: Naios <naios-dev@live.de> Signed-off-by: Nayd <dnpd.dd@gmail.com>
Diffstat (limited to 'src')
-rw-r--r--src/server/authserver/CMakeLists.txt2
-rw-r--r--src/server/authserver/Main.cpp33
-rw-r--r--src/server/authserver/authserver.conf.dist81
-rw-r--r--src/server/game/CMakeLists.txt1
-rw-r--r--src/server/scripts/CMakeLists.txt1
-rw-r--r--src/server/shared/CMakeLists.txt4
-rw-r--r--src/server/shared/Database/DatabaseLoader.cpp191
-rw-r--r--src/server/shared/Database/DatabaseLoader.h71
-rw-r--r--src/server/shared/Database/DatabaseWorkerPool.h91
-rw-r--r--src/server/shared/Database/MySQLConnection.cpp8
-rw-r--r--src/server/shared/Database/MySQLConnection.h5
-rw-r--r--src/server/shared/Updater/DBUpdater.cpp381
-rw-r--r--src/server/shared/Updater/DBUpdater.h79
-rw-r--r--src/server/shared/Updater/UpdateFetcher.cpp387
-rw-r--r--src/server/shared/Updater/UpdateFetcher.h128
-rw-r--r--src/server/worldserver/CMakeLists.txt2
-rw-r--r--src/server/worldserver/Main.cpp80
-rw-r--r--src/server/worldserver/worldserver.conf.dist85
18 files changed, 1498 insertions, 132 deletions
diff --git a/src/server/authserver/CMakeLists.txt b/src/server/authserver/CMakeLists.txt
index 077cd15404b..994a645c528 100644
--- a/src/server/authserver/CMakeLists.txt
+++ b/src/server/authserver/CMakeLists.txt
@@ -44,6 +44,7 @@ endif()
include_directories(
${CMAKE_BINARY_DIR}
${CMAKE_SOURCE_DIR}/dep/cppformat
+ ${CMAKE_SOURCE_DIR}/dep/process
${CMAKE_SOURCE_DIR}/src/server/shared
${CMAKE_SOURCE_DIR}/src/server/shared/Configuration
${CMAKE_SOURCE_DIR}/src/server/shared/Database
@@ -54,6 +55,7 @@ include_directories(
${CMAKE_SOURCE_DIR}/src/server/shared/Logging
${CMAKE_SOURCE_DIR}/src/server/shared/Networking
${CMAKE_SOURCE_DIR}/src/server/shared/Threading
+ ${CMAKE_SOURCE_DIR}/src/server/shared/Updater
${CMAKE_SOURCE_DIR}/src/server/shared/Utilities
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/Authentication
diff --git a/src/server/authserver/Main.cpp b/src/server/authserver/Main.cpp
index cd58ec2bf68..26f1c872150 100644
--- a/src/server/authserver/Main.cpp
+++ b/src/server/authserver/Main.cpp
@@ -28,6 +28,7 @@
#include "Common.h"
#include "Config.h"
#include "DatabaseEnv.h"
+#include "DatabaseLoader.h"
#include "Log.h"
#include "ProcessPriority.h"
#include "RealmList.h"
@@ -150,33 +151,15 @@ bool StartDB()
{
MySQL::Library_Init();
- std::string dbstring = sConfigMgr->GetStringDefault("LoginDatabaseInfo", "");
- if (dbstring.empty())
- {
- TC_LOG_ERROR("server.authserver", "Database not specified");
- return false;
- }
-
- int32 worker_threads = sConfigMgr->GetIntDefault("LoginDatabase.WorkerThreads", 1);
- if (worker_threads < 1 || worker_threads > 32)
- {
- TC_LOG_ERROR("server.authserver", "Improper value specified for LoginDatabase.WorkerThreads, defaulting to 1.");
- worker_threads = 1;
- }
+ // Load databases
+ // NOTE: While authserver is singlethreaded you should keep synch_threads == 1.
+ // Increasing it is just silly since only 1 will be used ever.
+ DatabaseLoader loader("server.authserver", DatabaseLoader::DATABASE_NONE);
+ loader
+ .AddDatabase(LoginDatabase, "Login");
- int32 synch_threads = sConfigMgr->GetIntDefault("LoginDatabase.SynchThreads", 1);
- if (synch_threads < 1 || synch_threads > 32)
- {
- TC_LOG_ERROR("server.authserver", "Improper value specified for LoginDatabase.SynchThreads, defaulting to 1.");
- synch_threads = 1;
- }
-
- // NOTE: While authserver is singlethreaded you should keep synch_threads == 1. Increasing it is just silly since only 1 will be used ever.
- if (!LoginDatabase.Open(dbstring, uint8(worker_threads), uint8(synch_threads)))
- {
- TC_LOG_ERROR("server.authserver", "Cannot connect to database");
+ if (!loader.Load())
return false;
- }
TC_LOG_INFO("server.authserver", "Started auth database connection pool.");
sLog->SetRealmId(0); // Enables DB appenders when realm is set.
diff --git a/src/server/authserver/authserver.conf.dist b/src/server/authserver/authserver.conf.dist
index b7dee9ac08b..6ba05e89d72 100644
--- a/src/server/authserver/authserver.conf.dist
+++ b/src/server/authserver/authserver.conf.dist
@@ -9,6 +9,7 @@
# EXAMPLE CONFIG
# AUTH SERVER SETTINGS
# MYSQL SETTINGS
+# UPDATE SETTINGS
# LOGGING SYSTEM SETTINGS
#
###################################################################################################
@@ -158,6 +159,86 @@ Wrong.Password.Login.Logging = 0
###################################################################################################
###################################################################################################
+# UPDATE SETTINGS
+#
+# Updates.EnableDatabases
+# Description: A mask that describes which databases shall be updated.
+#
+# Following flags are available
+# DATABASE_LOGIN = 1, // Auth database
+#
+# Default: 0 - (All Disabled)
+# 1 - (All Enabled)
+
+Updates.EnableDatabases = 0
+
+#
+# Updates.SourcePath
+# Description: The path to your TrinityCore source directory.
+# If the path is left empty, built-in CMAKE_SOURCE_DIR is used.
+# Example: "../TrinityCore"
+# Default: ""
+
+Updates.SourcePath = ""
+
+#
+# Updates.SourcePath
+# Description: The path to your mysql cli binary.
+# If the path is left empty, built-in path from cmake is used.
+# Example: "C:/Program Files/MySQL/MySQL Server 5.6/bin/mysql.exe"
+# "mysql.exe"
+# "/usr/bin/mysql"
+# Default: ""
+
+Updates.MySqlCLIPath = ""
+
+#
+# Updates.AutoSetup
+# Description: Auto populate empty databases.
+# Default: 1 - (Enabled)
+# 0 - (Disabled)
+
+Updates.AutoSetup = 1
+
+#
+# Updates.Redundancy
+# Description: Perform data redundancy checks through hashing
+# to detect changes on sql updates and reapply it.
+# Default: 1 - (Enabled)
+# 0 - (Disabled)
+
+Updates.Redundancy = 1
+
+#
+# Updates.ArchivedRedundancy
+# Description: Check hashes of archived updates (slows down startup).
+# Default: 0 - (Disabled)
+# 1 - (Enabled)
+
+Updates.ArchivedRedundancy = 0
+
+#
+# Updates.AllowRehash
+# Description: Inserts the current file hash in the database if it is left empty.
+# Useful if you want to mark a file as applied but you don't know its hash.
+# Default: 1 - (Enabled)
+# 0 - (Disabled)
+
+Updates.AllowRehash = 1
+
+#
+# Updates.CleanDeadRef
+# Description: Cleans dead/ orphaned references that occure if a update was deleted or renamed and edited.
+# Disable this if you want to know if the database is in a possible "dirty state".
+# Default: 1 - (Enabled)
+# 0 - (Disabled)
+
+Updates.CleanDeadRef = 1
+
+#
+###################################################################################################
+
+###################################################################################################
#
# LOGGING SYSTEM SETTINGS
#
diff --git a/src/server/game/CMakeLists.txt b/src/server/game/CMakeLists.txt
index 2f7bc4a9c69..fc9265b6ff4 100644
--- a/src/server/game/CMakeLists.txt
+++ b/src/server/game/CMakeLists.txt
@@ -129,6 +129,7 @@ include_directories(
${CMAKE_SOURCE_DIR}/src/server/shared/Networking
${CMAKE_SOURCE_DIR}/src/server/shared/Packets
${CMAKE_SOURCE_DIR}/src/server/shared/Threading
+ ${CMAKE_SOURCE_DIR}/src/server/shared/Updater
${CMAKE_SOURCE_DIR}/src/server/shared/Utilities
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/Accounts
diff --git a/src/server/scripts/CMakeLists.txt b/src/server/scripts/CMakeLists.txt
index 8bf84c32447..875e2c240ca 100644
--- a/src/server/scripts/CMakeLists.txt
+++ b/src/server/scripts/CMakeLists.txt
@@ -63,6 +63,7 @@ include_directories(
${CMAKE_SOURCE_DIR}/src/server/shared/Logging
${CMAKE_SOURCE_DIR}/src/server/shared/Packets
${CMAKE_SOURCE_DIR}/src/server/shared/Threading
+ ${CMAKE_SOURCE_DIR}/src/server/shared/Updater
${CMAKE_SOURCE_DIR}/src/server/shared/Utilities
${CMAKE_SOURCE_DIR}/src/server/collision
${CMAKE_SOURCE_DIR}/src/server/collision/Management
diff --git a/src/server/shared/CMakeLists.txt b/src/server/shared/CMakeLists.txt
index 5f319191c14..4be94e334da 100644
--- a/src/server/shared/CMakeLists.txt
+++ b/src/server/shared/CMakeLists.txt
@@ -21,6 +21,7 @@ file(GLOB_RECURSE sources_Logging Logging/*.cpp Logging/*.h)
file(GLOB_RECURSE sources_Networking Networking/*.cpp Networking/*.h)
file(GLOB_RECURSE sources_Packets Packets/*.cpp Packets/*.h)
file(GLOB_RECURSE sources_Threading Threading/*.cpp Threading/*.h)
+file(GLOB_RECURSE sources_Updater Updater/*.cpp Updater/*.h)
file(GLOB_RECURSE sources_Utilities Utilities/*.cpp Utilities/*.h)
file(GLOB sources_localdir *.cpp *.h)
@@ -51,6 +52,7 @@ set(shared_STAT_SRCS
${sources_Networking}
${sources_Packets}
${sources_Threading}
+ ${sources_Updater}
${sources_Utilities}
${sources_localdir}
)
@@ -61,6 +63,7 @@ include_directories(
${CMAKE_SOURCE_DIR}/dep/SFMT
${CMAKE_SOURCE_DIR}/dep/cppformat
${CMAKE_SOURCE_DIR}/dep/utf8cpp
+ ${CMAKE_SOURCE_DIR}/dep/process
${CMAKE_SOURCE_DIR}/src/server
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/Configuration
@@ -74,6 +77,7 @@ include_directories(
${CMAKE_CURRENT_SOURCE_DIR}/Packets
${CMAKE_CURRENT_SOURCE_DIR}/Threading
${CMAKE_CURRENT_SOURCE_DIR}/Utilities
+ ${CMAKE_CURRENT_SOURCE_DIR}/Updater
${CMAKE_SOURCE_DIR}/src/server/game/Entities/Object
${MYSQL_INCLUDE_DIR}
${OPENSSL_INCLUDE_DIR}
diff --git a/src/server/shared/Database/DatabaseLoader.cpp b/src/server/shared/Database/DatabaseLoader.cpp
new file mode 100644
index 00000000000..36ee4b12c83
--- /dev/null
+++ b/src/server/shared/Database/DatabaseLoader.cpp
@@ -0,0 +1,191 @@
+/*
+ * Copyright (C) 2008-2015 TrinityCore <http://www.trinitycore.org/>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "DatabaseLoader.h"
+#include "DBUpdater.h"
+#include "Config.h"
+
+#include <mysqld_error.h>
+
+DatabaseLoader::DatabaseLoader(std::string const& logger, uint32 const defaultUpdateMask)
+ : _logger(logger), _autoSetup(sConfigMgr->GetBoolDefault("Updates.AutoSetup", true)),
+ _updateFlags(sConfigMgr->GetIntDefault("Updates.EnableDatabases", defaultUpdateMask))
+{
+}
+
+template <class T>
+DatabaseLoader& DatabaseLoader::AddDatabase(DatabaseWorkerPool<T>& pool, std::string const& name)
+{
+ bool const updatesEnabledForThis = DBUpdater<T>::IsEnabled(_updateFlags);
+
+ _open.push(std::make_pair([this, name, updatesEnabledForThis, &pool]() -> bool
+ {
+ std::string const dbString = sConfigMgr->GetStringDefault(name + "DatabaseInfo", "");
+ if (dbString.empty())
+ {
+ TC_LOG_ERROR(_logger.c_str(), "Database %s not specified in configuration file!", name.c_str());
+ return false;
+ }
+
+ uint8 const asyncThreads = uint8(sConfigMgr->GetIntDefault(name + "Database.WorkerThreads", 1));
+ if (asyncThreads < 1 || asyncThreads > 32)
+ {
+ TC_LOG_ERROR(_logger.c_str(), "%s database: invalid number of worker threads specified. "
+ "Please pick a value between 1 and 32.", name.c_str());
+ return false;
+ }
+
+ uint8 const synchThreads = uint8(sConfigMgr->GetIntDefault(name + "Database.SynchThreads", 1));
+
+ pool.SetConnectionInfo(dbString, asyncThreads, synchThreads);
+ if (uint32 error = pool.Open())
+ {
+ // Database does not exist
+ if ((error == ER_BAD_DB_ERROR) && updatesEnabledForThis && _autoSetup)
+ {
+ // Try to create the database and connect again if auto setup is enabled
+ if (DBUpdater<T>::Create(pool) && (!pool.Open()))
+ error = 0;
+ }
+
+ // If the error wasn't handled quit
+ if (error)
+ {
+ TC_LOG_ERROR("sql.driver", "\nDatabasePool %s NOT opened. There were errors opening the MySQL connections. Check your SQLDriverLogFile "
+ "for specific errors. Read wiki at http://collab.kpsn.org/display/tc/TrinityCore+Home", name.c_str());
+
+ return false;
+ }
+ }
+ return true;
+ },
+ [&pool]()
+ {
+ pool.Close();
+ }));
+
+ // Populate and update only if updates are enabled for this pool
+ if (updatesEnabledForThis)
+ {
+ _populate.push([this, name, &pool]() -> bool
+ {
+ if (!DBUpdater<T>::Populate(pool))
+ {
+ TC_LOG_ERROR(_logger.c_str(), "Could not populate the %s database, see log for details.", name.c_str());
+ return false;
+ }
+ return true;
+ });
+
+ _update.push([this, name, &pool]() -> bool
+ {
+ if (!DBUpdater<T>::Update(pool))
+ {
+ TC_LOG_ERROR(_logger.c_str(), "Could not update the %s database, see log for details.", name.c_str());
+ return false;
+ }
+ return true;
+ });
+ }
+
+ _prepare.push([this, name, &pool]() -> bool
+ {
+ if (!pool.PrepareStatements())
+ {
+ TC_LOG_ERROR(_logger.c_str(), "Could not prepare statements of the %s database, see log for details.", name.c_str());
+ return false;
+ }
+ return true;
+ });
+
+ return *this;
+}
+
+bool DatabaseLoader::Load()
+{
+ if (!OpenDatabases())
+ return false;
+
+ if (!PopulateDatabases())
+ return false;
+
+ if (!UpdateDatabases())
+ return false;
+
+ if (!PrepareStatements())
+ return false;
+
+ return true;
+}
+
+bool DatabaseLoader::OpenDatabases()
+{
+ while (!_open.empty())
+ {
+ std::pair<Predicate, std::function<void()>> const load = _open.top();
+ if (load.first())
+ _close.push(load.second);
+ else
+ {
+ // Close all loaded databases
+ while (!_close.empty())
+ {
+ _close.top()();
+ _close.pop();
+ }
+ return false;
+ }
+
+ _open.pop();
+ }
+ return true;
+}
+
+// Processes the elements of the given stack until a predicate returned false.
+bool DatabaseLoader::Process(std::stack<Predicate>& stack)
+{
+ while (!stack.empty())
+ {
+ if (!stack.top()())
+ return false;
+
+ stack.pop();
+ }
+ return true;
+}
+
+bool DatabaseLoader::PopulateDatabases()
+{
+ return Process(_populate);
+}
+
+bool DatabaseLoader::UpdateDatabases()
+{
+ return Process(_update);
+}
+
+bool DatabaseLoader::PrepareStatements()
+{
+ return Process(_prepare);
+}
+
+template
+DatabaseLoader& DatabaseLoader::AddDatabase<LoginDatabaseConnection>(DatabaseWorkerPool<LoginDatabaseConnection>& pool, std::string const& name);
+template
+DatabaseLoader& DatabaseLoader::AddDatabase<WorldDatabaseConnection>(DatabaseWorkerPool<WorldDatabaseConnection>& pool, std::string const& name);
+template
+DatabaseLoader& DatabaseLoader::AddDatabase<CharacterDatabaseConnection>(DatabaseWorkerPool<CharacterDatabaseConnection>& pool, std::string const& name);
diff --git a/src/server/shared/Database/DatabaseLoader.h b/src/server/shared/Database/DatabaseLoader.h
new file mode 100644
index 00000000000..d35597ba807
--- /dev/null
+++ b/src/server/shared/Database/DatabaseLoader.h
@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) 2008-2015 TrinityCore <http://www.trinitycore.org/>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef DatabaseLoader_h__
+#define DatabaseLoader_h__
+
+#include "DatabaseWorkerPool.h"
+#include "DatabaseEnv.h"
+
+#include <stack>
+#include <functional>
+
+// A helper class to initiate all database worker pools,
+// handles updating, delays preparing of statements and cleans up on failure.
+class DatabaseLoader
+{
+public:
+ DatabaseLoader(std::string const& logger, uint32 const defaultUpdateMask);
+
+ // Register a database to the loader (lazy implemented)
+ template <class T>
+ DatabaseLoader& AddDatabase(DatabaseWorkerPool<T>& pool, std::string const& name);
+
+ // Load all databases
+ bool Load();
+
+ enum DatabaseTypeFlags
+ {
+ DATABASE_NONE = 0,
+
+ DATABASE_LOGIN = 1,
+ DATABASE_CHARACTER = 2,
+ DATABASE_WORLD = 4,
+
+ DATABASE_MASK_ALL = DATABASE_LOGIN | DATABASE_CHARACTER | DATABASE_WORLD
+ };
+
+private:
+ bool OpenDatabases();
+ bool PopulateDatabases();
+ bool UpdateDatabases();
+ bool PrepareStatements();
+
+ using Predicate = std::function<bool()>;
+
+ static bool Process(std::stack<Predicate>& stack);
+
+ std::string const _logger;
+ bool const _autoSetup;
+ uint32 const _updateFlags;
+
+ std::stack<std::pair<Predicate, std::function<void()>>> _open;
+ std::stack<std::function<void()>> _close;
+ std::stack<Predicate> _populate, _update, _prepare;
+};
+
+#endif // DatabaseLoader_h__
diff --git a/src/server/shared/Database/DatabaseWorkerPool.h b/src/server/shared/Database/DatabaseWorkerPool.h
index f0ddbe91ad8..af4d5fc9c73 100644
--- a/src/server/shared/Database/DatabaseWorkerPool.h
+++ b/src/server/shared/Database/DatabaseWorkerPool.h
@@ -30,6 +30,7 @@
#include "AdhocStatement.h"
#include <mysqld_error.h>
+#include <memory>
#define MIN_MYSQL_SERVER_VERSION 50100u
#define MIN_MYSQL_CLIENT_VERSION 50100u
@@ -57,9 +58,9 @@ class DatabaseWorkerPool
public:
/* Activity state */
- DatabaseWorkerPool() : _connectionInfo(NULL)
+ DatabaseWorkerPool() : _connectionInfo(nullptr), _queue(new ProducerConsumerQueue<SQLOperation*>()),
+ _async_threads(0), _synch_threads(0)
{
- _queue = new ProducerConsumerQueue<SQLOperation*>();
memset(_connectionCount, 0, sizeof(_connectionCount));
_connections.resize(IDX_SIZE);
@@ -70,31 +71,37 @@ class DatabaseWorkerPool
~DatabaseWorkerPool()
{
_queue->Cancel();
+ }
- delete _queue;
+ void SetConnectionInfo(std::string const& infoString, uint8 const asyncThreads, uint8 const synchThreads)
+ {
+ _connectionInfo.reset(new MySQLConnectionInfo(infoString));
- delete _connectionInfo;
+ _async_threads = asyncThreads;
+ _synch_threads = synchThreads;
}
- bool Open(const std::string& infoString, uint8 async_threads, uint8 synch_threads)
+ uint32 Open()
{
- _connectionInfo = new MySQLConnectionInfo(infoString);
+ WPFatal(_connectionInfo.get(), "Connection info was not set!");
TC_LOG_INFO("sql.driver", "Opening DatabasePool '%s'. Asynchronous connections: %u, synchronous connections: %u.",
- GetDatabaseName(), async_threads, synch_threads);
+ GetDatabaseName(), _async_threads, _synch_threads);
- bool res = OpenConnections(IDX_ASYNC, async_threads);
+ uint32 error = OpenConnections(IDX_ASYNC, _async_threads);
- if (!res)
- return res;
+ if (error)
+ return error;
- res = OpenConnections(IDX_SYNCH, synch_threads);
+ error = OpenConnections(IDX_SYNCH, _synch_threads);
- if (res)
+ if (!error)
+ {
TC_LOG_INFO("sql.driver", "DatabasePool '%s' opened successfully. %u total connections running.", GetDatabaseName(),
(_connectionCount[IDX_SYNCH] + _connectionCount[IDX_ASYNC]));
+ }
- return res;
+ return error;
}
void Close()
@@ -120,6 +127,32 @@ class DatabaseWorkerPool
TC_LOG_INFO("sql.driver", "All connections on DatabasePool '%s' closed.", GetDatabaseName());
}
+ //! Prepares all prepared statements
+ bool PrepareStatements()
+ {
+ for (uint8 i = 0; i < IDX_SIZE; ++i)
+ for (uint32 c = 0; c < _connectionCount[i]; ++c)
+ {
+ T* t = _connections[i][c];
+ t->LockIfReady();
+ if (!t->PrepareStatements())
+ {
+ t->Unlock();
+ Close();
+ return false;
+ }
+ else
+ t->Unlock();
+ }
+
+ return true;
+ }
+
+ inline MySQLConnectionInfo const* GetConnectionInfo() const
+ {
+ return _connectionInfo.get();
+ }
+
/**
Delayed one-way statement methods.
*/
@@ -461,7 +494,7 @@ class DatabaseWorkerPool
}
private:
- bool OpenConnections(InternalIndex type, uint8 numConnections)
+ uint32 OpenConnections(InternalIndex type, uint8 numConnections)
{
_connections[type].resize(numConnections);
for (uint8 i = 0; i < numConnections; ++i)
@@ -469,7 +502,7 @@ class DatabaseWorkerPool
T* t;
if (type == IDX_ASYNC)
- t = new T(_queue, *_connectionInfo);
+ t = new T(_queue.get(), *_connectionInfo);
else if (type == IDX_SYNCH)
t = new T(*_connectionInfo);
else
@@ -478,35 +511,32 @@ class DatabaseWorkerPool
_connections[type][i] = t;
++_connectionCount[type];
- bool res = t->Open();
+ uint32 error = t->Open();
- if (res)
+ if (!error)
{
if (mysql_get_server_version(t->GetHandle()) < MIN_MYSQL_SERVER_VERSION)
{
TC_LOG_ERROR("sql.driver", "TrinityCore does not support MySQL versions below 5.1");
- res = false;
+ error = 1;
}
}
// Failed to open a connection or invalid version, abort and cleanup
- if (!res)
+ if (error)
{
- TC_LOG_ERROR("sql.driver", "DatabasePool %s NOT opened. There were errors opening the MySQL connections. Check your SQLDriverLogFile "
- "for specific errors. Read wiki at http://collab.kpsn.org/display/tc/TrinityCore+Home", GetDatabaseName());
-
while (_connectionCount[type] != 0)
{
T* t = _connections[type][i--];
delete t;
--_connectionCount[type];
}
-
- return false;
+ return error;
}
}
- return true;
+ // Everything is fine
+ return 0;
}
unsigned long EscapeString(char *to, const char *from, unsigned long length)
@@ -546,10 +576,13 @@ class DatabaseWorkerPool
return _connectionInfo->database.c_str();
}
- ProducerConsumerQueue<SQLOperation*>* _queue; //! Queue shared by async worker threads.
- std::vector< std::vector<T*> > _connections;
- uint32 _connectionCount[2]; //! Counter of MySQL connections;
- MySQLConnectionInfo* _connectionInfo;
+ //! Queue shared by async worker threads.
+ std::unique_ptr<ProducerConsumerQueue<SQLOperation*>> _queue;
+ std::vector<std::vector<T*>> _connections;
+ //! Counter of MySQL connections;
+ uint32 _connectionCount[IDX_SIZE];
+ std::unique_ptr<MySQLConnectionInfo> _connectionInfo;
+ uint8 _async_threads, _synch_threads;
};
#endif
diff --git a/src/server/shared/Database/MySQLConnection.cpp b/src/server/shared/Database/MySQLConnection.cpp
index 1a9f973d47b..1fa3f01a5e1 100644
--- a/src/server/shared/Database/MySQLConnection.cpp
+++ b/src/server/shared/Database/MySQLConnection.cpp
@@ -72,7 +72,7 @@ void MySQLConnection::Close()
delete this;
}
-bool MySQLConnection::Open()
+uint32 MySQLConnection::Open()
{
MYSQL *mysqlInit;
mysqlInit = mysql_init(NULL);
@@ -137,13 +137,13 @@ bool MySQLConnection::Open()
// set connection properties to UTF8 to properly handle locales for different
// server configs - core sends data in UTF8, so MySQL must expect UTF8 too
mysql_set_character_set(m_Mysql, "utf8");
- return PrepareStatements();
+ return 0;
}
else
{
- TC_LOG_ERROR("sql.sql", "Could not connect to MySQL database at %s: %s\n", m_connectionInfo.host.c_str(), mysql_error(mysqlInit));
+ TC_LOG_ERROR("sql.sql", "Could not connect to MySQL database at %s: %s", m_connectionInfo.host.c_str(), mysql_error(mysqlInit));
mysql_close(mysqlInit);
- return false;
+ return mysql_errno(mysqlInit);
}
}
diff --git a/src/server/shared/Database/MySQLConnection.h b/src/server/shared/Database/MySQLConnection.h
index d486f5b4679..78d8d2fb5dd 100644
--- a/src/server/shared/Database/MySQLConnection.h
+++ b/src/server/shared/Database/MySQLConnection.h
@@ -72,9 +72,11 @@ class MySQLConnection
MySQLConnection(ProducerConsumerQueue<SQLOperation*>* queue, MySQLConnectionInfo& connInfo); //! Constructor for asynchronous connections.
virtual ~MySQLConnection();
- virtual bool Open();
+ virtual uint32 Open();
void Close();
+ bool PrepareStatements();
+
public:
bool Execute(const char* sql);
bool Execute(PreparedStatement* stmt);
@@ -111,7 +113,6 @@ class MySQLConnection
MySQLPreparedStatement* GetPreparedStatement(uint32 index);
void PrepareStatement(uint32 index, const char* sql, ConnectionFlags flags);
- bool PrepareStatements();
virtual void DoPrepareStatements() = 0;
protected:
diff --git a/src/server/shared/Updater/DBUpdater.cpp b/src/server/shared/Updater/DBUpdater.cpp
new file mode 100644
index 00000000000..743dd37477f
--- /dev/null
+++ b/src/server/shared/Updater/DBUpdater.cpp
@@ -0,0 +1,381 @@
+/*
+ * Copyright (C) 2008-2015 TrinityCore <http://www.trinitycore.org/>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "DBUpdater.h"
+#include "Log.h"
+#include "revision.h"
+#include "UpdateFetcher.h"
+#include "DatabaseLoader.h"
+#include "Config.h"
+
+#include <fstream>
+#include <iostream>
+#include <unordered_map>
+#include <boost/process.hpp>
+#include <boost/process/mitigate.hpp>
+#include <boost/iostreams/device/file_descriptor.hpp>
+#include <boost/system/system_error.hpp>
+
+using namespace boost::process;
+using namespace boost::process::initializers;
+using namespace boost::iostreams;
+
+template<class T>
+std::string DBUpdater<T>::GetSourceDirectory()
+{
+ std::string const entry = sConfigMgr->GetStringDefault("Updates.SourcePath", "");
+ if (!entry.empty())
+ return entry;
+ else
+ return _SOURCE_DIRECTORY;
+}
+
+template<class T>
+std::string DBUpdater<T>::GetMySqlCli()
+{
+ std::string const entry = sConfigMgr->GetStringDefault("Updates.MySqlCLIPath", "");
+ if (!entry.empty())
+ return entry;
+ else
+ return _MYSQL_EXECUTABLE;
+}
+
+// Auth Database
+template<>
+std::string DBUpdater<LoginDatabaseConnection>::GetConfigEntry()
+{
+ return "Updates.Auth";
+}
+
+template<>
+std::string DBUpdater<LoginDatabaseConnection>::GetTableName()
+{
+ return "Auth";
+}
+
+template<>
+std::string DBUpdater<LoginDatabaseConnection>::GetBaseFile()
+{
+ return DBUpdater<LoginDatabaseConnection>::GetSourceDirectory() + "/sql/base/auth_database.sql";
+}
+
+template<>
+bool DBUpdater<LoginDatabaseConnection>::IsEnabled(uint32 const updateMask)
+{
+ // This way silences warnings under msvc
+ return (updateMask & DatabaseLoader::DATABASE_LOGIN) ? true : false;
+}
+
+// World Database
+template<>
+std::string DBUpdater<WorldDatabaseConnection>::GetConfigEntry()
+{
+ return "Updates.World";
+}
+
+template<>
+std::string DBUpdater<WorldDatabaseConnection>::GetTableName()
+{
+ return "World";
+}
+
+template<>
+std::string DBUpdater<WorldDatabaseConnection>::GetBaseFile()
+{
+ return _FULL_DATABASE;
+}
+
+template<>
+bool DBUpdater<WorldDatabaseConnection>::IsEnabled(uint32 const updateMask)
+{
+ // This way silences warnings under msvc
+ return (updateMask & DatabaseLoader::DATABASE_WORLD) ? true : false;
+}
+
+template<>
+BaseLocation DBUpdater<WorldDatabaseConnection>::GetBaseLocationType()
+{
+ return LOCATION_DOWNLOAD;
+}
+
+// Character Database
+template<>
+std::string DBUpdater<CharacterDatabaseConnection>::GetConfigEntry()
+{
+ return "Updates.Character";
+}
+
+template<>
+std::string DBUpdater<CharacterDatabaseConnection>::GetTableName()
+{
+ return "Character";
+}
+
+template<>
+std::string DBUpdater<CharacterDatabaseConnection>::GetBaseFile()
+{
+ return DBUpdater<CharacterDatabaseConnection>::GetSourceDirectory() + "/sql/base/characters_database.sql";
+}
+
+template<>
+bool DBUpdater<CharacterDatabaseConnection>::IsEnabled(uint32 const updateMask)
+{
+ // This way silences warnings under msvc
+ return (updateMask & DatabaseLoader::DATABASE_CHARACTER) ? true : false;
+}
+
+// All
+template<class T>
+BaseLocation DBUpdater<T>::GetBaseLocationType()
+{
+ return LOCATION_REPOSITORY;
+}
+
+template<class T>
+bool DBUpdater<T>::CheckExecutable()
+{
+ DBUpdater<T>::Path const exe(DBUpdater<T>::GetMySqlCli());
+ if (!exists(exe))
+ {
+ // Check for mysql in path
+ std::vector<std::string> args = {"--version"};
+ uint32 ret;
+ try
+ {
+ child c = execute(run_exe("mysql"), set_args(args), throw_on_error(), close_stdout());
+ ret = wait_for_exit(c);
+ }
+ catch (boost::system::system_error&)
+ {
+ ret = EXIT_FAILURE;
+ }
+
+ if (ret == EXIT_FAILURE)
+ {
+ TC_LOG_FATAL("sql.updates", "Didn't find executeable mysql binary at \'%s\', correct the path in the *.conf (\"Updates.MySqlCLIPath\").",
+ absolute(exe).generic_string().c_str());
+
+ return false;
+ }
+ }
+ return true;
+}
+
+template<class T>
+bool DBUpdater<T>::Create(DatabaseWorkerPool<T>& pool)
+{
+ TC_LOG_INFO("sql.updates", "Database \"%s\" does not exist, do you want to create it? [yes (default) / no]: ",
+ pool.GetConnectionInfo()->database.c_str());
+
+ std::string answer;
+ std::getline(std::cin, answer);
+ if (!answer.empty() && !(answer.substr(0, 1) == "y"))
+ return false;
+
+ TC_LOG_INFO("sql.updates", "Creating database \"%s\"...", pool.GetConnectionInfo()->database.c_str());
+
+ // Path of temp file
+ static Path const temp("create_table.sql");
+
+ // Create temporary query to use external mysql cli
+ std::ofstream file(temp.generic_string());
+ if (!file.is_open())
+ {
+ TC_LOG_FATAL("sql.updates", "Failed to create temporary query file \"%s\"!", temp.generic_string().c_str());
+ return false;
+ }
+
+ file << "CREATE DATABASE `" << pool.GetConnectionInfo()->database << "` DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci\n\n";
+
+ file.close();
+
+ try
+ {
+ DBUpdater<T>::ApplyFile(pool, pool.GetConnectionInfo()->host, pool.GetConnectionInfo()->user, pool.GetConnectionInfo()->password,
+ pool.GetConnectionInfo()->port_or_socket, "", temp);
+ }
+ catch (UpdateException&)
+ {
+ TC_LOG_FATAL("sql.updates", "Failed to create database %s! Has the user `CREATE` priviliges?", pool.GetConnectionInfo()->database.c_str());
+ boost::filesystem::remove(temp);
+ return false;
+ }
+
+ TC_LOG_INFO("sql.updates", "Done.");
+ boost::filesystem::remove(temp);
+ return true;
+}
+
+template<class T>
+bool DBUpdater<T>::Update(DatabaseWorkerPool<T>& pool)
+{
+ if (!DBUpdater<T>::CheckExecutable())
+ return false;
+
+ TC_LOG_INFO("sql.updates", "Updating %s database...", DBUpdater<T>::GetTableName().c_str());
+
+ Path const sourceDirectory(GetSourceDirectory());
+
+ if (!is_directory(sourceDirectory))
+ {
+ TC_LOG_ERROR("sql.updates", "DBUpdater: Given source directory %s does not exist, skipped!", sourceDirectory.generic_string().c_str());
+ return false;
+ }
+
+ UpdateFetcher updateFetcher(sourceDirectory, [&](std::string const& query) { DBUpdater<T>::Apply(pool, query); },
+ [&](Path const& file) { DBUpdater<T>::ApplyFile(pool, file); },
+ [&](std::string const& query) -> QueryResult { return DBUpdater<T>::Retrieve(pool, query); });
+
+ uint32 const count = updateFetcher.Update(
+ sConfigMgr->GetBoolDefault("Updates.Redundancy", true),
+ sConfigMgr->GetBoolDefault("Updates.AllowRehash", true),
+ sConfigMgr->GetBoolDefault("Updates.ArchivedRedundancy", false),
+ sConfigMgr->GetBoolDefault("Updates.CleanDeadRef", true));
+
+ if (!count)
+ TC_LOG_INFO("sql.updates", ">> %s database is up-to-date!", DBUpdater<T>::GetTableName().c_str());
+ else
+ TC_LOG_INFO("sql.updates", ">> Applied %d %s.", count, count == 1 ? "query" : "queries");
+
+ return true;
+}
+
+template<class T>
+bool DBUpdater<T>::Populate(DatabaseWorkerPool<T>& pool)
+{
+ {
+ QueryResult const result = Retrieve(pool, "SHOW TABLES");
+ if (result && (result->GetRowCount() > 0))
+ return true;
+ }
+
+ if (!DBUpdater<T>::CheckExecutable())
+ return false;
+
+ TC_LOG_INFO("sql.updates", "Database %s is empty, auto populating it...", DBUpdater<T>::GetTableName().c_str());
+
+ std::string const p = DBUpdater<T>::GetBaseFile();
+ if (p.empty())
+ {
+ TC_LOG_INFO("sql.updates", ">> No base file provided, skipped!");
+ return true;
+ }
+
+ Path const base(p);
+ if (!exists(base))
+ {
+ switch (DBUpdater<T>::GetBaseLocationType())
+ {
+ case LOCATION_REPOSITORY:
+ {
+ TC_LOG_ERROR("sql.updates", ">> Base file \"%s\" is missing, try to clone the source again.",
+ base.generic_string().c_str(), base.filename().generic_string().c_str());
+
+ break;
+ }
+ case LOCATION_DOWNLOAD:
+ {
+ TC_LOG_ERROR("sql.updates", ">> File \"%s\" is missing, download it from \"http://www.trinitycore.org/f/files/category/1-database/\"" \
+ " and place it in your server directory.", base.filename().generic_string().c_str());
+ break;
+ }
+ }
+ return false;
+ }
+
+ // Update database
+ TC_LOG_INFO("sql.updates", ">> Applying \'%s\'...", base.generic_string().c_str());
+ ApplyFile(pool, base);
+
+ TC_LOG_INFO("sql.updates", ">> Done!");
+ return true;
+}
+
+template<class T>
+QueryResult DBUpdater<T>::Retrieve(DatabaseWorkerPool<T>& pool, std::string const& query)
+{
+ return pool.PQuery(query.c_str());
+}
+
+template<class T>
+void DBUpdater<T>::Apply(DatabaseWorkerPool<T>& pool, std::string const& query)
+{
+ pool.DirectExecute(query.c_str());
+}
+
+template<class T>
+void DBUpdater<T>::ApplyFile(DatabaseWorkerPool<T>& pool, Path const& path)
+{
+ DBUpdater<T>::ApplyFile(pool, pool.GetConnectionInfo()->host, pool.GetConnectionInfo()->user, pool.GetConnectionInfo()->password,
+ pool.GetConnectionInfo()->port_or_socket, pool.GetConnectionInfo()->database, path);
+}
+
+template<class T>
+void DBUpdater<T>::ApplyFile(DatabaseWorkerPool<T>& pool, std::string const& host, std::string const& user,
+ std::string const& password, std::string const& port_or_socket, std::string const& database, Path const& path)
+{
+ std::vector<std::string> args;
+ args.reserve(7);
+
+ // CLI Client connection info
+ args.push_back("-h" + host);
+ args.push_back("-u" + user);
+ args.push_back("-p" + password);
+ args.push_back("-P" + port_or_socket);
+
+ // Set the default charset to utf8
+ args.push_back("--default-character-set=utf8");
+
+ // Set max allowed packet to 1 GB
+ args.push_back("--max-allowed-packet=1GB");
+
+ // Database
+ if (!database.empty())
+ args.push_back(database);
+
+ // ToDo: use the existing query in memory as virtual file if possible
+ file_descriptor_source source(path);
+
+ uint32 ret;
+ try
+ {
+ child c = execute(run_exe(DBUpdater<T>::GetMySqlCli().empty() ? "mysql" :
+ boost::filesystem::absolute(DBUpdater<T>::GetMySqlCli()).generic_string()),
+ set_args(args), bind_stdin(source), throw_on_error());
+
+ ret = wait_for_exit(c);
+ }
+ catch (boost::system::system_error&)
+ {
+ ret = EXIT_FAILURE;
+ }
+
+ source.close();
+
+ if (ret != EXIT_SUCCESS)
+ {
+ TC_LOG_FATAL("sql.updates", "Applying of file \'%s\' to database \'%s\' failed!" \
+ " If you are an user pull the latest revision from the repository. If you are a developer fix your sql query.",
+ path.generic_string().c_str(), pool.GetConnectionInfo()->database.c_str());
+
+ throw UpdateException("update failed");
+ }
+}
+
+template class DBUpdater<LoginDatabaseConnection>;
+template class DBUpdater<WorldDatabaseConnection>;
+template class DBUpdater<CharacterDatabaseConnection>;
diff --git a/src/server/shared/Updater/DBUpdater.h b/src/server/shared/Updater/DBUpdater.h
new file mode 100644
index 00000000000..0caf8a438fb
--- /dev/null
+++ b/src/server/shared/Updater/DBUpdater.h
@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2008-2015 TrinityCore <http://www.trinitycore.org/>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef DBUpdater_h__
+#define DBUpdater_h__
+
+#include "DatabaseEnv.h"
+
+#include <string>
+#include <boost/filesystem.hpp>
+
+class UpdateException : public std::exception
+{
+public:
+ UpdateException(std::string const& msg) : _msg(msg) { }
+ ~UpdateException() throw() { }
+
+ char const* what() const throw() override { return _msg.c_str(); }
+
+private:
+ std::string const _msg;
+};
+
+enum BaseLocation
+{
+ LOCATION_REPOSITORY,
+ LOCATION_DOWNLOAD
+};
+
+template <class T>
+class DBUpdater
+{
+public:
+ using Path = boost::filesystem::path;
+
+ static std::string GetSourceDirectory();
+
+ static inline std::string GetConfigEntry();
+
+ static inline std::string GetTableName();
+
+ static std::string GetBaseFile();
+
+ static bool IsEnabled(uint32 const updateMask);
+
+ static BaseLocation GetBaseLocationType();
+
+ static bool Create(DatabaseWorkerPool<T>& pool);
+
+ static bool Update(DatabaseWorkerPool<T>& pool);
+
+ static bool Populate(DatabaseWorkerPool<T>& pool);
+
+private:
+ static std::string GetMySqlCli();
+ static bool CheckExecutable();
+
+ static QueryResult Retrieve(DatabaseWorkerPool<T>& pool, std::string const& query);
+ static void Apply(DatabaseWorkerPool<T>& pool, std::string const& query);
+ static void ApplyFile(DatabaseWorkerPool<T>& pool, Path const& path);
+ static void ApplyFile(DatabaseWorkerPool<T>& pool, std::string const& host, std::string const& user,
+ std::string const& password, std::string const& port_or_socket, std::string const& database, Path const& path);
+};
+
+#endif // DBUpdater_h__
diff --git a/src/server/shared/Updater/UpdateFetcher.cpp b/src/server/shared/Updater/UpdateFetcher.cpp
new file mode 100644
index 00000000000..8084c6ba37f
--- /dev/null
+++ b/src/server/shared/Updater/UpdateFetcher.cpp
@@ -0,0 +1,387 @@
+/*
+ * Copyright (C) 2008-2015 TrinityCore <http://www.trinitycore.org/>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "UpdateFetcher.h"
+#include "Log.h"
+#include "Util.h"
+
+#include <fstream>
+#include <chrono>
+#include <vector>
+#include <sstream>
+#include <exception>
+#include <unordered_map>
+#include <openssl/sha.h>
+
+using namespace boost::filesystem;
+
+UpdateFetcher::UpdateFetcher(Path const& sourceDirectory,
+ std::function<void(std::string const&)> const& apply,
+ std::function<void(Path const& path)> const& applyFile,
+ std::function<QueryResult(std::string const&)> const& retrieve) :
+ _sourceDirectory(sourceDirectory), _apply(apply), _applyFile(applyFile),
+ _retrieve(retrieve)
+{
+}
+
+UpdateFetcher::LocaleFileStorage UpdateFetcher::GetFileList() const
+{
+ LocaleFileStorage files;
+ DirectoryStorage directories = ReceiveIncludedDirectories();
+ for (auto const& entry : directories)
+ FillFileListRecursively(entry.path, files, entry.state, 1);
+
+ return files;
+}
+
+void UpdateFetcher::FillFileListRecursively(Path const& path, LocaleFileStorage& storage, State const state, uint32 const depth) const
+{
+ static uint32 const MAX_DEPTH = 10;
+ static directory_iterator const end;
+
+ for (directory_iterator itr(path); itr != end; ++itr)
+ {
+ if (is_directory(itr->path()))
+ {
+ if (depth < MAX_DEPTH)
+ FillFileListRecursively(itr->path(), storage, state, depth + 1);
+ }
+ else if (itr->path().extension() == ".sql")
+ {
+ TC_LOG_TRACE("sql.updates", "Added locale file \"%s\".", itr->path().filename().generic_string().c_str());
+
+ LocaleFileEntry const entry = { itr->path(), state };
+
+ // Check for doubled filenames
+ // Since elements are only compared through their filenames this is ok
+ if (storage.find(entry) != storage.end())
+ {
+ TC_LOG_FATAL("sql.updates", "Duplicated filename occurred \"%s\", since updates are ordered " \
+ "through its filename every name needs to be unique!", itr->path().generic_string().c_str());
+
+ throw UpdateException("Updating failed, see the log for details.");
+ }
+
+ storage.insert(entry);
+ }
+ }
+}
+
+UpdateFetcher::DirectoryStorage UpdateFetcher::ReceiveIncludedDirectories() const
+{
+ DirectoryStorage directories;
+
+ QueryResult const result = _retrieve("SELECT `path`, `state` FROM `updates_include`");
+ if (!result)
+ return directories;
+
+ do
+ {
+ Field* fields = result->Fetch();
+
+ std::string path = fields[0].GetString();
+ if (path.substr(0, 1) == "$")
+ path = _sourceDirectory.generic_string() + path.substr(1);
+
+ Path const p(path);
+
+ if (!is_directory(p))
+ {
+ TC_LOG_ERROR("sql.updates", "DBUpdater: Given update include directory \"%s\" isn't existing, skipped!", p.generic_string().c_str());
+ continue;
+ }
+
+ DirectoryEntry const entry = { p, AppliedFileEntry::StateConvert(fields[1].GetString()) };
+ directories.push_back(entry);
+
+ TC_LOG_TRACE("sql.updates", "Added applied file \"%s\" from remote.", p.filename().generic_string().c_str());
+
+ } while (result->NextRow());
+
+ return directories;
+}
+
+UpdateFetcher::AppliedFileStorage UpdateFetcher::ReceiveAppliedFiles() const
+{
+ AppliedFileStorage map;
+
+ QueryResult result = _retrieve("SELECT `name`, `hash`, `state`, `timestamp` FROM `updates` ORDER BY `name` ASC");
+ if (!result)
+ return map;
+
+ do
+ {
+ Field* fields = result->Fetch();
+
+ AppliedFileEntry const entry = { fields[0].GetString(), fields[1].GetString(),
+ AppliedFileEntry::StateConvert(fields[2].GetString()), fields[3].GetUInt32() };
+
+ map.insert(std::make_pair(entry.name, entry));
+ }
+ while (result->NextRow());
+
+ return map;
+}
+
+UpdateFetcher::SQLUpdate UpdateFetcher::ReadSQLUpdate(boost::filesystem::path const& file) const
+{
+ std::ifstream in(file.c_str());
+ WPFatal(in.is_open(), "Could not read an update file.");
+
+ auto const start_pos = in.tellg();
+ in.ignore(std::numeric_limits<std::streamsize>::max());
+ auto const char_count = in.gcount();
+ in.seekg(start_pos);
+
+ SQLUpdate const update(new std::string(char_count, char{}));
+
+ in.read(&(*update)[0], update->size());
+ in.close();
+ return update;
+}
+
+uint32 UpdateFetcher::Update(bool const redundancyChecks, bool const allowRehash, bool const archivedRedundancy, bool const cleanDeadReferences) const
+{
+ LocaleFileStorage const available = GetFileList();
+ AppliedFileStorage applied = ReceiveAppliedFiles();
+
+ // Fill hash to name cache
+ HashToFileNameStorage hashToName;
+ for (auto entry : applied)
+ hashToName.insert(std::make_pair(entry.second.hash, entry.first));
+
+ uint32 importedUpdates = 0;
+
+ for (auto const& availableQuery : available)
+ {
+ TC_LOG_DEBUG("sql.updates", "Checking update \"%s\"...", availableQuery.first.filename().generic_string().c_str());
+
+ AppliedFileStorage::const_iterator iter = applied.find(availableQuery.first.filename().string());
+ if (iter != applied.end())
+ {
+ // If redundancy is disabled skip it since the update is already applied.
+ if (!redundancyChecks)
+ {
+ TC_LOG_DEBUG("sql.updates", ">> Update is already applied, skipping redundancy checks.");
+ applied.erase(iter);
+ continue;
+ }
+
+ // If the update is in an archived directory and is marked as archived in our database skip redundancy checks (archived updates never change).
+ if (!archivedRedundancy && (iter->second.state == ARCHIVED) && (availableQuery.second == ARCHIVED))
+ {
+ TC_LOG_DEBUG("sql.updates", ">> Update is archived and marked as archived in database, skipping redundancy checks.");
+ applied.erase(iter);
+ continue;
+ }
+ }
+
+ // Read update from file
+ SQLUpdate const update = ReadSQLUpdate(availableQuery.first);
+
+ // Calculate hash
+ std::string const hash = CalculateHash(update);
+
+ UpdateMode mode = MODE_APPLY;
+
+ // Update is not in our applied list
+ if (iter == applied.end())
+ {
+ // Catch renames (different filename but same hash)
+ HashToFileNameStorage::const_iterator const hashIter = hashToName.find(hash);
+ if (hashIter != hashToName.end())
+ {
+ // Check if the original file was removed if not we've got a problem.
+ LocaleFileStorage::const_iterator localeIter;
+ // Push localeIter forward
+ for (localeIter = available.begin(); (localeIter != available.end()) &&
+ (localeIter->first.filename().string() != hashIter->second); ++localeIter);
+
+ // Conflict!
+ if (localeIter != available.end())
+ {
+ TC_LOG_WARN("sql.updates", ">> Seems like update \"%s\" \'%s\' was renamed, but the old file is still there! " \
+ "Trade it as a new file! (Probably its an unmodified copy of file \"%s\")",
+ availableQuery.first.filename().string().c_str(), hash.substr(0, 7).c_str(),
+ localeIter->first.filename().string().c_str());
+ }
+ // Its save to trade the file as renamed here
+ else
+ {
+ TC_LOG_INFO("sql.updates", ">> Renaming update \"%s\" to \"%s\" \'%s\'.",
+ hashIter->second.c_str(), availableQuery.first.filename().string().c_str(), hash.substr(0, 7).c_str());
+
+ RenameEntry(hashIter->second, availableQuery.first.filename().string());
+ applied.erase(hashIter->second);
+ continue;
+ }
+ }
+ // Apply the update if it was never seen before.
+ else
+ {
+ TC_LOG_INFO("sql.updates", ">> Applying update \"%s\" \'%s\'...",
+ availableQuery.first.filename().string().c_str(), hash.substr(0, 7).c_str());
+ }
+ }
+ // Rehash the update entry if it is contained in our database but with an empty hash.
+ else if (allowRehash && iter->second.hash.empty())
+ {
+ mode = MODE_REHASH;
+
+ TC_LOG_INFO("sql.updates", ">> Re-hashing update \"%s\" \'%s\'...", availableQuery.first.filename().string().c_str(),
+ hash.substr(0, 7).c_str());
+ }
+ else
+ {
+ // If the hash of the files differs from the one stored in our database reapply the update (because it was changed).
+ if (iter->second.hash != hash)
+ {
+ TC_LOG_INFO("sql.updates", ">> Reapplying update \"%s\" \'%s\' -> \'%s\' (it changed)...", availableQuery.first.filename().string().c_str(),
+ iter->second.hash.substr(0, 7).c_str(), hash.substr(0, 7).c_str());
+ }
+ else
+ {
+ // If the file wasn't changed and just moved update its state if necessary.
+ if (iter->second.state != availableQuery.second)
+ {
+ TC_LOG_DEBUG("sql.updates", ">> Updating state of \"%s\" to \'%s\'...",
+ availableQuery.first.filename().string().c_str(), AppliedFileEntry::StateConvert(availableQuery.second).c_str());
+
+ UpdateState(availableQuery.first.filename().string(), availableQuery.second);
+ }
+
+ TC_LOG_DEBUG("sql.updates", ">> Update is already applied and is matching hash \'%s\'.", hash.substr(0, 7).c_str());
+
+ applied.erase(iter);
+ continue;
+ }
+ }
+
+ uint32 speed = 0;
+ AppliedFileEntry const file = { availableQuery.first.filename().string(), hash, availableQuery.second, 0 };
+
+ switch (mode)
+ {
+ case MODE_APPLY:
+ speed = Apply(availableQuery.first);
+ /*no break*/
+ case MODE_REHASH:
+ UpdateEntry(file, speed);
+ break;
+ }
+
+ if (iter != applied.end())
+ applied.erase(iter);
+
+ if (mode == MODE_APPLY)
+ ++importedUpdates;
+ }
+
+ for (auto const& entry : applied)
+ {
+ TC_LOG_WARN("sql.updates", ">> File \'%s\' was applied to the database but is missing in" \
+ " your update directory now!%s", entry.first.c_str(), cleanDeadReferences ? " Deleting orphaned entry..." : "");
+ }
+
+ if (cleanDeadReferences)
+ CleanUp(applied);
+
+ return importedUpdates;
+}
+
+std::string UpdateFetcher::CalculateHash(SQLUpdate const& query) const
+{
+ // Calculate a Sha1 hash based on query content.
+ unsigned char digest[SHA_DIGEST_LENGTH];
+ SHA1((unsigned char*)query->c_str(), query->length(), (unsigned char*)&digest);
+
+ return ByteArrayToHexStr(digest, SHA_DIGEST_LENGTH);
+}
+
+uint32 UpdateFetcher::Apply(Path const& path) const
+{
+ using Time = std::chrono::high_resolution_clock;
+ using ms = std::chrono::milliseconds;
+
+ // Benchmark query speed
+ auto const begin = Time::now();
+
+ // Update database
+ _applyFile(path);
+
+ // Return time the query took to apply
+ return std::chrono::duration_cast<ms>(Time::now() - begin).count();
+}
+
+void UpdateFetcher::UpdateEntry(AppliedFileEntry const& entry, uint32 const speed) const
+{
+ std::string const update = "REPLACE INTO `updates` (`name`, `hash`, `state`, `speed`) VALUES (\"" +
+ entry.name + "\", \"" + entry.hash + "\", \'" + entry.GetStateAsString() + "\', " + std::to_string(speed) + ")";
+
+ // Update database
+ _apply(update);
+}
+
+void UpdateFetcher::RenameEntry(std::string const& from, std::string const& to) const
+{
+ // Delete target if it exists
+ {
+ std::string const update = "DELETE FROM `updates` WHERE `name`=\"" + to + "\"";
+
+ // Update database
+ _apply(update);
+ }
+
+ // Rename
+ {
+ std::string const update = "UPDATE `updates` SET `name`=\"" + to + "\" WHERE `name`=\"" + from + "\"";
+
+ // Update database
+ _apply(update);
+ }
+}
+
+void UpdateFetcher::CleanUp(AppliedFileStorage const& storage) const
+{
+ if (storage.empty())
+ return;
+
+ std::stringstream update;
+ size_t remaining = storage.size();
+
+ update << "DELETE FROM `updates` WHERE `name` IN(";
+
+ for (auto const& entry : storage)
+ {
+ update << "\"" << entry.first << "\"";
+ if ((--remaining) > 0)
+ update << ", ";
+ }
+
+ update << ")";
+
+ // Update database
+ _apply(update.str());
+}
+
+void UpdateFetcher::UpdateState(std::string const& name, State const state) const
+{
+ std::string const update = "UPDATE `updates` SET `state`=\'" + AppliedFileEntry::StateConvert(state) + "\' WHERE `name`=\"" + name + "\"";
+
+ // Update database
+ _apply(update);
+}
diff --git a/src/server/shared/Updater/UpdateFetcher.h b/src/server/shared/Updater/UpdateFetcher.h
new file mode 100644
index 00000000000..b348eecf4e1
--- /dev/null
+++ b/src/server/shared/Updater/UpdateFetcher.h
@@ -0,0 +1,128 @@
+/*
+ * Copyright (C) 2008-2015 TrinityCore <http://www.trinitycore.org/>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef UpdateFetcher_h__
+#define UpdateFetcher_h__
+
+#include <DBUpdater.h>
+
+#include <functional>
+#include <string>
+#include <memory>
+#include <vector>
+#include <boost/container/flat_set.hpp>
+
+class UpdateFetcher
+{
+ using Path = boost::filesystem::path;
+
+public:
+ UpdateFetcher(Path const& updateDirectory,
+ std::function<void(std::string const&)> const& apply,
+ std::function<void(Path const& path)> const& applyFile,
+ std::function<QueryResult(std::string const&)> const& retrieve);
+
+ uint32 Update(bool const redundancyChecks, bool const allowRehash,
+ bool const archivedRedundancy, bool const cleanDeadReferences) const;
+
+private:
+ enum UpdateMode
+ {
+ MODE_APPLY,
+ MODE_REHASH
+ };
+
+ enum State
+ {
+ RELEASED,
+ ARCHIVED
+ };
+
+ struct AppliedFileEntry
+ {
+ std::string const name;
+
+ std::string const hash;
+
+ State const state;
+
+ uint32 const timestamp;
+
+ static inline State StateConvert(std::string const& state)
+ {
+ return (state == "RELEASED") ? RELEASED : ARCHIVED;
+ }
+
+ static inline std::string StateConvert(State const state)
+ {
+ return (state == RELEASED) ? "RELEASED" : "ARCHIVED";
+ }
+
+ std::string GetStateAsString() const
+ {
+ return StateConvert(state);
+ }
+ };
+
+ struct DirectoryEntry
+ {
+ Path const path;
+
+ State const state;
+ };
+
+ using LocaleFileEntry = std::pair<Path, State>;
+
+ struct PathCompare
+ {
+ inline bool operator() (LocaleFileEntry const& left, LocaleFileEntry const& right) const
+ {
+ return left.first.filename().string() < right.first.filename().string();
+ }
+ };
+
+ using LocaleFileStorage = boost::container::flat_set<LocaleFileEntry, PathCompare>;
+ using HashToFileNameStorage = std::unordered_map<std::string, std::string>;
+ using AppliedFileStorage = std::unordered_map<std::string, AppliedFileEntry>;
+ using DirectoryStorage = std::vector<UpdateFetcher::DirectoryEntry>;
+ using SQLUpdate = std::shared_ptr<std::string>;
+
+ LocaleFileStorage GetFileList() const;
+ void FillFileListRecursively(Path const& path, LocaleFileStorage& storage, State const state, uint32 const depth) const;
+
+ DirectoryStorage ReceiveIncludedDirectories() const;
+ AppliedFileStorage ReceiveAppliedFiles() const;
+
+ SQLUpdate ReadSQLUpdate(Path const& file) const;
+ std::string CalculateHash(SQLUpdate const& query) const;
+
+ uint32 Apply(Path const& path) const;
+
+ void UpdateEntry(AppliedFileEntry const& entry, uint32 const speed = 0) const;
+ void RenameEntry(std::string const& from, std::string const& to) const;
+ void CleanUp(AppliedFileStorage const& storage) const;
+
+ void UpdateState(std::string const& name, State const state) const;
+
+ Path const _sourceDirectory;
+
+ std::function<void(std::string const&)> const _apply;
+ std::function<void(Path const& path)> const _applyFile;
+ std::function<QueryResult(std::string const&)> const _retrieve;
+};
+
+#endif // UpdateFetcher_h__
diff --git a/src/server/worldserver/CMakeLists.txt b/src/server/worldserver/CMakeLists.txt
index ce9ae8d5219..ded076d8925 100644
--- a/src/server/worldserver/CMakeLists.txt
+++ b/src/server/worldserver/CMakeLists.txt
@@ -48,6 +48,7 @@ include_directories(
${CMAKE_SOURCE_DIR}/dep/sockets/include
${CMAKE_SOURCE_DIR}/dep/SFMT
${CMAKE_SOURCE_DIR}/dep/cppformat
+ ${CMAKE_SOURCE_DIR}/dep/process
${CMAKE_SOURCE_DIR}/src/server/collision
${CMAKE_SOURCE_DIR}/src/server/collision/Management
${CMAKE_SOURCE_DIR}/src/server/collision/Models
@@ -64,6 +65,7 @@ include_directories(
${CMAKE_SOURCE_DIR}/src/server/shared/Networking
${CMAKE_SOURCE_DIR}/src/server/shared/Packets
${CMAKE_SOURCE_DIR}/src/server/shared/Threading
+ ${CMAKE_SOURCE_DIR}/src/server/shared/Updater
${CMAKE_SOURCE_DIR}/src/server/shared/Utilities
${CMAKE_SOURCE_DIR}/src/server/game
${CMAKE_SOURCE_DIR}/src/server/game/Accounts
diff --git a/src/server/worldserver/Main.cpp b/src/server/worldserver/Main.cpp
index 7b47dba8759..fbf2c74ac0b 100644
--- a/src/server/worldserver/Main.cpp
+++ b/src/server/worldserver/Main.cpp
@@ -47,6 +47,7 @@
#include "SystemConfig.h"
#include "WorldSocket.h"
#include "WorldSocketMgr.h"
+#include "DatabaseLoader.h"
using namespace boost::program_options;
@@ -447,80 +448,15 @@ bool StartDB()
{
MySQL::Library_Init();
- std::string dbString;
- uint8 asyncThreads, synchThreads;
+ // Load databases
+ DatabaseLoader loader("server.worldserver", DatabaseLoader::DATABASE_NONE);
+ loader
+ .AddDatabase(WorldDatabase, "World")
+ .AddDatabase(CharacterDatabase, "Character")
+ .AddDatabase(LoginDatabase, "Login");
- dbString = sConfigMgr->GetStringDefault("WorldDatabaseInfo", "");
- if (dbString.empty())
- {
- TC_LOG_ERROR("server.worldserver", "World database not specified in configuration file");
- return false;
- }
-
- asyncThreads = uint8(sConfigMgr->GetIntDefault("WorldDatabase.WorkerThreads", 1));
- if (asyncThreads < 1 || asyncThreads > 32)
- {
- TC_LOG_ERROR("server.worldserver", "World database: invalid number of worker threads specified. "
- "Please pick a value between 1 and 32.");
- return false;
- }
-
- synchThreads = uint8(sConfigMgr->GetIntDefault("WorldDatabase.SynchThreads", 1));
- ///- Initialize the world database
- if (!WorldDatabase.Open(dbString, asyncThreads, synchThreads))
- {
- TC_LOG_ERROR("server.worldserver", "Cannot connect to world database %s", dbString.c_str());
+ if (!loader.Load())
return false;
- }
-
- ///- Get character database info from configuration file
- dbString = sConfigMgr->GetStringDefault("CharacterDatabaseInfo", "");
- if (dbString.empty())
- {
- TC_LOG_ERROR("server.worldserver", "Character database not specified in configuration file");
- return false;
- }
-
- asyncThreads = uint8(sConfigMgr->GetIntDefault("CharacterDatabase.WorkerThreads", 1));
- if (asyncThreads < 1 || asyncThreads > 32)
- {
- TC_LOG_ERROR("server.worldserver", "Character database: invalid number of worker threads specified. "
- "Please pick a value between 1 and 32.");
- return false;
- }
-
- synchThreads = uint8(sConfigMgr->GetIntDefault("CharacterDatabase.SynchThreads", 2));
-
- ///- Initialize the Character database
- if (!CharacterDatabase.Open(dbString, asyncThreads, synchThreads))
- {
- TC_LOG_ERROR("server.worldserver", "Cannot connect to Character database %s", dbString.c_str());
- return false;
- }
-
- ///- Get login database info from configuration file
- dbString = sConfigMgr->GetStringDefault("LoginDatabaseInfo", "");
- if (dbString.empty())
- {
- TC_LOG_ERROR("server.worldserver", "Login database not specified in configuration file");
- return false;
- }
-
- asyncThreads = uint8(sConfigMgr->GetIntDefault("LoginDatabase.WorkerThreads", 1));
- if (asyncThreads < 1 || asyncThreads > 32)
- {
- TC_LOG_ERROR("server.worldserver", "Login database: invalid number of worker threads specified. "
- "Please pick a value between 1 and 32.");
- return false;
- }
-
- synchThreads = uint8(sConfigMgr->GetIntDefault("LoginDatabase.SynchThreads", 1));
- ///- Initialise the login database
- if (!LoginDatabase.Open(dbString, asyncThreads, synchThreads))
- {
- TC_LOG_ERROR("server.worldserver", "Cannot connect to login database %s", dbString.c_str());
- return false;
- }
///- Get the realm Id from the configuration file
realmID = sConfigMgr->GetIntDefault("RealmID", 0);
diff --git a/src/server/worldserver/worldserver.conf.dist b/src/server/worldserver/worldserver.conf.dist
index ecb5824b1c5..c7464721968 100644
--- a/src/server/worldserver/worldserver.conf.dist
+++ b/src/server/worldserver/worldserver.conf.dist
@@ -11,6 +11,7 @@
# PERFORMANCE SETTINGS
# SERVER LOGGING
# SERVER SETTINGS
+# UPDATE SETTINGS
# WARDEN SETTINGS
# PLAYER INTERACTION
# CREATURE SETTINGS
@@ -1121,6 +1122,89 @@ BirthdayTime = 1222964635
###################################################################################################
###################################################################################################
+# UPDATE SETTINGS
+#
+# Updates.EnableDatabases
+# Description: A mask that describes which databases shall be updated.
+#
+# Following flags are available
+# DATABASE_LOGIN = 1, // Auth database
+# DATABASE_CHARACTER = 2, // Character database
+# DATABASE_WORLD = 4, // World database
+#
+# Default: 0 - (All Disabled)
+# 4 - (Enable world only)
+# 7 - (All enabled)
+
+Updates.EnableDatabases = 0
+
+#
+# Updates.SourcePath
+# Description: The path to your TrinityCore source directory.
+# If the path is left empty, built-in CMAKE_SOURCE_DIR is used.
+# Example: "../TrinityCore"
+# Default: ""
+
+Updates.SourcePath = ""
+
+#
+# Updates.SourcePath
+# Description: The path to your mysql cli binary.
+# If the path is left empty, built-in path from cmake is used.
+# Example: "C:/Program Files/MySQL/MySQL Server 5.6/bin/mysql.exe"
+# "mysql.exe"
+# "/usr/bin/mysql"
+# Default: ""
+
+Updates.MySqlCLIPath = ""
+
+#
+# Updates.AutoSetup
+# Description: Auto populate empty databases.
+# Default: 1 - (Enabled)
+# 0 - (Disabled)
+
+Updates.AutoSetup = 1
+
+#
+# Updates.Redundancy
+# Description: Perform data redundancy checks through hashing
+# to detect changes on sql updates and reapply it.
+# Default: 1 - (Enabled)
+# 0 - (Disabled)
+
+Updates.Redundancy = 1
+
+#
+# Updates.ArchivedRedundancy
+# Description: Check hashes of archived updates (slows down startup).
+# Default: 0 - (Disabled)
+# 1 - (Enabled)
+
+Updates.ArchivedRedundancy = 0
+
+#
+# Updates.AllowRehash
+# Description: Inserts the current file hash in the database if it is left empty.
+# Useful if you want to mark a file as applied but you don't know its hash.
+# Default: 1 - (Enabled)
+# 0 - (Disabled)
+
+Updates.AllowRehash = 1
+
+#
+# Updates.CleanDeadRef
+# Description: Cleans dead/ orphaned references that occure if a update was deleted or renamed and edited.
+# Disable this if you want to know if the database is in a possible "dirty state".
+# Default: 1 - (Enabled)
+# 0 - (Disabled)
+
+Updates.CleanDeadRef = 1
+
+#
+###################################################################################################
+
+###################################################################################################
# WARDEN SETTINGS
#
# Warden.Enabled
@@ -3173,6 +3257,7 @@ Logger.root=5,Console Server
Logger.server=3,Console Server
Logger.commands.gm=3,Console GM
Logger.sql.sql=5,Console DBErrors
+Logger.sql.updates=3,Console Server
#Logger.achievement=3,Console Server
#Logger.ahbot=3,Console Server