Core/Updates: Add an automatic database update system. Automatically detects new and edited sql updates through file lists and hashing. Detects renames, deletes and is able to create and auto import full databases. * cleanups in main.cpp of world & bnetserver * refactoring in DatabaseWorkerPool.h & MySQLConnection.cpp

Make sure you re-run cmake, because boost::iostreams was added as dependency.
Maybe you need to install libboost-iostreams1.55-dev on unix as well.

Import every update manual until (included) those INSERT IGNORE updates for each database.

Thanks DDuarte and Shauren for your amazing ideas, help and advises.

In hope that nobody gets a "Your database structure is not up to date..." anymore ,-)

Signed-off-by: Naios <naios-dev@live.de>
Signed-off-by: Nayd <dnpd.dd@gmail.com>

5 files changed, 334 insertions, 35 deletions

diff --git a/src/server/shared/Database/DatabaseLoader.cpp b/src/server/shared/Database/DatabaseLoader.cpp
new file mode 100644
index 00000000000..25c400fdfa8
--- /dev/null
+++ b/src/server/shared/Database/DatabaseLoader.cpp
@@ -0,0 +1,193 @@
+/*
+ * Copyright (C) 2008-2015 TrinityCore <http://www.trinitycore.org/>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "DatabaseLoader.h"
+#include "DBUpdater.h"
+#include "Config.h"
+
+#include <mysqld_error.h>
+
+DatabaseLoader::DatabaseLoader(std::string const& logger, uint32 const defaultUpdateMask)
+    : _logger(logger), _autoSetup(sConfigMgr->GetBoolDefault("Updates.AutoSetup", true)),
+    _updateFlags(sConfigMgr->GetIntDefault("Updates.EnableDatabases", defaultUpdateMask))
+{
+}
+
+template <class T>
+DatabaseLoader& DatabaseLoader::AddDatabase(DatabaseWorkerPool<T>& pool, std::string const& name)
+{
+    bool const updatesEnabledForThis = DBUpdater<T>::IsEnabled(_updateFlags);
+
+    _open.push(std::make_pair([this, name, updatesEnabledForThis, &pool]() -> bool
+    {
+        std::string const dbString = sConfigMgr->GetStringDefault(name + "DatabaseInfo", "");
+        if (dbString.empty())
+        {
+            TC_LOG_ERROR(_logger.c_str(), "Database %s not specified in configuration file!", name.c_str());
+            return false;
+        }
+
+        uint8 const asyncThreads = uint8(sConfigMgr->GetIntDefault(name + "Database.WorkerThreads", 1));
+        if (asyncThreads < 1 || asyncThreads > 32)
+        {
+            TC_LOG_ERROR(_logger.c_str(), "%s database: invalid number of worker threads specified. "
+                "Please pick a value between 1 and 32.", name.c_str());
+            return false;
+        }
+
+        uint8 const synchThreads = uint8(sConfigMgr->GetIntDefault(name + "Database.SynchThreads", 1));
+
+        pool.SetConnectionInfo(dbString, asyncThreads, synchThreads);
+        if (uint32 error = pool.Open())
+        {
+            // Database does not exist
+            if ((error == ER_BAD_DB_ERROR) && updatesEnabledForThis && _autoSetup)
+            {
+                // Try to create the database and connect again if auto setup is enabled
+                if (DBUpdater<T>::Create(pool) && (!pool.Open()))
+                    error = 0;
+            }
+
+            // If the error wasn't handled quit
+            if (error)
+            {
+                TC_LOG_ERROR("sql.driver", "\nDatabasePool %s NOT opened. There were errors opening the MySQL connections. Check your SQLDriverLogFile "
+                    "for specific errors. Read wiki at http://collab.kpsn.org/display/tc/TrinityCore+Home", name.c_str());
+
+                return false;
+            }
+        }
+        return true;
+    },
+    [&pool]()
+    {
+        pool.Close();
+    }));
+
+    // Populate and update only if updates are enabled for this pool
+    if (updatesEnabledForThis)
+    {
+        _populate.push([this, name, &pool]() -> bool
+        {
+            if (!DBUpdater<T>::Populate(pool))
+            {
+                TC_LOG_ERROR(_logger.c_str(), "Could not populate the %s database, see log for details.", name.c_str());
+                return false;
+            }
+            return true;
+        });
+
+        _update.push([this, name, &pool]() -> bool
+        {
+            if (!DBUpdater<T>::Update(pool))
+            {
+                TC_LOG_ERROR(_logger.c_str(), "Could not update the %s database, see log for details.", name.c_str());
+                return false;
+            }
+            return true;
+        });
+    }
+
+    _prepare.push([this, name, &pool]() -> bool
+    {
+        if (!pool.PrepareStatements())
+        {
+            TC_LOG_ERROR(_logger.c_str(), "Could not prepare statements of the %s database, see log for details.", name.c_str());
+            return false;
+        }
+        return true;
+    });
+
+    return *this;
+}
+
+bool DatabaseLoader::Load()
+{
+    if (!OpenDatabases())
+        return false;
+
+    if (!PopulateDatabases())
+        return false;
+
+    if (!UpdateDatabases())
+        return false;
+
+    if (!PrepareStatements())
+        return false;
+
+    return true;
+}
+
+bool DatabaseLoader::OpenDatabases()
+{
+    while (!_open.empty())
+    {
+        std::pair<Predicate, std::function<void()>> const load = _open.top();
+        if (load.first())
+            _close.push(load.second);
+        else
+        {
+            // Close all loaded databases
+            while (!_close.empty())
+            {
+                _close.top()();
+                _close.pop();
+            }
+            return false;
+        }
+
+        _open.pop();
+    }
+    return true;
+}
+
+// Processes the elements of the given stack until a predicate returned false.
+bool DatabaseLoader::Process(std::stack<Predicate>& stack)
+{
+    while (!stack.empty())
+    {
+        if (!stack.top()())
+            return false;
+
+        stack.pop();
+    }
+    return true;
+}
+
+bool DatabaseLoader::PopulateDatabases()
+{
+    return Process(_populate);
+}
+
+bool DatabaseLoader::UpdateDatabases()
+{
+    return Process(_update);
+}
+
+bool DatabaseLoader::PrepareStatements()
+{
+    return Process(_prepare);
+}
+
+template
+DatabaseLoader& DatabaseLoader::AddDatabase<LoginDatabaseConnection>(DatabaseWorkerPool<LoginDatabaseConnection>& pool, std::string const& name);
+template
+DatabaseLoader& DatabaseLoader::AddDatabase<WorldDatabaseConnection>(DatabaseWorkerPool<WorldDatabaseConnection>& pool, std::string const& name);
+template
+DatabaseLoader& DatabaseLoader::AddDatabase<CharacterDatabaseConnection>(DatabaseWorkerPool<CharacterDatabaseConnection>& pool, std::string const& name);
+template
+DatabaseLoader& DatabaseLoader::AddDatabase<HotfixDatabaseConnection>(DatabaseWorkerPool<HotfixDatabaseConnection>& pool, std::string const& name);
diff --git a/src/server/shared/Database/DatabaseLoader.h b/src/server/shared/Database/DatabaseLoader.h
new file mode 100644
index 00000000000..0d31ef1686c
--- /dev/null
+++ b/src/server/shared/Database/DatabaseLoader.h
@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2008-2015 TrinityCore <http://www.trinitycore.org/>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef DatabaseLoader_h__
+#define DatabaseLoader_h__
+
+#include "DatabaseWorkerPool.h"
+#include "DatabaseEnv.h"
+
+#include <stack>
+#include <functional>
+
+// A helper class to initiate all database worker pools,
+// handles updating, delays preparing of statements and cleans up on failure.
+class DatabaseLoader
+{
+public:
+    DatabaseLoader(std::string const& logger, uint32 const defaultUpdateMask);
+
+    // Register a database to the loader (lazy implemented)
+    template <class T>
+    DatabaseLoader& AddDatabase(DatabaseWorkerPool<T>& pool, std::string const& name);
+
+    // Load all databases
+    bool Load();
+
+    enum DatabaseTypeFlags
+    {
+        DATABASE_NONE       = 0,
+
+        DATABASE_LOGIN      = 1,
+        DATABASE_CHARACTER  = 2,
+        DATABASE_WORLD      = 4,
+        DATABASE_HOTFIX     = 8,
+
+        DATABASE_MASK_ALL   = DATABASE_LOGIN | DATABASE_CHARACTER | DATABASE_WORLD | DATABASE_HOTFIX
+    };
+
+private:
+    bool OpenDatabases();
+    bool PopulateDatabases();
+    bool UpdateDatabases();
+    bool PrepareStatements();
+
+    using Predicate = std::function<bool()>;
+
+    static bool Process(std::stack<Predicate>& stack);
+
+    bool const _autoSetup;
+    uint32 const _updateFlags;
+    std::string const _logger;
+
+    std::stack<std::pair<Predicate, std::function<void()>>> _open;
+    std::stack<std::function<void()>> _close;
+    std::stack<Predicate> _populate, _update, _prepare;
+};
+
+#endif // DatabaseLoader_h__
diff --git a/src/server/shared/Database/DatabaseWorkerPool.h b/src/server/shared/Database/DatabaseWorkerPool.h
index f1c6a7acbf5..fa142c9020c 100644
--- a/src/server/shared/Database/DatabaseWorkerPool.h
+++ b/src/server/shared/Database/DatabaseWorkerPool.h
@@ -30,6 +30,7 @@
 #include "AdhocStatement.h"
 
 #include <mysqld_error.h>
+#include <memory>
 
 #define MIN_MYSQL_SERVER_VERSION 50100u
 #define MIN_MYSQL_CLIENT_VERSION 50100u
@@ -57,9 +58,9 @@ class DatabaseWorkerPool
 
     public:
         /* Activity state */
-        DatabaseWorkerPool() : _connectionInfo(NULL)
+        DatabaseWorkerPool() : _connectionInfo(nullptr), _queue(new ProducerConsumerQueue<SQLOperation*>()),
+            _async_threads(0), _synch_threads(0)
         {
-            _queue = new ProducerConsumerQueue<SQLOperation*>();
             memset(_connectionCount, 0, sizeof(_connectionCount));
             _connections.resize(IDX_SIZE);
 
@@ -70,31 +71,37 @@ class DatabaseWorkerPool
         ~DatabaseWorkerPool()
         {
             _queue->Cancel();
+        }
 
-            delete _queue;
+        void SetConnectionInfo(std::string const& infoString, uint8 const asyncThreads, uint8 const synchThreads)
+        {
+            _connectionInfo.reset(new MySQLConnectionInfo(infoString));
 
-            delete _connectionInfo;
+            _async_threads = asyncThreads;
+            _synch_threads = synchThreads;
         }
 
-        bool Open(const std::string& infoString, uint8 async_threads, uint8 synch_threads)
+        uint32 Open()
         {
-            _connectionInfo = new MySQLConnectionInfo(infoString);
+            WPFatal(_connectionInfo.get(), "Connection info was not set!");
 
             TC_LOG_INFO("sql.driver", "Opening DatabasePool '%s'. Asynchronous connections: %u, synchronous connections: %u.",
-                GetDatabaseName(), async_threads, synch_threads);
+                GetDatabaseName(), _async_threads, _synch_threads);
 
-            bool res = OpenConnections(IDX_ASYNC, async_threads);
+            uint32 error = OpenConnections(IDX_ASYNC, _async_threads);
 
-            if (!res)
-                return res;
+            if (error)
+                return error;
 
-            res = OpenConnections(IDX_SYNCH, synch_threads);
+            error = OpenConnections(IDX_SYNCH, _synch_threads);
 
-            if (res)
+            if (!error)
+            {
                 TC_LOG_INFO("sql.driver", "DatabasePool '%s' opened successfully. %u total connections running.", GetDatabaseName(),
                     (_connectionCount[IDX_SYNCH] + _connectionCount[IDX_ASYNC]));
+            }
 
-            return res;
+            return error;
         }
 
         void Close()
@@ -120,6 +127,32 @@ class DatabaseWorkerPool
             TC_LOG_INFO("sql.driver", "All connections on DatabasePool '%s' closed.", GetDatabaseName());
         }
 
+        //! Prepares all prepared statements
+        bool PrepareStatements()
+        {
+            for (uint8 i = 0; i < IDX_SIZE; ++i)
+                for (uint32 c = 0; c < _connectionCount[i]; ++c)
+                {
+                    T* t = _connections[i][c];
+                    t->LockIfReady();
+                    if (!t->PrepareStatements())
+                    {
+                        t->Unlock();
+                        Close();
+                        return false;
+                    }
+                    else
+                        t->Unlock();
+                }
+
+            return true;
+        }
+
+        inline MySQLConnectionInfo const* GetConnectionInfo() const
+        {
+            return _connectionInfo.get();
+        }
+
         /**
             Delayed one-way statement methods.
         */
@@ -461,7 +494,7 @@ class DatabaseWorkerPool
         }
 
     private:
-        bool OpenConnections(InternalIndex type, uint8 numConnections)
+        uint32 OpenConnections(InternalIndex type, uint8 numConnections)
         {
             _connections[type].resize(numConnections);
             for (uint8 i = 0; i < numConnections; ++i)
@@ -469,7 +502,7 @@ class DatabaseWorkerPool
                 T* t;
 
                 if (type == IDX_ASYNC)
-                    t = new T(_queue, *_connectionInfo);
+                    t = new T(_queue.get(), *_connectionInfo);
                 else if (type == IDX_SYNCH)
                     t = new T(*_connectionInfo);
                 else
@@ -478,35 +511,32 @@ class DatabaseWorkerPool
                 _connections[type][i] = t;
                 ++_connectionCount[type];
 
-                bool res = t->Open();
+                uint32 error = t->Open();
 
-                if (res)
+                if (!error)
                 {
                     if (mysql_get_server_version(t->GetHandle()) < MIN_MYSQL_SERVER_VERSION)
                     {
                         TC_LOG_ERROR("sql.driver", "TrinityCore does not support MySQL versions below 5.1");
-                        res = false;
+                        error = 1;
                     }
                 }
 
                 // Failed to open a connection or invalid version, abort and cleanup
-                if (!res)
+                if (error)
                 {
-                    TC_LOG_ERROR("sql.driver", "DatabasePool %s NOT opened. There were errors opening the MySQL connections. Check your SQLDriverLogFile "
-                        "for specific errors. Read wiki at http://collab.kpsn.org/display/tc/TrinityCore+Home", GetDatabaseName());
-
                     while (_connectionCount[type] != 0)
                     {
                         T* t = _connections[type][i--];
                         delete t;
                         --_connectionCount[type];
                     }
-
-                    return false;
+                    return error;
                 }
             }
 
-            return true;
+            // Everything is fine
+            return 0;
         }
 
         unsigned long EscapeString(char *to, const char *from, unsigned long length)
@@ -546,10 +576,13 @@ class DatabaseWorkerPool
             return _connectionInfo->database.c_str();
         }
 
-        ProducerConsumerQueue<SQLOperation*>* _queue;             //! Queue shared by async worker threads.
-        std::vector< std::vector<T*> >        _connections;
-        uint32                                _connectionCount[2];       //! Counter of MySQL connections;
-        MySQLConnectionInfo*                  _connectionInfo;
+        //! Queue shared by async worker threads.
+        std::unique_ptr<ProducerConsumerQueue<SQLOperation*>> _queue;
+        std::vector<std::vector<T*>> _connections;
+        //! Counter of MySQL connections;
+        uint32 _connectionCount[IDX_SIZE];
+        std::unique_ptr<MySQLConnectionInfo> _connectionInfo;
+        uint8 _async_threads, _synch_threads;
 };
 
 #endif
diff --git a/src/server/shared/Database/MySQLConnection.cpp b/src/server/shared/Database/MySQLConnection.cpp
index 1a9f973d47b..1fa3f01a5e1 100644
--- a/src/server/shared/Database/MySQLConnection.cpp
+++ b/src/server/shared/Database/MySQLConnection.cpp
@@ -72,7 +72,7 @@ void MySQLConnection::Close()
     delete this;
 }
 
-bool MySQLConnection::Open()
+uint32 MySQLConnection::Open()
 {
     MYSQL *mysqlInit;
     mysqlInit = mysql_init(NULL);
@@ -137,13 +137,13 @@ bool MySQLConnection::Open()
         // set connection properties to UTF8 to properly handle locales for different
         // server configs - core sends data in UTF8, so MySQL must expect UTF8 too
         mysql_set_character_set(m_Mysql, "utf8");
-        return PrepareStatements();
+        return 0;
     }
     else
     {
-        TC_LOG_ERROR("sql.sql", "Could not connect to MySQL database at %s: %s\n", m_connectionInfo.host.c_str(), mysql_error(mysqlInit));
+        TC_LOG_ERROR("sql.sql", "Could not connect to MySQL database at %s: %s", m_connectionInfo.host.c_str(), mysql_error(mysqlInit));
         mysql_close(mysqlInit);
-        return false;
+        return mysql_errno(mysqlInit);
     }
 }
 
diff --git a/src/server/shared/Database/MySQLConnection.h b/src/server/shared/Database/MySQLConnection.h
index d486f5b4679..78d8d2fb5dd 100644
--- a/src/server/shared/Database/MySQLConnection.h
+++ b/src/server/shared/Database/MySQLConnection.h
@@ -72,9 +72,11 @@ class MySQLConnection
         MySQLConnection(ProducerConsumerQueue<SQLOperation*>* queue, MySQLConnectionInfo& connInfo);  //! Constructor for asynchronous connections.
         virtual ~MySQLConnection();
 
-        virtual bool Open();
+        virtual uint32 Open();
         void Close();
 
+        bool PrepareStatements();
+
     public:
         bool Execute(const char* sql);
         bool Execute(PreparedStatement* stmt);
@@ -111,7 +113,6 @@ class MySQLConnection
         MySQLPreparedStatement* GetPreparedStatement(uint32 index);
         void PrepareStatement(uint32 index, const char* sql, ConnectionFlags flags);
 
-        bool PrepareStatements();
         virtual void DoPrepareStatements() = 0;
 
     protected: