Core/Database: Use RAII for resource management in MySQLConnection

* Prevents double deletion of MySQLConnection after errors
* The object stays valid after an error and will wait for a reconnect
* Also crash the server if 5 reconnects fail
* Corrects an issue where the server was crashed after one reconnect
  because mysql_thread_id was invoked with an invalid handle

(cherry picked from commit 62815c6e1c0427e1d0229d02b1ba70449654cded)

2 files changed, 58 insertions, 37 deletions

diff --git a/src/server/database/Database/MySQLConnection.cpp b/src/server/database/Database/MySQLConnection.cpp
index a7afaac41c5..fed368f2aac 100644
--- a/src/server/database/Database/MySQLConnection.cpp
+++ b/src/server/database/Database/MySQLConnection.cpp
@@ -39,7 +39,6 @@ MySQLConnection::MySQLConnection(MySQLConnectionInfo& connInfo) :
 m_reconnecting(false),
 m_prepareError(false),
 m_queue(NULL),
-m_worker(NULL),
 m_Mysql(NULL),
 m_connectionInfo(connInfo),
 m_connectionFlags(CONNECTION_SYNCH) { }
@@ -52,24 +51,26 @@ m_Mysql(NULL),
 m_connectionInfo(connInfo),
 m_connectionFlags(CONNECTION_ASYNC)
 {
-    m_worker = new DatabaseWorker(m_queue, this);
+    m_worker = Trinity::make_unique<DatabaseWorker>(m_queue, this);
 }
 
 MySQLConnection::~MySQLConnection()
 {
-    delete m_worker;
-
-    for (size_t i = 0; i < m_stmts.size(); ++i)
-        delete m_stmts[i];
-
-    if (m_Mysql)
-        mysql_close(m_Mysql);
+    Close();
 }
 
 void MySQLConnection::Close()
 {
-    /// Only close us if we're not operating
-    delete this;
+    // Stop the worker thread before the statements are cleared
+    m_worker.reset();
+
+    m_stmts.clear();
+
+    if (m_Mysql)
+    {
+        mysql_close(m_Mysql);
+        m_Mysql = nullptr;
+    }
 }
 
 uint32 MySQLConnection::Open()
@@ -414,7 +415,7 @@ int MySQLConnection::ExecuteTransaction(SQLTransaction& transaction)
 MySQLPreparedStatement* MySQLConnection::GetPreparedStatement(uint32 index)
 {
     ASSERT(index < m_stmts.size());
-    MySQLPreparedStatement* ret = m_stmts[index];
+    MySQLPreparedStatement* ret = m_stmts[index].get();
     if (!ret)
         TC_LOG_ERROR("sql.sql", "Could not fetch prepared statement %u on database `%s`, connection type: %s.",
             index, m_connectionInfo.database.c_str(), (m_connectionFlags & CONNECTION_ASYNC) ? "asynchronous" : "synchronous");
@@ -426,16 +427,12 @@ void MySQLConnection::PrepareStatement(uint32 index, const char* sql, Connection
 {
     m_queries.insert(PreparedStatementMap::value_type(index, std::make_pair(sql, flags)));
 
-    // For reconnection case
-    if (m_reconnecting)
-        delete m_stmts[index];
-
     // Check if specified query should be prepared on this connection
     // i.e. don't prepare async statements on synchronous connections
     // to save memory that will not be used.
     if (!(m_connectionFlags & flags))
     {
-        m_stmts[index] = NULL;
+        m_stmts[index].reset();
         return;
     }
 
@@ -457,8 +454,7 @@ void MySQLConnection::PrepareStatement(uint32 index, const char* sql, Connection
         }
         else
         {
-            MySQLPreparedStatement* mStmt = new MySQLPreparedStatement(stmt);
-            m_stmts[index] = mStmt;
+            m_stmts[index] = Trinity::make_unique<MySQLPreparedStatement>(stmt);
         }
     }
 }
@@ -479,7 +475,7 @@ PreparedResultSet* MySQLConnection::Query(PreparedStatement* stmt)
     return new PreparedResultSet(stmt->m_stmt->GetSTMT(), result, rowCount, fieldCount);
 }
 
-bool MySQLConnection::_HandleMySQLErrno(uint32 errNo)
+bool MySQLConnection::_HandleMySQLErrno(uint32 errNo, uint8 attempts /*= 5*/)
 {
     switch (errNo)
     {
@@ -488,9 +484,21 @@ bool MySQLConnection::_HandleMySQLErrno(uint32 errNo)
         case CR_INVALID_CONN_HANDLE:
         case CR_SERVER_LOST_EXTENDED:
         {
+            if (m_Mysql)
+            {
+                TC_LOG_ERROR("sql.sql", "Lost the connection to the MySQL server!");
+
+                mysql_close(GetHandle());
+                m_Mysql = nullptr;
+            }
+
+            /*no break*/
+        }
+        case CR_CONN_HOST_ERROR:
+        {
+            TC_LOG_INFO("sql.sql", "Attempting to reconnect to the MySQL server...");
+
             m_reconnecting = true;
-            uint64 oldThreadId = mysql_thread_id(GetHandle());
-            mysql_close(GetHandle());
 
             uint32 const lErrno = Open();
             if (!lErrno)
@@ -498,24 +506,37 @@ bool MySQLConnection::_HandleMySQLErrno(uint32 errNo)
                 // Don't remove 'this' pointer unless you want to skip loading all prepared statements...
                 if (!this->PrepareStatements())
                 {
-                    TC_LOG_ERROR("sql.sql", "Could not re-prepare statements!");
-                    Close();
-                    return false;
+                    TC_LOG_FATAL("sql.sql", "Could not re-prepare statements!");
+                    std::this_thread::sleep_for(std::chrono::seconds(10));
+                    std::abort();
                 }
 
-                TC_LOG_INFO("sql.sql", "Connection to the MySQL server is active.");
-                if (oldThreadId != mysql_thread_id(GetHandle()))
-                    TC_LOG_INFO("sql.sql", "Successfully reconnected to %s @%s:%s (%s).",
-                        m_connectionInfo.database.c_str(), m_connectionInfo.host.c_str(), m_connectionInfo.port_or_socket.c_str(),
-                            (m_connectionFlags & CONNECTION_ASYNC) ? "asynchronous" : "synchronous");
+                TC_LOG_INFO("sql.sql", "Successfully reconnected to %s @%s:%s (%s).",
+                    m_connectionInfo.database.c_str(), m_connectionInfo.host.c_str(), m_connectionInfo.port_or_socket.c_str(),
+                        (m_connectionFlags & CONNECTION_ASYNC) ? "asynchronous" : "synchronous");
 
                 m_reconnecting = false;
                 return true;
             }
 
-            // It's possible this attempted reconnect throws 2006 at us. To prevent crazy recursive calls, sleep here.
-            std::this_thread::sleep_for(std::chrono::seconds(3)); // Sleep 3 seconds
-            return _HandleMySQLErrno(lErrno);                     // Call self (recursive)
+            if ((--attempts) == 0)
+            {
+                // Shut down the server when the mysql server isn't
+                // reachable for some time
+                TC_LOG_FATAL("sql.sql", "Failed to reconnect to the MySQL server, "
+                             "terminating the server to prevent data corruption!");
+
+                // We could also initiate a shutdown through using std::raise(SIGTERM)
+                std::this_thread::sleep_for(std::chrono::seconds(10));
+                std::abort();
+            }
+            else
+            {
+                // It's possible this attempted reconnect throws 2006 at us.
+                // To prevent crazy recursive calls, sleep here.
+                std::this_thread::sleep_for(std::chrono::seconds(3)); // Sleep 3 seconds
+                return _HandleMySQLErrno(lErrno, attempts); // Call self (recursive)
+            }
         }
 
         case ER_LOCK_DEADLOCK:
diff --git a/src/server/database/Database/MySQLConnection.h b/src/server/database/Database/MySQLConnection.h
index a981caa607e..a0b908593df 100644
--- a/src/server/database/Database/MySQLConnection.h
+++ b/src/server/database/Database/MySQLConnection.h
@@ -116,18 +116,18 @@ class MySQLConnection
         virtual void DoPrepareStatements() = 0;
 
     protected:
-        std::vector<MySQLPreparedStatement*> m_stmts;         //! PreparedStatements storage
+        std::vector<std::unique_ptr<MySQLPreparedStatement>> m_stmts; //! PreparedStatements storage
         PreparedStatementMap                 m_queries;       //! Query storage
         bool                                 m_reconnecting;  //! Are we reconnecting?
         bool                                 m_prepareError;  //! Was there any error while preparing statements?
 
     private:
-        bool _HandleMySQLErrno(uint32 errNo);
+        bool _HandleMySQLErrno(uint32 errNo, uint8 attempts = 5);
 
     private:
         ProducerConsumerQueue<SQLOperation*>* m_queue;      //! Queue shared with other asynchronous connections.
-        DatabaseWorker*       m_worker;                     //! Core worker task.
-        MYSQL *               m_Mysql;                      //! MySQL Handle.
+        std::unique_ptr<DatabaseWorker> m_worker;           //! Core worker task.
+        MYSQL*                m_Mysql;                      //! MySQL Handle.
         MySQLConnectionInfo&  m_connectionInfo;             //! Connection info (used for logging)
         ConnectionFlags       m_connectionFlags;            //! Connection flags (for preparing relevant statements)
         std::mutex            m_Mutex;