Shared/Database: Improve dead-lock error handling

Improve dead-lock error handling in async transactions by retrying the transaction for up to 60 seconds instead of just 5 times

1 files changed, 19 insertions, 3 deletions

diff --git a/src/server/database/Database/Transaction.cpp b/src/server/database/Database/Transaction.cpp
index 02c08f4b70e..499720a7841 100644
--- a/src/server/database/Database/Transaction.cpp
+++ b/src/server/database/Database/Transaction.cpp
@@ -15,13 +15,19 @@
  * with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "Log.h"
 #include "Transaction.h"
 #include "MySQLConnection.h"
 #include "PreparedStatement.h"
+#include "Timer.h"
 #include <mysqld_error.h>
+#include <sstream>
+#include <thread>
 
 std::mutex TransactionTask::_deadlockLock;
 
+#define DEADLOCK_MAX_RETRY_TIME_MS 60000
+
 //- Append a raw ad-hoc query to the transaction
 void Transaction::Append(char const* sql)
 {
@@ -71,12 +77,22 @@ bool TransactionTask::Execute()
 
     if (errorCode == ER_LOCK_DEADLOCK)
     {
+        std::ostringstream threadIdStream;
+        threadIdStream << std::this_thread::get_id();
+        std::string threadId = threadIdStream.str();
+
         // Make sure only 1 async thread retries a transaction so they don't keep dead-locking each other
         std::lock_guard<std::mutex> lock(_deadlockLock);
-        uint8 loopBreaker = 5;  // Handle MySQL Errno 1213 without extending deadlock to the core itself
-        for (uint8 i = 0; i < loopBreaker; ++i)
-            if (!m_conn->ExecuteTransaction(m_trans))
+        
+        for (uint32 loopDuration = 0, startMSTime = getMSTime(); loopDuration <= DEADLOCK_MAX_RETRY_TIME_MS; loopDuration = GetMSTimeDiffToNow(startMSTime))
+        {
+            if (!m_conn->ExecuteTransaction(m_trans))                
                 return true;
+
+            TC_LOG_WARN("sql.sql", "Deadlocked SQL Transaction, retrying. Loop timer: %u ms, Thread Id: %s", loopDuration, threadId.c_str());
+        }
+
+        TC_LOG_ERROR("sql.sql", "Fatal deadlocked SQL Transaction, it will not be retried anymore. Thread Id: %s", threadId.c_str());
     }
 
     // Clean up now.