Core/Crypto: Transitional Cryptography update for OpenSSL 1.1 (#21533)

Support for both OpenSSL 1.0 LTS and OpenSSL 1.1 versions.

Many Linux distributions are still on 1.0 and will stay on LTS for quite
some time.

Port of CMaNGOS commit: https://github.com/cmangos/mangos-wotlk/commit/e1b0048f052eda46bb27d20224d0339960816ac2

5 files changed, 48 insertions, 33 deletions

diff --git a/cmake/macros/FindOpenSSL.cmake b/cmake/macros/FindOpenSSL.cmake
index ca67601427c..d970ef02f65 100644
--- a/cmake/macros/FindOpenSSL.cmake
+++ b/cmake/macros/FindOpenSSL.cmake
@@ -26,7 +26,7 @@
 # http://www.slproweb.com/products/Win32OpenSSL.html
 
 set(OPENSSL_EXPECTED_VERSION "1.0")
-set(OPENSSL_MAX_VERSION "1.1")
+set(OPENSSL_MAX_VERSION "1.2")
 
 SET(_OPENSSL_ROOT_HINTS
   "[HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\OpenSSL (32-bit)_is1;Inno Setup: App Path]"
diff --git a/src/common/Cryptography/ARC4.cpp b/src/common/Cryptography/ARC4.cpp
index a05cf49d6f8..a5e77e92646 100644
--- a/src/common/Cryptography/ARC4.cpp
+++ b/src/common/Cryptography/ARC4.cpp
@@ -18,34 +18,34 @@
 
 #include "ARC4.h"
 
-ARC4::ARC4(uint32 len) : m_ctx()
+ARC4::ARC4(uint32 len) : m_ctx(EVP_CIPHER_CTX_new())
 {
-    EVP_CIPHER_CTX_init(&m_ctx);
-    EVP_EncryptInit_ex(&m_ctx, EVP_rc4(), nullptr, nullptr, nullptr);
-    EVP_CIPHER_CTX_set_key_length(&m_ctx, len);
+    EVP_CIPHER_CTX_init(m_ctx);
+    EVP_EncryptInit_ex(m_ctx, EVP_rc4(), nullptr, nullptr, nullptr);
+    EVP_CIPHER_CTX_set_key_length(m_ctx, len);
 }
 
-ARC4::ARC4(uint8 *seed, uint32 len) : m_ctx()
+ARC4::ARC4(uint8* seed, uint32 len) : m_ctx(EVP_CIPHER_CTX_new())
 {
-    EVP_CIPHER_CTX_init(&m_ctx);
-    EVP_EncryptInit_ex(&m_ctx, EVP_rc4(), nullptr, nullptr, nullptr);
-    EVP_CIPHER_CTX_set_key_length(&m_ctx, len);
-    EVP_EncryptInit_ex(&m_ctx, nullptr, nullptr, seed, nullptr);
+    EVP_CIPHER_CTX_init(m_ctx);
+    EVP_EncryptInit_ex(m_ctx, EVP_rc4(), nullptr, nullptr, nullptr);
+    EVP_CIPHER_CTX_set_key_length(m_ctx, len);
+    EVP_EncryptInit_ex(m_ctx, nullptr, nullptr, seed, nullptr);
 }
 
 ARC4::~ARC4()
 {
-    EVP_CIPHER_CTX_cleanup(&m_ctx);
+    EVP_CIPHER_CTX_free(m_ctx);
 }
 
-void ARC4::Init(uint8 *seed)
+void ARC4::Init(uint8* seed)
 {
-    EVP_EncryptInit_ex(&m_ctx, nullptr, nullptr, seed, nullptr);
+    EVP_EncryptInit_ex(m_ctx, nullptr, nullptr, seed, nullptr);
 }
 
-void ARC4::UpdateData(int len, uint8 *data)
+void ARC4::UpdateData(int len, uint8* data)
 {
     int outlen = 0;
-    EVP_EncryptUpdate(&m_ctx, data, &outlen, data, len);
-    EVP_EncryptFinal_ex(&m_ctx, data, &outlen);
+    EVP_EncryptUpdate(m_ctx, data, &outlen, data, len);
+    EVP_EncryptFinal_ex(m_ctx, data, &outlen);
 }
diff --git a/src/common/Cryptography/ARC4.h b/src/common/Cryptography/ARC4.h
index 4f7fba1f31d..629be510bbc 100644
--- a/src/common/Cryptography/ARC4.h
+++ b/src/common/Cryptography/ARC4.h
@@ -19,19 +19,19 @@
 #ifndef _AUTH_SARC4_H
 #define _AUTH_SARC4_H
 
-#include <openssl/evp.h>
 #include "Define.h"
+#include <openssl/evp.h>
 
 class TC_COMMON_API ARC4
 {
     public:
         ARC4(uint32 len);
-        ARC4(uint8 *seed, uint32 len);
+        ARC4(uint8* seed, uint32 len);
         ~ARC4();
-        void Init(uint8 *seed);
-        void UpdateData(int len, uint8 *data);
+        void Init(uint8* seed);
+        void UpdateData(int len, uint8* data);
     private:
-        EVP_CIPHER_CTX m_ctx;
+        EVP_CIPHER_CTX* m_ctx;
 };
 
 #endif
diff --git a/src/common/Cryptography/HMACSHA1.cpp b/src/common/Cryptography/HMACSHA1.cpp
index ce8df47b605..c5e6e448fe3 100644
--- a/src/common/Cryptography/HMACSHA1.cpp
+++ b/src/common/Cryptography/HMACSHA1.cpp
@@ -21,38 +21,53 @@
 #include "Errors.h"
 #include <cstring>
 
+#if defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER < 0x10100000L
+HMAC_CTX* HMAC_CTX_new()
+{
+    HMAC_CTX *ctx = new HMAC_CTX();
+    HMAC_CTX_init(ctx);
+    return ctx;
+}
+
+void HMAC_CTX_free(HMAC_CTX* ctx)
+{
+    HMAC_CTX_cleanup(ctx);
+    delete ctx;
+}
+#endif
+
 HmacHash::HmacHash(uint32 len, uint8* seed)
 {
-    HMAC_CTX_init(&m_ctx);
-    HMAC_Init_ex(&m_ctx, seed, len, EVP_sha1(), nullptr);
+    m_ctx = HMAC_CTX_new();
+    HMAC_Init_ex(m_ctx, seed, len, EVP_sha1(), nullptr);
     memset(m_digest, 0, sizeof(m_digest));
 }
 
 HmacHash::~HmacHash()
 {
-    HMAC_CTX_cleanup(&m_ctx);
+    HMAC_CTX_free(m_ctx);
 }
 
 void HmacHash::UpdateData(std::string const& str)
 {
-    HMAC_Update(&m_ctx, (uint8 const*)str.c_str(), str.length());
+    HMAC_Update(m_ctx, reinterpret_cast<uint8 const*>(str.c_str()), str.length());
 }
 
 void HmacHash::UpdateData(uint8 const* data, size_t len)
 {
-    HMAC_Update(&m_ctx, data, len);
+    HMAC_Update(m_ctx, data, len);
 }
 
 void HmacHash::Finalize()
 {
     uint32 length = 0;
-    HMAC_Final(&m_ctx, (uint8*)m_digest, &length);
+    HMAC_Final(m_ctx, m_digest, &length);
     ASSERT(length == SHA_DIGEST_LENGTH);
 }
 
-uint8 *HmacHash::ComputeHash(BigNumber* bn)
+uint8* HmacHash::ComputeHash(BigNumber* bn)
 {
-    HMAC_Update(&m_ctx, bn->AsByteArray().get(), bn->GetNumBytes());
+    HMAC_Update(m_ctx, bn->AsByteArray().get(), bn->GetNumBytes());
     Finalize();
-    return (uint8*)m_digest;
+    return m_digest;
 }
diff --git a/src/common/Cryptography/HMACSHA1.h b/src/common/Cryptography/HMACSHA1.h
index 2e5437ebca7..5057791e4ed 100644
--- a/src/common/Cryptography/HMACSHA1.h
+++ b/src/common/Cryptography/HMACSHA1.h
@@ -36,11 +36,11 @@ class TC_COMMON_API HmacHash
         void UpdateData(std::string const& str);
         void UpdateData(uint8 const* data, size_t len);
         void Finalize();
-        uint8 *ComputeHash(BigNumber* bn);
-        uint8 *GetDigest() { return (uint8*)m_digest; }
+        uint8* ComputeHash(BigNumber* bn);
+        uint8* GetDigest() { return m_digest; }
         int GetLength() const { return SHA_DIGEST_LENGTH; }
     private:
-        HMAC_CTX m_ctx;
+        HMAC_CTX* m_ctx;
         uint8 m_digest[SHA_DIGEST_LENGTH];
 };
 #endif