Core/Crypto: Switch away from most deprecated openssl functions and removed upper version limit

3 files changed, 28 insertions, 48 deletions

diff --git a/src/server/game/Server/Packets/AuthenticationPackets.cpp b/src/server/game/Server/Packets/AuthenticationPackets.cpp
index f999709a061..e9004a8f5da 100644
--- a/src/server/game/Server/Packets/AuthenticationPackets.cpp
+++ b/src/server/game/Server/Packets/AuthenticationPackets.cpp
@@ -234,13 +234,13 @@ OHYtKG3GK3GEcFDwZU2LPHq21EroUAdtRfbrJ4KW2yc8igtXKxTBYw==
 -----END RSA PRIVATE KEY-----
 )";
 
-std::unique_ptr<Trinity::Crypto::RSA> ConnectToRSA;
+std::unique_ptr<Trinity::Crypto::RsaSignature> ConnectToRSA;
 }
 
 bool WorldPackets::Auth::ConnectTo::InitializeEncryption()
 {
-    std::unique_ptr<Trinity::Crypto::RSA> rsa = std::make_unique<Trinity::Crypto::RSA>();
-    if (!rsa->LoadFromString(RSAPrivateKey, Trinity::Crypto::RSA::PrivateKey{}))
+    std::unique_ptr<Trinity::Crypto::RsaSignature> rsa = std::make_unique<Trinity::Crypto::RsaSignature>();
+    if (!rsa->LoadKeyFromString(RSAPrivateKey))
         return false;
 
     ConnectToRSA = std::move(rsa);
@@ -270,16 +270,15 @@ WorldPacket const* WorldPackets::Auth::ConnectTo::Write()
             break;
     }
 
-    uint32 type = Payload.Where.Type;
-    Trinity::Crypto::SHA256 hash;
-    hash.UpdateData(whereBuffer.contents(), whereBuffer.size());
-    hash.UpdateData(reinterpret_cast<uint8 const*>(&type), 4);
-    hash.UpdateData(reinterpret_cast<uint8 const*>(&Payload.Port), 2);
-    hash.Finalize();
+    ByteBuffer signBuffer;
+    signBuffer.append(whereBuffer);
+    signBuffer << uint32(Payload.Where.Type);
+    signBuffer << uint16(Payload.Port);
+    Trinity::Crypto::RsaSignature::SHA256 digestGenerator;
+    std::vector<uint8> signature;
+    ConnectToRSA->Sign(signBuffer.contents(), signBuffer.size(), digestGenerator, signature);
 
-    ConnectToRSA->Sign(hash.GetDigest(), Payload.Signature.data(), Trinity::Crypto::RSA::SHA256{});
-
-    _worldPacket.append(Payload.Signature.data(), Payload.Signature.size());
+    _worldPacket.append(signature.data(), signature.size());
     _worldPacket.append(whereBuffer);
     _worldPacket << uint16(Payload.Port);
     _worldPacket << uint32(Serial);
@@ -307,15 +306,16 @@ uint8 constexpr EnableEncryptionSeed[16] = { 0x90, 0x9C, 0xD0, 0x50, 0x5A, 0x2C,
 
 WorldPacket const* WorldPackets::Auth::EnterEncryptedMode::Write()
 {
-    Trinity::Crypto::HMAC_SHA256 hash(EncryptionKey, 16);
-    hash.UpdateData(reinterpret_cast<uint8 const*>(&Enabled), 1);
-    hash.UpdateData(EnableEncryptionSeed, 16);
-    hash.Finalize();
+    std::array<uint8, 17> msg{};
+    msg[0] = Enabled ? 1 : 0;
+    std::copy_n(std::begin(EnableEncryptionSeed), std::size(EnableEncryptionSeed), &msg[1]);
 
-    _worldPacket.resize(_worldPacket.size() + ConnectToRSA->GetOutputSize());
+    Trinity::Crypto::RsaSignature::HMAC_SHA256 digestGenerator(EncryptionKey, 16);
+    std::vector<uint8> signature;
 
-    ConnectToRSA->Sign(hash.GetDigest(), _worldPacket.contents(), Trinity::Crypto::RSA::SHA256{});
+    ConnectToRSA->Sign(msg, digestGenerator, signature);
 
+    _worldPacket.append(signature.data(), signature.size());
     _worldPacket.WriteBit(Enabled);
     _worldPacket.FlushBits();
 
diff --git a/src/server/game/Warden/Warden.cpp b/src/server/game/Warden/Warden.cpp
index e4cba951f44..cd3fc94c467 100644
--- a/src/server/game/Warden/Warden.cpp
+++ b/src/server/game/Warden/Warden.cpp
@@ -19,6 +19,7 @@
 #include "AccountMgr.h"
 #include "ByteBuffer.h"
 #include "Common.h"
+#include "CryptoHash.h"
 #include "GameTime.h"
 #include "Log.h"
 #include "SmartEnum.h"
@@ -27,10 +28,6 @@
 #include "World.h"
 #include "WorldPacket.h"
 #include "WorldSession.h"
-
-#include <openssl/md5.h>
-#include <openssl/sha.h>
-
 #include <charconv>
 
 Warden::Warden() : _session(nullptr), _checkTimer(10 * IN_MILLISECONDS), _clientResponseTimer(0),
@@ -48,10 +45,7 @@ void Warden::MakeModuleForClient()
     TC_LOG_DEBUG("warden", "Make module for client");
     InitializeModuleForClient(_module.emplace());
 
-    MD5_CTX ctx;
-    MD5_Init(&ctx);
-    MD5_Update(&ctx, _module->CompressedData, _module->CompressedSize);
-    MD5_Final(_module->Id.data(), &ctx);
+    _module->Id = Trinity::Crypto::MD5::GetDigestOf(_module->CompressedData, _module->CompressedSize);
 }
 
 void Warden::SendModuleToClient()
@@ -161,28 +155,19 @@ bool Warden::IsValidCheckSum(uint32 checksum, uint8 const* data, const uint16 le
     }
 }
 
-struct keyData {
-    union
-    {
-        struct
-        {
-            uint8 bytes[20];
-        } bytes;
-
-        struct
-        {
-            uint32 ints[5];
-        } ints;
-    };
+union keyData
+{
+    std::array<uint8, 20> bytes;
+    std::array<uint32, 5> ints;
 };
 
 uint32 Warden::BuildChecksum(uint8 const* data, uint32 length)
 {
     keyData hash;
-    SHA1(data, length, hash.bytes.bytes);
+    hash.bytes = Trinity::Crypto::SHA1::GetDigestOf(data, size_t(length));
     uint32 checkSum = 0;
     for (uint8 i = 0; i < 5; ++i)
-        checkSum = checkSum ^ hash.ints.ints[i];
+        checkSum = checkSum ^ hash.ints[i];
 
     return checkSum;
 }
diff --git a/src/server/game/Warden/WardenMac.cpp b/src/server/game/Warden/WardenMac.cpp
index f8a82f6b6b6..e7c7541c8ea 100644
--- a/src/server/game/Warden/WardenMac.cpp
+++ b/src/server/game/Warden/WardenMac.cpp
@@ -18,6 +18,7 @@
 #include "WardenMac.h"
 #include "ByteBuffer.h"
 #include "Common.h"
+#include "CryptoHash.h"
 #include "GameTime.h"
 #include "Log.h"
 #include "Opcodes.h"
@@ -27,7 +28,6 @@
 #include "WorldPacket.h"
 #include "WorldSession.h"
 
-#include <openssl/md5.h>
 #include <array>
 
 WardenMac::WardenMac() : Warden() { }
@@ -230,12 +230,7 @@ void WardenMac::HandleCheckResult(ByteBuffer &buff)
         //found = true;
     }
 
-    MD5_CTX ctx;
-    MD5_Init(&ctx);
-    MD5_Update(&ctx, str.c_str(), str.size());
-    std::array<uint8, 16> ourMD5Hash;
-    MD5_Final(ourMD5Hash.data(), &ctx);
-
+    std::array<uint8, 16> ourMD5Hash = Trinity::Crypto::MD5::GetDigestOf(str);
     std::array<uint8, 16> theirsMD5Hash;
     buff.read(theirsMD5Hash);