Core/Crypto: Fixed openssl 3.0 compatibility for custom hmac_sha256 digest for RSA

2 files changed, 292 insertions, 36 deletions

diff --git a/src/common/Cryptography/RSA.cpp b/src/common/Cryptography/RSA.cpp
index 69f2916b343..844af854148 100644
--- a/src/common/Cryptography/RSA.cpp
+++ b/src/common/Cryptography/RSA.cpp
@@ -17,21 +17,28 @@
 
 #include "RSA.h"
 #include "HMAC.h"
+#include "Memory.h"
 #include <openssl/pem.h>
 #include <algorithm>
 #include <memory>
 #include <vector>
 #include <cstring>
 
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+#include <openssl/core_names.h>
+#include <openssl/params.h>
+#include <openssl/provider.h>
+#endif
+
 namespace
 {
-struct BIODeleter
-{
-    void operator()(BIO* bio)
-    {
-        BIO_free(bio);
-    }
-};
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+
+extern OSSL_DISPATCH const HMAC_SHA256_funcs[];
+extern OSSL_ALGORITHM const HMAC_SHA256_algs[];
+extern OSSL_DISPATCH const HMAC_SHA256_method[];
+
+#endif
 
 struct HMAC_SHA256_MD
 {
@@ -40,13 +47,7 @@ struct HMAC_SHA256_MD
         Trinity::Crypto::HMAC_SHA256* hmac;
     };
 
-#if TRINITY_COMPILER == TRINITY_COMPILER_GNU
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-#else
-#pragma warning(push)
-#pragma warning(disable: 4996)
-#endif
+#if OPENSSL_VERSION_NUMBER < 0x30000000L
 
     HMAC_SHA256_MD()
     {
@@ -74,13 +75,7 @@ struct HMAC_SHA256_MD
         _md = nullptr;
     }
 
-#if TRINITY_COMPILER == TRINITY_COMPILER_GNU
-#pragma GCC diagnostic pop
-#else
-#pragma warning(pop)
-#endif
-
-    operator EVP_MD const* () const
+    EVP_MD* GetMd() const
     {
         return _md;
     }
@@ -138,23 +133,242 @@ struct HMAC_SHA256_MD
 
 private:
     EVP_MD* _md;
-} HmacSha256Md;
+
+#else
+
+    HMAC_SHA256_MD()
+    {
+        _lib = OSSL_LIB_CTX_new();
+        OSSL_PROVIDER_add_builtin(_lib, "trinity-rsa-hmac-sha256", &InitProvider);
+        _handle = OSSL_PROVIDER_load(_lib, "trinity-rsa-hmac-sha256");
+        OSSL_PROVIDER_load(_lib, "default");
+    }
+
+    ~HMAC_SHA256_MD()
+    {
+        if (_handle)
+            OSSL_PROVIDER_unload(_handle);
+        if (_lib)
+            OSSL_LIB_CTX_free(_lib);
+    }
+
+    OSSL_LIB_CTX* GetLib() const
+    {
+        return _lib;
+    }
+
+    static int InitProvider(const OSSL_CORE_HANDLE* /*handle*/, const OSSL_DISPATCH* /*in*/, const OSSL_DISPATCH** out, void** /*provctx*/)
+    {
+        *out = HMAC_SHA256_method;
+        return 1;
+    }
+
+    static OSSL_ALGORITHM const* QueryProvider(void* /*provctx*/, int operation_id, int* no_cache)
+    {
+        *no_cache = 0;
+        if (operation_id == OSSL_OP_DIGEST)
+            return HMAC_SHA256_algs;
+
+        return nullptr;
+    }
+
+    static CTX_DATA* DigestNew()
+    {
+        CTX_DATA* data = new CTX_DATA();
+        data->hmac = nullptr;
+        return data;
+    }
+
+    static int DigestInit(void* dctx, OSSL_PARAM const* params)
+    {
+        CTX_DATA* ctxData = reinterpret_cast<CTX_DATA*>(dctx);
+
+        delete ctxData->hmac;
+        if (OSSL_PARAM const* keyParam = OSSL_PARAM_locate_const(params, "hmac-key"))
+        {
+            uint8 const* key = nullptr;
+            size_t keyLength = 0;
+            if (OSSL_PARAM_get_octet_ptr(keyParam, reinterpret_cast<void const**>(&key), &keyLength))
+            {
+                ctxData->hmac = new Trinity::Crypto::HMAC_SHA256(key, keyLength);
+                return 1;
+            }
+        }
+
+        return 0;
+    }
+
+    static int DigestUpdate(void* dctx, const unsigned char* in, size_t inl)
+    {
+        reinterpret_cast<CTX_DATA*>(dctx)->hmac->UpdateData(in, inl);
+        return 1;
+    }
+
+    static int DigestFinal(void* dctx, unsigned char* out, size_t* outl, size_t outsz)
+    {
+        CTX_DATA* ctxData = reinterpret_cast<CTX_DATA*>(dctx);
+        ctxData->hmac->Finalize();
+        *outl = std::min(ctxData->hmac->GetDigest().size(), outsz);
+        memcpy(out, ctxData->hmac->GetDigest().data(), *outl);
+        return 1;
+    }
+
+    static void DigestFree(void* dctx)
+    {
+        CTX_DATA* data = reinterpret_cast<CTX_DATA*>(dctx);
+        if (data->hmac)
+        {
+            delete data->hmac;
+            data->hmac = nullptr;
+        }
+        delete data;
+    }
+
+    static void* DigestDup(void* dctx)
+    {
+        CTX_DATA const* ctxDataFrom = reinterpret_cast<CTX_DATA const*>(dctx);
+        CTX_DATA* ctxDataTo = DigestNew();
+        if (ctxDataFrom->hmac)
+            ctxDataTo->hmac = new Trinity::Crypto::HMAC_SHA256(*ctxDataFrom->hmac);
+
+        return ctxDataTo;
+    }
+
+    static int DigestGetParams(OSSL_PARAM params[])
+    {
+        OSSL_PARAM* p = nullptr;
+
+        p = OSSL_PARAM_locate(params, OSSL_DIGEST_PARAM_BLOCK_SIZE);
+        if (p != nullptr && !OSSL_PARAM_set_size_t(p, SHA256_CBLOCK))
+            return 0;
+
+        p = OSSL_PARAM_locate(params, OSSL_DIGEST_PARAM_SIZE);
+        if (p != nullptr && !OSSL_PARAM_set_size_t(p, Trinity::Crypto::Constants::SHA256_DIGEST_LENGTH_BYTES))
+            return 0;
+
+        p = OSSL_PARAM_locate(params, OSSL_DIGEST_PARAM_XOF);
+        if (p != nullptr && !OSSL_PARAM_set_int(p, 0))
+            return 0;
+
+        p = OSSL_PARAM_locate(params, OSSL_DIGEST_PARAM_ALGID_ABSENT);
+        if (p != nullptr && !OSSL_PARAM_set_int(p, 1))
+            return 0;
+
+        return 1;
+    }
+
+    static OSSL_PARAM const* DigestGettableParams()
+    {
+        static constexpr OSSL_PARAM Params[] =
+        {
+            OSSL_PARAM_size_t(OSSL_DIGEST_PARAM_BLOCK_SIZE, NULL),
+            OSSL_PARAM_size_t(OSSL_DIGEST_PARAM_SIZE, NULL),
+            OSSL_PARAM_int(OSSL_DIGEST_PARAM_XOF, NULL),
+            OSSL_PARAM_int(OSSL_DIGEST_PARAM_ALGID_ABSENT, NULL),
+            OSSL_PARAM_END
+        };
+
+        return Params;
+    }
+
+private:
+    OSSL_LIB_CTX* _lib;
+    OSSL_PROVIDER* _handle;
+#endif
+} const HmacSha256Md;
+
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+
+OSSL_DISPATCH const HMAC_SHA256_funcs[] =
+{
+    { OSSL_FUNC_DIGEST_NEWCTX, (void (*)())HMAC_SHA256_MD::DigestNew },
+    { OSSL_FUNC_DIGEST_INIT, (void (*)())HMAC_SHA256_MD::DigestInit },
+    { OSSL_FUNC_DIGEST_UPDATE, (void (*)())HMAC_SHA256_MD::DigestUpdate },
+    { OSSL_FUNC_DIGEST_FINAL, (void (*)())HMAC_SHA256_MD::DigestFinal },
+    { OSSL_FUNC_DIGEST_FREECTX, (void (*)())HMAC_SHA256_MD::DigestFree },
+    { OSSL_FUNC_DIGEST_DUPCTX, (void (*)())HMAC_SHA256_MD::DigestDup },
+    { OSSL_FUNC_DIGEST_GET_PARAMS, (void (*)())HMAC_SHA256_MD::DigestGetParams },
+    { OSSL_FUNC_DIGEST_GETTABLE_PARAMS, (void (*)())HMAC_SHA256_MD::DigestGettableParams },
+    { 0, nullptr}
+};
+
+OSSL_ALGORITHM const HMAC_SHA256_algs[] =
+{
+    // pretend this custom HMAC_SHA256 is a regular SHA256 - openssl has a whitelist of allowed digests for RSA and HMAC_SHA256 is not on it
+    { OSSL_DIGEST_NAME_SHA2_256, "provider=trinity-rsa-hmac-sha256", HMAC_SHA256_funcs, "HMAC SHA265 \"digest\" for RSA" },
+    { nullptr, nullptr, nullptr, nullptr}
+};
+
+OSSL_DISPATCH const HMAC_SHA256_method[] =
+{
+    { OSSL_FUNC_PROVIDER_QUERY_OPERATION, (void(*)())HMAC_SHA256_MD::QueryProvider },
+    { 0, nullptr },
+};
+#endif
+
 }
 
 namespace Trinity::Crypto
 {
-EVP_MD const* RsaSignature::SHA256::GetGenerator() const
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+
+void RsaSignature::DigestGenerator::EVP_MD_Deleter::operator()(EVP_MD* md) const
+{
+    EVP_MD_free(md);
+}
+
+std::unique_ptr<EVP_MD, RsaSignature::DigestGenerator::EVP_MD_Deleter> RsaSignature::SHA256::GetGenerator() const
+{
+    return std::unique_ptr<EVP_MD, EVP_MD_Deleter>(EVP_MD_fetch(nullptr, OSSL_DIGEST_NAME_SHA2_256, "provider=default"));
+}
+
+OSSL_LIB_CTX* RsaSignature::SHA256::GetLib() const
 {
-    return EVP_sha256();
+    return nullptr;
+}
+
+std::unique_ptr<OSSL_PARAM[]> RsaSignature::SHA256::GetParams() const
+{
+    return nullptr;
+}
+
+std::unique_ptr<EVP_MD, RsaSignature::DigestGenerator::EVP_MD_Deleter> RsaSignature::HMAC_SHA256::GetGenerator() const
+{
+    return std::unique_ptr<EVP_MD, EVP_MD_Deleter>(EVP_MD_fetch(HmacSha256Md.GetLib(), OSSL_DIGEST_NAME_SHA2_256, "provider=trinity-rsa-hmac-sha256"));
+}
+
+OSSL_LIB_CTX* RsaSignature::HMAC_SHA256::GetLib() const
+{
+    return HmacSha256Md.GetLib();
+}
+
+std::unique_ptr<OSSL_PARAM[]> RsaSignature::HMAC_SHA256::GetParams() const
+{
+    return std::unique_ptr<OSSL_PARAM[]>(new OSSL_PARAM[2]
+    {
+        OSSL_PARAM_octet_ptr("hmac-key", const_cast<void**>(reinterpret_cast<void const* const*>(&_key)), _keyLength),
+        OSSL_PARAM_END
+    });
+}
+
+#else
+
+void RsaSignature::DigestGenerator::EVP_MD_Deleter::operator()(EVP_MD* /*md*/) const
+{
+}
+
+std::unique_ptr<EVP_MD, RsaSignature::DigestGenerator::EVP_MD_Deleter> RsaSignature::SHA256::GetGenerator() const
+{
+    return std::unique_ptr<EVP_MD, EVP_MD_Deleter>(const_cast<EVP_MD*>(EVP_sha256()));
 }
 
 void RsaSignature::SHA256::PostInitCustomizeContext(EVP_MD_CTX*)
 {
 }
 
-EVP_MD const* RsaSignature::HMAC_SHA256::GetGenerator() const
+std::unique_ptr<EVP_MD, RsaSignature::DigestGenerator::EVP_MD_Deleter> RsaSignature::HMAC_SHA256::GetGenerator() const
 {
-    return HmacSha256Md;
+    return std::unique_ptr<EVP_MD, EVP_MD_Deleter>(HmacSha256Md.GetMd());
 }
 
 void RsaSignature::HMAC_SHA256::PostInitCustomizeContext(EVP_MD_CTX* ctx)
@@ -165,6 +379,8 @@ void RsaSignature::HMAC_SHA256::PostInitCustomizeContext(EVP_MD_CTX* ctx)
     ctxData->hmac = new Crypto::HMAC_SHA256(_key, _keyLength);
 }
 
+#endif
+
 RsaSignature::RsaSignature() : _ctx(Impl::GenericHashImpl::MakeCTX())
 {
 }
@@ -214,7 +430,7 @@ bool RsaSignature::LoadKeyFromFile(std::string const& fileName)
         _key = nullptr;
     }
 
-    std::unique_ptr<BIO, BIODeleter> keyBIO(BIO_new_file(fileName.c_str(), "r"));
+    auto keyBIO = make_unique_ptr_with_deleter(BIO_new_file(fileName.c_str(), "r"), BIO_free);
     if (!keyBIO)
         return false;
 
@@ -233,9 +449,9 @@ bool RsaSignature::LoadKeyFromString(std::string const& keyPem)
         _key = nullptr;
     }
 
-    std::unique_ptr<BIO, BIODeleter> keyBIO(BIO_new_mem_buf(
+    auto keyBIO = make_unique_ptr_with_deleter(BIO_new_mem_buf(
         const_cast<char*>(keyPem.c_str()) /*api hack - this function assumes memory is readonly but lacks const modifier*/,
-        keyPem.length() + 1));
+        keyPem.length() + 1), BIO_free);
     if (!keyBIO)
         return false;
 
@@ -248,11 +464,27 @@ bool RsaSignature::LoadKeyFromString(std::string const& keyPem)
 
 bool RsaSignature::Sign(uint8 const* message, std::size_t messageLength, DigestGenerator& generator, std::vector<uint8>& output)
 {
-    size_t signatureLength = 0;
-    EVP_DigestSignInit(_ctx, nullptr, generator.GetGenerator(), nullptr, _key);
+    std::unique_ptr<EVP_MD, DigestGenerator::EVP_MD_Deleter> digestGenerator = generator.GetGenerator();
+
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+    auto keyCtx = make_unique_ptr_with_deleter(EVP_PKEY_CTX_new_from_pkey(generator.GetLib(), _key, nullptr), EVP_PKEY_CTX_free);
+    EVP_MD_CTX_set_pkey_ctx(_ctx, keyCtx.get());
+
+    std::unique_ptr<OSSL_PARAM[]> params = generator.GetParams();
+    int result = EVP_DigestSignInit_ex(_ctx, nullptr, EVP_MD_get0_name(digestGenerator.get()), generator.GetLib(),  nullptr, _key, params.get());
+#else
+    int result = EVP_DigestSignInit(_ctx, nullptr, digestGenerator.get(), nullptr, _key);
     generator.PostInitCustomizeContext(_ctx);
-    EVP_DigestSignUpdate(_ctx, message, messageLength);
-    int result = EVP_DigestSignFinal(_ctx, nullptr, &signatureLength);
+#endif
+    if (result == 0)
+        return false;
+
+    result = EVP_DigestSignUpdate(_ctx, message, messageLength);
+    if (result == 0)
+        return false;
+
+    size_t signatureLength = 0;
+    result = EVP_DigestSignFinal(_ctx, nullptr, &signatureLength);
     if (result == 0)
         return false;
 
diff --git a/src/common/Cryptography/RSA.h b/src/common/Cryptography/RSA.h
index 18771f14926..63f6b7b393c 100644
--- a/src/common/Cryptography/RSA.h
+++ b/src/common/Cryptography/RSA.h
@@ -21,6 +21,7 @@
 #include "Define.h"
 #include <openssl/evp.h>
 #include <array>
+#include <memory>
 #include <string>
 #include <vector>
 
@@ -34,16 +35,33 @@ public:
     class TC_COMMON_API DigestGenerator
     {
     public:
+        struct EVP_MD_Deleter
+        {
+            void operator()(EVP_MD* md) const;
+        };
+
         virtual ~DigestGenerator() = default;
-        virtual EVP_MD const* GetGenerator() const = 0;
+        virtual std::unique_ptr<EVP_MD, EVP_MD_Deleter> GetGenerator() const = 0;
+
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+        virtual OSSL_LIB_CTX* GetLib() const = 0;
+        virtual std::unique_ptr<OSSL_PARAM[]> GetParams() const = 0;
+#else
         virtual void PostInitCustomizeContext(EVP_MD_CTX* ctx) = 0;
+#endif
     };
 
     class TC_COMMON_API SHA256 : public DigestGenerator
     {
     public:
-        EVP_MD const* GetGenerator() const override;
+        std::unique_ptr<EVP_MD, EVP_MD_Deleter> GetGenerator() const override;
+
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+        OSSL_LIB_CTX* GetLib() const override;
+        std::unique_ptr<OSSL_PARAM[]> GetParams() const override;
+#else
         void PostInitCustomizeContext(EVP_MD_CTX* ctx) override;
+#endif
     };
 
     class TC_COMMON_API HMAC_SHA256 : public DigestGenerator
@@ -51,8 +69,14 @@ public:
     public:
         explicit HMAC_SHA256(uint8 const* key, size_t keyLength) : _key(key), _keyLength(keyLength) { }
 
-        EVP_MD const* GetGenerator() const override;
+        std::unique_ptr<EVP_MD, EVP_MD_Deleter> GetGenerator() const override;
+
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+        OSSL_LIB_CTX* GetLib() const override;
+        std::unique_ptr<OSSL_PARAM[]> GetParams() const override;
+#else
         void PostInitCustomizeContext(EVP_MD_CTX* ctx) override;
+#endif
 
     private:
         uint8 const* _key;