Core/Authserver: Auth cleanup phase 1a, the "stuff I ran across while making phase 2" commit.

- Did you know BigNumber quietly assumes every byte array it gets is little-endian, even though openssl bignums use big-endian? Now you do!
- In entirely unrelated news, make the above behavior explicit through a default-true boolean, same as existing GetBytes derivatives.
- Also, if you are in the enlightened openssl 1.1 crowd, there's no more endian wrangling involved, because openssl now does all of that for us. Progress!

(cherry picked from commit 5e36bf7c67e077bd1664eee59d5758fbae7666cd)

1 files changed, 21 insertions, 7 deletions

diff --git a/src/common/Cryptography/BigNumber.cpp b/src/common/Cryptography/BigNumber.cpp
index ef5a413d964..613cd49ae3f 100644
--- a/src/common/Cryptography/BigNumber.cpp
+++ b/src/common/Cryptography/BigNumber.cpp
@@ -47,16 +47,25 @@ void BigNumber::SetQword(uint64 val)
     BN_add_word(_bn, (uint32)(val & 0xFFFFFFFF));
 }
 
-void BigNumber::SetBinary(uint8 const* bytes, int32 len)
+void BigNumber::SetBinary(uint8 const* bytes, int32 len, bool littleEndian)
 {
-    uint8* array = new uint8[len];
+    if (littleEndian)
+    {
+#if defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER < 0x10100000L
+        uint8* array = new uint8[len];
 
-    for (int i = 0; i < len; i++)
-        array[i] = bytes[len - 1 - i];
+        for (int i = 0; i < len; i++)
+            array[i] = bytes[len - 1 - i];
 
-    BN_bin2bn(array, len, _bn);
+        BN_bin2bn(array, len, _bn);
 
-    delete[] array;
+        delete[] array;
+#else
+        BN_lebin2bn(bytes, len, _bn);
+#endif
+    }
+    else
+        BN_bin2bn(bytes, len, _bn);
 }
 
 bool BigNumber::SetHexStr(char const* str)
@@ -168,8 +177,9 @@ bool BigNumber::IsNegative() const
     return BN_is_negative(_bn);
 }
 
-void BigNumber::GetBytes(uint8* buf, std::size_t bufsize, bool littleEndian) const
+void BigNumber::GetBytes(uint8* buf, size_t bufsize, bool littleEndian) const
 {
+#if defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER < 0x10100000L
     int nBytes = GetNumBytes();
     ASSERT(nBytes >= 0, "Bignum has negative number of bytes (%d).", nBytes);
     std::size_t numBytes = static_cast<std::size_t>(nBytes);
@@ -186,6 +196,10 @@ void BigNumber::GetBytes(uint8* buf, std::size_t bufsize, bool littleEndian) con
     // openssl's BN stores data internally in big endian format, reverse if little endian desired
     if (littleEndian)
         std::reverse(buf, buf + bufsize);
+#else
+    int res = littleEndian ? BN_bn2lebinpad(_bn, buf, bufsize) : BN_bn2binpad(_bn, buf, bufsize);
+    ASSERT(res > 0, "Buffer of size %zu is too small to hold bignum with %zu bytes.\n", bufsize, BN_num_bytes(_bn));
+#endif
 }
 
 std::vector<uint8> BigNumber::ToByteVector(int32 minSize, bool littleEndian) const