Core/Authserver: Auth cleanup phase 1a, the "stuff I ran across while making phase 2" commit.

- Did you know BigNumber quietly assumes every byte array it gets is little-endian, even though openssl bignums use big-endian? Now you do!
- In entirely unrelated news, make the above behavior explicit through a default-true boolean, same as existing GetBytes derivatives.
- Also, if you are in the enlightened openssl 1.1 crowd, there's no more endian wrangling involved, because openssl now does all of that for us. Progress!

(cherry picked from commit 5e36bf7c67e077bd1664eee59d5758fbae7666cd)

2 files changed, 24 insertions, 10 deletions

diff --git a/src/common/Cryptography/BigNumber.cpp b/src/common/Cryptography/BigNumber.cpp
index ef5a413d964..613cd49ae3f 100644
--- a/src/common/Cryptography/BigNumber.cpp
+++ b/src/common/Cryptography/BigNumber.cpp
@@ -47,16 +47,25 @@ void BigNumber::SetQword(uint64 val)
     BN_add_word(_bn, (uint32)(val & 0xFFFFFFFF));
 }
 
-void BigNumber::SetBinary(uint8 const* bytes, int32 len)
+void BigNumber::SetBinary(uint8 const* bytes, int32 len, bool littleEndian)
 {
-    uint8* array = new uint8[len];
+    if (littleEndian)
+    {
+#if defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER < 0x10100000L
+        uint8* array = new uint8[len];
 
-    for (int i = 0; i < len; i++)
-        array[i] = bytes[len - 1 - i];
+        for (int i = 0; i < len; i++)
+            array[i] = bytes[len - 1 - i];
 
-    BN_bin2bn(array, len, _bn);
+        BN_bin2bn(array, len, _bn);
 
-    delete[] array;
+        delete[] array;
+#else
+        BN_lebin2bn(bytes, len, _bn);
+#endif
+    }
+    else
+        BN_bin2bn(bytes, len, _bn);
 }
 
 bool BigNumber::SetHexStr(char const* str)
@@ -168,8 +177,9 @@ bool BigNumber::IsNegative() const
     return BN_is_negative(_bn);
 }
 
-void BigNumber::GetBytes(uint8* buf, std::size_t bufsize, bool littleEndian) const
+void BigNumber::GetBytes(uint8* buf, size_t bufsize, bool littleEndian) const
 {
+#if defined(OPENSSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER < 0x10100000L
     int nBytes = GetNumBytes();
     ASSERT(nBytes >= 0, "Bignum has negative number of bytes (%d).", nBytes);
     std::size_t numBytes = static_cast<std::size_t>(nBytes);
@@ -186,6 +196,10 @@ void BigNumber::GetBytes(uint8* buf, std::size_t bufsize, bool littleEndian) con
     // openssl's BN stores data internally in big endian format, reverse if little endian desired
     if (littleEndian)
         std::reverse(buf, buf + bufsize);
+#else
+    int res = littleEndian ? BN_bn2lebinpad(_bn, buf, bufsize) : BN_bn2binpad(_bn, buf, bufsize);
+    ASSERT(res > 0, "Buffer of size %zu is too small to hold bignum with %zu bytes.\n", bufsize, BN_num_bytes(_bn));
+#endif
 }
 
 std::vector<uint8> BigNumber::ToByteVector(int32 minSize, bool littleEndian) const
diff --git a/src/common/Cryptography/BigNumber.h b/src/common/Cryptography/BigNumber.h
index 1f5050bc4f5..66ea0e53b76 100644
--- a/src/common/Cryptography/BigNumber.h
+++ b/src/common/Cryptography/BigNumber.h
@@ -34,15 +34,15 @@ class TC_COMMON_API BigNumber
         BigNumber(uint32 v) : BigNumber() { SetDword(v); }
         BigNumber(std::string const& v) : BigNumber() { SetHexStr(v); }
         template <size_t Size>
-        BigNumber(std::array<uint8, Size> const& v) : BigNumber() { SetBinary(v.data(), Size); }
+        BigNumber(std::array<uint8, Size> const& v, bool littleEndian = true) : BigNumber() { SetBinary(v.data(), Size, littleEndian); }
 
         ~BigNumber();
 
         void SetDword(uint32);
         void SetQword(uint64);
-        void SetBinary(uint8 const* bytes, int32 len);
+        void SetBinary(uint8 const* bytes, int32 len, bool littleEndian = true);
         template <typename Container>
-        void SetBinary(Container const& c) { SetBinary(advstd::data(c), advstd::size(c)); }
+        auto SetBinary(Container const& c, bool littleEndian = true) -> std::enable_if_t<!std::is_pointer_v<Container>> { SetBinary(advstd::data(c), advstd::size(c), littleEndian); }
         bool SetHexStr(char const* str);
         bool SetHexStr(std::string const& str) { return SetHexStr(str.c_str()); }