From 8332ab8974d9df6e3923a5da9ea04e62988a4450 Mon Sep 17 00:00:00 2001
From: David Hall <david.hall@mariadb.com>
Date: Tue, 6 Jul 2021 19:50:00 -0500
Subject: [PATCH] MCOL-4738 AVG() returns a wrong result On AMD64 machines, the
 fpu is 80 bits. The unused bits must be masked for memcmp to work properly.
 For other archetectures, we don't want to mask those bits.

---
 CMakeLists.txt              |  4 +++-
 cmake/configureEngine.cmake | 18 ++++++++++++++++++
 utils/rowgroup/rowgroup.h   | 10 ++++------
 3 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2f48dc57e..4bbdfe3fb 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -198,7 +198,9 @@ ELSE ()
     MY_CHECK_AND_SET_COMPILER_FLAG("-D_DEBUG -O0" DEBUG)
 ENDIF()
 
-
+IF (MASK_LONGDOUBLE)
+    MY_CHECK_AND_SET_COMPILER_FLAG("-DMASK_LONGDOUBLE")
+ENDIF()
 
 SET (ENGINE_LDFLAGS    "-Wl,--no-as-needed -Wl,--add-needed")
 SET (ENGINE_DT_LIB datatypes)
diff --git a/cmake/configureEngine.cmake b/cmake/configureEngine.cmake
index 07c5326d6..305f9f9a8 100644
--- a/cmake/configureEngine.cmake
+++ b/cmake/configureEngine.cmake
@@ -6,6 +6,7 @@
 INCLUDE (CheckIncludeFiles)
 INCLUDE (CheckIncludeFileCXX)
 INCLUDE (CheckCSourceCompiles)
+INCLUDE (CheckCXXSourceRuns)
 INCLUDE (CheckCXXSourceCompiles)
 INCLUDE (CheckStructHasMember)
 INCLUDE (CheckLibraryExists)
@@ -719,3 +720,20 @@ EXECUTE_PROCESS(
     COMMAND rm -f conftest.data conftest.file conftest.sym
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
     )
+    
+CHECK_CXX_SOURCE_RUNS("
+#include <limits>
+int main()
+{
+    // If long double is 16 bytes and digits and exponent are 64 and 16384 respectively, then we need to mask out the 
+    // unused bits, as they contain garbage. There are times we test for equality by memcmp of a buffer containing,
+    // in part, the long double set here. Garbage bytes will adversly affect that compare.
+    // Note: There may be compilers that store 80 bit floats in 12 bytes. We do not account for that here. I don't believe
+    // there are any modern Linux compilers that do that as a default. Windows uses 64 bits, so no masking is needed.
+    if (std::numeric_limits<long double>::digits == 64 
+     && std::numeric_limits<long double>::max_exponent == 16384 
+     && sizeof(long double) == 16)
+        return 0;
+    return 1; 
+}"
+MASK_LONGDOUBLE)
diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h
index 79a0f4b29..7ea19979e 100644
--- a/utils/rowgroup/rowgroup.h
+++ b/utils/rowgroup/rowgroup.h
@@ -1369,12 +1369,10 @@ inline void Row::setFloatField(float val, uint32_t colIndex)
 inline void Row::setLongDoubleField(const long double& val, uint32_t colIndex)
 {
     uint8_t* p = &data[offsets[colIndex]];
-    *((long double*)p) = val;
-    if (sizeof(long double) == 16)
-    {
-        // zero out the unused portion as there may be garbage there.
-        *((uint64_t*)p+1) &= 0x000000000000FFFFULL;
-    }
+    *reinterpret_cast<long double*>(p) = val;
+#ifdef MASK_LONGDOUBLE
+    *(reinterpret_cast<uint64_t*>(p)+1) &= 0x000000000000FFFFULL;
+#endif    
 }
 
 inline void Row::setInt128Field(const int128_t& val, uint32_t colIndex)