database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

MCOL-4738 AVG() returns a wrong result

Browse Source 
On AMD64 machines, the fpu is 80 bits. The unused bits must be masked for memcmp to work properly. For other archetectures, we don't want to mask those bits.
This commit is contained in:
David Hall
2021-07-06 19:50:00 -05:00
parent b9bd207d3b
commit 8332ab8974
3 changed files with 25 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
CMakeLists.txt
View File

@ -198,7 +198,9 @@ ELSE ()
MY_CHECK_AND_SET_COMPILER_FLAG("-D_DEBUG -O0" DEBUG)
ENDIF()
IF (MASK_LONGDOUBLE)
MY_CHECK_AND_SET_COMPILER_FLAG("-DMASK_LONGDOUBLE")
ENDIF()
SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed")
SET (ENGINE_DT_LIB datatypes)

18
cmake/configureEngine.cmake
View File

@ -6,6 +6,7 @@
INCLUDE (CheckIncludeFiles)
INCLUDE (CheckIncludeFileCXX)
INCLUDE (CheckCSourceCompiles)
INCLUDE (CheckCXXSourceRuns)
INCLUDE (CheckCXXSourceCompiles)
INCLUDE (CheckStructHasMember)
INCLUDE (CheckLibraryExists)
@ -719,3 +720,20 @@ EXECUTE_PROCESS(
COMMAND rm -f conftest.data conftest.file conftest.sym
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
)
CHECK_CXX_SOURCE_RUNS("
#include <limits>
int main()
{
// If long double is 16 bytes and digits and exponent are 64 and 16384 respectively, then we need to mask out the
// unused bits, as they contain garbage. There are times we test for equality by memcmp of a buffer containing,
// in part, the long double set here. Garbage bytes will adversly affect that compare.
// Note: There may be compilers that store 80 bit floats in 12 bytes. We do not account for that here. I don't believe
// there are any modern Linux compilers that do that as a default. Windows uses 64 bits, so no masking is needed.
if (std::numeric_limits<long double>::digits == 64
&& std::numeric_limits<long double>::max_exponent == 16384
&& sizeof(long double) == 16)
return 0;
return 1;
}"
MASK_LONGDOUBLE)

10
utils/rowgroup/rowgroup.h
View File

@ -1369,12 +1369,10 @@ inline void Row::setFloatField(float val, uint32_t colIndex)
inline void Row::setLongDoubleField(const long double& val, uint32_t colIndex)
{
uint8_t* p = &data[offsets[colIndex]];
*((long double*)p) = val;
if (sizeof(long double) == 16)
{
// zero out the unused portion as there may be garbage there.
*((uint64_t*)p+1) &= 0x000000000000FFFFULL;
}
*reinterpret_cast<long double*>(p) = val;
#ifdef MASK_LONGDOUBLE
*(reinterpret_cast<uint64_t*>(p)+1) &= 0x000000000000FFFFULL;
#endif
}
inline void Row::setInt128Field(const int128_t& val, uint32_t colIndex)