From 95fcc3dcbbf65570844c64a99d6c1113f001da1d Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 20 Aug 2019 09:50:43 -0500 Subject: [PATCH] MCOL-3423 Don't move decimal for LONG DOUBLE. Clear long double extra bits after copy, not before. --- utils/rowgroup/rowgroup.h | 5 +++-- utils/windowfunction/wf_stats.cpp | 7 +++++-- utils/windowfunction/wf_sum_avg.cpp | 6 ++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 2334d22bc..1e5df447b 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -1036,12 +1036,13 @@ inline void Row::setFloatField(float val, uint32_t colIndex) inline void Row::setLongDoubleField(long double val, uint32_t colIndex) { + uint8_t* p = &data[offsets[colIndex]]; + *((long double*)p) = val; if (sizeof(long double) == 16) { // zero out the unused portion as there may be garbage there. - *((uint64_t*)&val+1) &= 0x000000000000FFFFULL; + *((uint64_t*)p+1) &= 0x000000000000FFFFULL; } - *((long double*) &data[offsets[colIndex]]) = val; } inline void Row::setVarBinaryField(const std::string& val, uint32_t colIndex) diff --git a/utils/windowfunction/wf_stats.cpp b/utils/windowfunction/wf_stats.cpp index db4b107ee..8b06eda61 100644 --- a/utils/windowfunction/wf_stats.cpp +++ b/utils/windowfunction/wf_stats.cpp @@ -140,6 +140,7 @@ void WF_stats::resetData() template void WF_stats::operator()(int64_t b, int64_t e, int64_t c) { + CDT cdt; if ((fFrameUnit == WF__FRAME_ROWS) || (fPrev == -1) || (!fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(fPrev))))) @@ -163,7 +164,7 @@ void WF_stats::operator()(int64_t b, int64_t e, int64_t c) continue; T valIn; - getValue(colIn, valIn); + getValue(colIn, valIn, &cdt); long double val = (long double) valIn; fSum1 += val; @@ -177,7 +178,9 @@ void WF_stats::operator()(int64_t b, int64_t e, int64_t c) int scale = fRow.getScale(colIn); long double factor = pow(10.0, scale); - if (scale != 0) // adjust the scale if necessary + // adjust the scale if necessary + if (scale != 0 && + cdt != CalpontSystemCatalog::LONGDOUBLE) { fSum1 /= factor; fSum2 /= factor * factor; diff --git a/utils/windowfunction/wf_sum_avg.cpp b/utils/windowfunction/wf_sum_avg.cpp index 4632496df..d0d5f46f7 100644 --- a/utils/windowfunction/wf_sum_avg.cpp +++ b/utils/windowfunction/wf_sum_avg.cpp @@ -264,13 +264,15 @@ void WF_sum_avg::operator()(int64_t b, int64_t e, int64_t c) continue; T valIn; - getValue(colIn, valIn); + CDT cdt; + getValue(colIn, valIn, &cdt); // checkSumLimit(fSum, valIn); if ((!fDistinct) || (fSet.find(valIn) == fSet.end())) { long double val = valIn; - if (scale) + if (scale && + cdt != CalpontSystemCatalog::LONGDOUBLE) { val /= pow(10.0, scale); }