database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

MCOL-3503 Use long double arithmetic for tie breaking to avoid integer truncation

Browse Source
This commit is contained in:
David Hall
2019-10-11 12:05:13 -05:00
parent 5a4c406216
commit c7d594304b
1 changed files with 5 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
utils/regr/moda.cpp
View File

@ -280,7 +280,7 @@ template<class T>
mcsv1_UDAF::ReturnCode Moda_impl_T<T>::evaluate(mcsv1Context* context, static_any::any& valOut)
{
uint64_t maxCnt = 0;
T avg = 0;
long double avg = 0;
T val = 0;
ModaData* data = static_cast<ModaData*>(context->getUserData());
std::unordered_map<T, uint32_t>* map = data->getMap<T>();
@ -304,8 +304,10 @@ mcsv1_UDAF::ReturnCode Moda_impl_T<T>::evaluate(mcsv1Context* context, static_an
else if (iter->second == maxCnt)
{
// Tie breaker: choose the closest to avg. If still tie, choose smallest
if ((std::fabs(val-avg) > std::fabs(iter->first-avg))
|| ((std::fabs(val-avg) == std::fabs(iter->first-avg)) && (std::fabs(val) > std::fabs(iter->first))))
long double dist1 = val > avg ? (long double)val-avg : avg-(long double)val;
long double dist2 = iter->first > avg ? (long double)iter->first-avg : avg-(long double)iter->first;
if ((dist1 > dist2)
|| ((dist1 == dist2) && (std::fabs(val) > std::fabs(iter->first))))
{
val = iter->first;
}