database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-01 06:46:55 +03:00
Code Activity

MCOL-4516 check for var_pop < 0

Browse Source 
In some cases, because of rounding error, var_pop will evaluate to some value just less than 0. We check for this and force to round to 0.
This commit is contained in:
David Hall
2021-03-09 13:36:10 -06:00
parent 508d5455a8
commit af20387985
8 changed files with 37 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
utils/regr/regr_sxx.cpp
View File

@ -128,8 +128,10 @@ mcsv1_UDAF::ReturnCode regr_sxx::evaluate(mcsv1Context* context, static_any::any
long double N = data->cnt;
if (N > 0)
{
long double var_popx = (data->sumx2 - (data->sumx * data->sumx / N));
valOut = static_cast<double>(var_popx);
long double regr_sxx = (data->sumx2 - (data->sumx * data->sumx / N));
if (regr_sxx < 0) // catch -0
regr_sxx = 0;
valOut = static_cast<double>(regr_sxx);
}
return mcsv1_UDAF::SUCCESS;
}