database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

MCOL-4516 check for var_pop < 0

Browse Source 
In some cases, because of rounding error, var_pop will evaluate to some value just less than 0. We check for this and force to round to 0.
This commit is contained in:
David Hall
2021-03-09 13:36:10 -06:00
parent 508d5455a8
commit af20387985
8 changed files with 37 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
utils/regr/regr_syy.cpp
View File

@ -129,6 +129,8 @@ mcsv1_UDAF::ReturnCode regr_syy::evaluate(mcsv1Context* context, static_any::any
if (N > 0)
{
long double var_popy = (data->sumy2 - (data->sumy * data->sumy / N));
if (var_popy < 0) // might be -0
var_popy = 0;
valOut = static_cast<double>(var_popy);
}
return mcsv1_UDAF::SUCCESS;