database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-01 06:46:55 +03:00
Code Activity

MCOL-4516 check for var_pop < 0

Browse Source 
In some cases, because of rounding error, var_pop will evaluate to some value just less than 0. We check for this and force to round to 0.
This commit is contained in:
David Hall
2021-03-09 13:36:10 -06:00
parent 508d5455a8
commit af20387985
8 changed files with 37 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
utils/regr/covar_pop.cpp
View File

@ -149,7 +149,7 @@ mcsv1_UDAF::ReturnCode covar_pop::evaluate(mcsv1Context* context, static_any::an
long double sumy = data->sumy;
long double sumxy = data->sumxy;
long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N ;
long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N;
valOut = static_cast<double>(covar_pop);
}
return mcsv1_UDAF::SUCCESS;