database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-01 06:46:55 +03:00
Code Activity

MCOL-1983 Use long double for internal calculations of regr_*** functions.

Browse Source
This commit is contained in:
David Hall
2019-01-21 14:39:51 -06:00
parent eb75d3cd7e
commit 16a6a0a27c
11 changed files with 100 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
utils/regr/regr_syy.cpp
View File

@ -39,8 +39,8 @@ static Add_regr_syy_ToUDAFMap addToMap;
struct regr_syy_data
{
uint64_t cnt;
double sumy;
double sumy2; // sum of (y squared)
long double sumy;
long double sumy2; // sum of (y squared)
};
@ -125,14 +125,14 @@ mcsv1_UDAF::ReturnCode regr_syy::subEvaluate(mcsv1Context* context, const UserDa
mcsv1_UDAF::ReturnCode regr_syy::evaluate(mcsv1Context* context, static_any::any& valOut)
{
struct regr_syy_data* data = (struct regr_syy_data*)context->getUserData()->data;
double N = data->cnt;
long double N = data->cnt;
if (N > 0)
{
double sumy = data->sumy;
double sumy2 = data->sumy2;
long double sumy = data->sumy;
long double sumy2 = data->sumy2;
double var_popy = (sumy2 - (sumy * sumy / N)) / N;
valOut = data->cnt * var_popy;
long double var_popy = (sumy2 - (sumy * sumy / N)) / N;
valOut = static_cast<double>(data->cnt * var_popy);
}
return mcsv1_UDAF::SUCCESS;
}