database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-4516 check for var_pop < 0

Browse Source 
In some cases, because of rounding error, var_pop will evaluate to some value just less than 0. We check for this and force to round to 0.
This commit is contained in:
David Hall
2021-03-09 13:36:10 -06:00
parent 508d5455a8
commit af20387985
8 changed files with 37 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
utils/regr/regr_slope.cpp
View File

@ -152,12 +152,14 @@ mcsv1_UDAF::ReturnCode regr_slope::evaluate(mcsv1Context* context, static_any::a
long double sumy = data->sumy;
long double sumx2 = data->sumx2;
long double sumxy = data->sumxy;
// These aren't really covar_pop and var_pop. For the purposes of this calculation
// we multiplied everything by N to reduce calc time and variance.
// It all comes out after the final divide
long double covar_pop = N * sumxy - sumx * sumy;
long double var_pop = N * sumx2 - sumx * sumx;
if (var_pop != 0)
if (var_pop > 0)
{
long double slope = covar_pop / var_pop;
valOut = static_cast<double>(slope);
valOut = static_cast<double>(covar_pop / var_pop);
}
}
return mcsv1_UDAF::SUCCESS;