database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-02 17:22:27 +03:00
Code Activity

MCOL-4516 check for var_pop < 0

Browse Source 
In some cases, because of rounding error, var_pop will evaluate to some value just less than 0. We check for this and force to round to 0.
This commit is contained in:
David Hall
2021-03-09 13:36:10 -06:00
parent 508d5455a8
commit af20387985
8 changed files with 37 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
utils/regr/regrmysql.cpp
View File

@ -537,7 +537,7 @@ extern "C"
long double sumxy = data->sumxy;
long double covar_pop = N * sumxy - sumx * sumy;
long double var_pop = N * sumx2 - sumx * sumx;
if (var_pop != 0)
if (var_pop > 0)
{
valOut = static_cast<double>(covar_pop / var_pop);
*is_null = 0;
@ -657,7 +657,7 @@ extern "C"
long double sumxy = data->sumxy;
long double numerator = sumy * sumx2 - sumx * sumxy;
long double var_pop = (N * sumx2) - (sumx * sumx);
if (var_pop != 0)
if (var_pop > 0)
{
valOut = static_cast<double>(numerator / var_pop);
*is_null = 0;
@ -781,14 +781,14 @@ extern "C"
long double sumy2 = data->sumy2;
long double sumxy = data->sumxy;
long double var_popx = (sumx2 - (sumx * sumx / N)) / N;
if (var_popx == 0)
if (var_popx <= 0) // Catch -0
{
// When var_popx is 0, NULL is the result.
*is_null = 1;
return 0;
}
long double var_popy = (sumy2 - (sumy * sumy / N)) / N;
if (var_popy == 0)
if (var_popy <= 0) // Catch -0
{
// When var_popy is 0, 1 is the result
return 1;
@ -921,14 +921,14 @@ extern "C"
long double sumy2 = data->sumy2;
long double sumxy = data->sumxy;
long double var_popx = (sumx2 - (sumx * sumx / N)) / N;
if (var_popx == 0)
if (var_popx <= 0) // Catch -0
{
// When var_popx is 0, NULL is the result.
*is_null = 1;
return 0;
}
long double var_popy = (sumy2 - (sumy * sumy / N)) / N;
if (var_popy == 0)
if (var_popy <= 0) // Catch -0
{
// When var_popy is 0, 1 is the result
return 1;
@ -1044,8 +1044,10 @@ extern "C"
{
long double sumx = data->sumx;
long double sumx2 = data->sumx2;
long double var_popx = (sumx2 - (sumx * sumx / N)) / N;
valOut = static_cast<double>(N * var_popx);
long double sxx = (sumx2 - (sumx * sumx / N));
if (sxx < 0) // catch -0
sxx = 0;
valOut = static_cast<double>(sxx);
}
else
{
@ -1151,8 +1153,10 @@ extern "C"
{
long double sumy = data->sumy;
long double sumy2 = data->sumy2;
long double var_popy = (sumy2 - (sumy * sumy / N)) / N;
valOut = static_cast<double>(N * var_popy);
long double syy = (sumy2 - (sumy * sumy / N));
if (syy < 0) // might be -0
syy = 0;
valOut = static_cast<double>(syy);
}
else
{
@ -1265,8 +1269,7 @@ extern "C"
long double sumx = data->sumx;
long double sumy = data->sumy;
long double sumxy = data->sumxy;
long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N;
long double regr_sxy = N * covar_pop;
long double regr_sxy = (sumxy - ((sumx * sumy) / N));
valOut = static_cast<double>(regr_sxy);
}
else
@ -1380,7 +1383,7 @@ extern "C"
long double sumx = data->sumx;
long double sumy = data->sumy;
long double sumxy = data->sumxy;
long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N ;
long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N;
valOut = static_cast<double>(covar_pop);
}
else