database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-1983 Use long double for internal calculations of regr_*** functions.

Browse Source
This commit is contained in:
David Hall
2019-01-21 14:39:51 -06:00
parent eb75d3cd7e
commit 16a6a0a27c
11 changed files with 100 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
utils/regr/corr.cpp
View File

@ -39,11 +39,11 @@ static Add_corr_ToUDAFMap addToMap;
struct corr_data struct corr_data
{ {
uint64_t cnt; uint64_t cnt;
double sumx; long double sumx;
double sumx2; // sum of (x squared) long double sumx2; // sum of (x squared)
double sumy; long double sumy;
double sumy2; // sum of (y squared) long double sumy2; // sum of (y squared)
double sumxy; // sum of x * y long double sumxy; // sum of x * y
}; };
@ -153,29 +153,29 @@ mcsv1_UDAF::ReturnCode corr::evaluate(mcsv1Context* context, static_any::any& va
double N = data->cnt; double N = data->cnt;
if (N > 1) if (N > 1)
{ {
double sumx = data->sumx; long double sumx = data->sumx;
double sumy = data->sumy; long double sumy = data->sumy;
double sumx2 = data->sumx2; long double sumx2 = data->sumx2;
double sumy2 = data->sumy2; long double sumy2 = data->sumy2;
double sumxy = data->sumxy; long double sumxy = data->sumxy;
double var_popx = (sumx2 - (sumx * sumx / N)) / N; long double var_popx = (sumx2 - (sumx * sumx / N)) / N;
if (var_popx == 0) if (var_popx == 0)
{ {
// When var_popx is 0, NULL is the result. // When var_popx is 0, NULL is the result.
return mcsv1_UDAF::SUCCESS; return mcsv1_UDAF::SUCCESS;
} }
double var_popy = (sumy2 - (sumy * sumy / N)) / N; long double var_popy = (sumy2 - (sumy * sumy / N)) / N;
if (var_popy == 0) if (var_popy == 0)
{ {
// When var_popy is 0, NULL is the result // When var_popy is 0, NULL is the result
return mcsv1_UDAF::SUCCESS; return mcsv1_UDAF::SUCCESS;
} }
double std_popx = sqrt(var_popx); long double std_popx = sqrt(var_popx);
double std_popy = sqrt(var_popy); long double std_popy = sqrt(var_popy);
double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N;
double corr = covar_pop / (std_popy * std_popx); long double corr = covar_pop / (std_popy * std_popx);
valOut = corr; valOut = static_cast<double>(corr);
} }
return mcsv1_UDAF::SUCCESS; return mcsv1_UDAF::SUCCESS;
} }

16
utils/regr/covar_pop.cpp
View File

@ -39,9 +39,9 @@ static Add_covar_pop_ToUDAFMap addToMap;
struct covar_pop_data struct covar_pop_data
{ {
uint64_t cnt; uint64_t cnt;
double sumx; long double sumx;
double sumy; long double sumy;
double sumxy; // sum of x * y long double sumxy; // sum of x * y
}; };
@ -145,12 +145,12 @@ mcsv1_UDAF::ReturnCode covar_pop::evaluate(mcsv1Context* context, static_any::an
double N = data->cnt; double N = data->cnt;
if (N > 0) if (N > 0)
{ {
double sumx = data->sumx; long double sumx = data->sumx;
double sumy = data->sumy; long double sumy = data->sumy;
double sumxy = data->sumxy; long double sumxy = data->sumxy;
double covar_pop = (sumxy - ((sumx * sumy) / N)) / N ; long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N ;
valOut = covar_pop; valOut = static_cast<double>(covar_pop);
} }
return mcsv1_UDAF::SUCCESS; return mcsv1_UDAF::SUCCESS;
} }

16
utils/regr/covar_samp.cpp
View File

@ -39,9 +39,9 @@ static Add_covar_samp_ToUDAFMap addToMap;
struct covar_samp_data struct covar_samp_data
{ {
uint64_t cnt; uint64_t cnt;
double sumx; long double sumx;
double sumy; long double sumy;
double sumxy; // sum of x * y long double sumxy; // sum of x * y
}; };
@ -145,12 +145,12 @@ mcsv1_UDAF::ReturnCode covar_samp::evaluate(mcsv1Context* context, static_any::a
double N = data->cnt; double N = data->cnt;
if (N > 1) if (N > 1)
{ {
double sumx = data->sumx; long double sumx = data->sumx;
double sumy = data->sumy; long double sumy = data->sumy;
double sumxy = data->sumxy; long double sumxy = data->sumxy;
double covar_samp = (sumxy - ((sumx * sumy) / N)) / (N - 1); long double covar_samp = (sumxy - ((sumx * sumy) / N)) / (N - 1);
valOut = covar_samp; valOut = static_cast<double>(covar_samp);
} }
else else
if (N == 1) if (N == 1)

4
utils/regr/regr_avgx.cpp
View File

@ -40,7 +40,7 @@ static Add_regr_avgx_ToUDAFMap addToMap;
// Use the simple data model // Use the simple data model
struct regr_avgx_data struct regr_avgx_data
{ {
double sum; long double sum;
uint64_t cnt; uint64_t cnt;
}; };
@ -132,7 +132,7 @@ mcsv1_UDAF::ReturnCode regr_avgx::evaluate(mcsv1Context* context, static_any::an
if (data->cnt > 0) if (data->cnt > 0)
{ {
valOut = data->sum / (double)data->cnt; valOut = static_cast<double>(data->sum / (long double)data->cnt);
} }
return mcsv1_UDAF::SUCCESS; return mcsv1_UDAF::SUCCESS;

4
utils/regr/regr_avgy.cpp
View File

@ -40,7 +40,7 @@ static Add_regr_avgy_ToUDAFMap addToMap;
// Use the simple data model // Use the simple data model
struct regr_avgy_data struct regr_avgy_data
{ {
double sum; long double sum;
uint64_t cnt; uint64_t cnt;
}; };
@ -130,7 +130,7 @@ mcsv1_UDAF::ReturnCode regr_avgy::evaluate(mcsv1Context* context, static_any::an
if (data->cnt > 0) if (data->cnt > 0)
{ {
valOut = data->sum / (double)data->cnt; valOut = static_cast<double>(data->sum / (long double)data->cnt);
} }
return mcsv1_UDAF::SUCCESS; return mcsv1_UDAF::SUCCESS;
} }

22
utils/regr/regr_intercept.cpp
View File

@ -39,10 +39,10 @@ static Add_regr_intercept_ToUDAFMap addToMap;
struct regr_intercept_data struct regr_intercept_data
{ {
uint64_t cnt; uint64_t cnt;
double sumx; long double sumx;
double sumx2; // sum of (x squared) long double sumx2; // sum of (x squared)
double sumy; long double sumy;
double sumxy; // sum of (x*y) long double sumxy; // sum of x * y
}; };
@ -148,15 +148,15 @@ mcsv1_UDAF::ReturnCode regr_intercept::evaluate(mcsv1Context* context, static_an
double N = data->cnt; double N = data->cnt;
if (N > 1) if (N > 1)
{ {
double sumx = data->sumx; long double sumx = data->sumx;
double sumy = data->sumy; long double sumy = data->sumy;
double sumx2 = data->sumx2; long double sumx2 = data->sumx2;
double sumxy = data->sumxy; long double sumxy = data->sumxy;
double numerator = sumy * sumx2 - sumx * sumxy; long double numerator = sumy * sumx2 - sumx * sumxy;
double var_pop = (N * sumx2) - (sumx * sumx); long double var_pop = (N * sumx2) - (sumx * sumx);
if (var_pop != 0) if (var_pop != 0)
{ {
valOut = numerator / var_pop; valOut = static_cast<double>(numerator / var_pop);
} }
} }
return mcsv1_UDAF::SUCCESS; return mcsv1_UDAF::SUCCESS;

32
utils/regr/regr_r2.cpp
View File

@ -39,11 +39,11 @@ static Add_regr_r2_ToUDAFMap addToMap;
struct regr_r2_data struct regr_r2_data
{ {
uint64_t cnt; uint64_t cnt;
double sumx; long double sumx;
double sumx2; // sum of (x squared) long double sumx2; // sum of (x squared)
double sumy; long double sumy;
double sumy2; // sum of (y squared) long double sumy2; // sum of (y squared)
double sumxy; // sum of x * y long double sumxy; // sum of x * y
}; };
@ -153,13 +153,13 @@ mcsv1_UDAF::ReturnCode regr_r2::evaluate(mcsv1Context* context, static_any::any&
double N = data->cnt; double N = data->cnt;
if (N > 1) if (N > 1)
{ {
double sumx = data->sumx; long double sumx = data->sumx;
double sumy = data->sumy; long double sumy = data->sumy;
double sumx2 = data->sumx2; long double sumx2 = data->sumx2;
double sumy2 = data->sumy2; long double sumy2 = data->sumy2;
double sumxy = data->sumxy; long double sumxy = data->sumxy;
double var_popx = (sumx2 - (sumx * sumx / N)) / N; long double var_popx = (sumx2 - (sumx * sumx / N)) / N;
if (var_popx == 0) if (var_popx == 0)
{ {
// When var_popx is 0, NULL is the result. // When var_popx is 0, NULL is the result.
@ -172,11 +172,11 @@ mcsv1_UDAF::ReturnCode regr_r2::evaluate(mcsv1Context* context, static_any::any&
valOut = 1.0; valOut = 1.0;
return mcsv1_UDAF::SUCCESS; return mcsv1_UDAF::SUCCESS;
} }
double std_popx = sqrt(var_popx); long double std_popx = sqrt(var_popx);
double std_popy = sqrt(var_popy); long double std_popy = sqrt(var_popy);
double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N;
double corr = covar_pop / (std_popy * std_popx); long double corr = covar_pop / (std_popy * std_popx);
valOut = corr * corr; valOut = static_cast<double>(corr * corr);
} }
return mcsv1_UDAF::SUCCESS; return mcsv1_UDAF::SUCCESS;
} }

24
utils/regr/regr_slope.cpp
View File

@ -39,10 +39,10 @@ static Add_regr_slope_ToUDAFMap addToMap;
struct regr_slope_data struct regr_slope_data
{ {
uint64_t cnt; uint64_t cnt;
double sumx; long double sumx;
double sumx2; // sum of (x squared) long double sumx2; // sum of (x squared)
double sumy; long double sumy;
double sumxy; // sum of (x*y) long double sumxy; // sum of x * y
}; };
@ -148,16 +148,16 @@ mcsv1_UDAF::ReturnCode regr_slope::evaluate(mcsv1Context* context, static_any::a
if (N > 1) if (N > 1)
{ {
// COVAR_POP(y, x) / VAR_POP(x) // COVAR_POP(y, x) / VAR_POP(x)
double sumx = data->sumx; long double sumx = data->sumx;
double sumy = data->sumy; long double sumy = data->sumy;
double sumx2 = data->sumx2; long double sumx2 = data->sumx2;
double sumxy = data->sumxy; long double sumxy = data->sumxy;
double covar_pop = N * sumxy - sumx * sumy; long double covar_pop = N * sumxy - sumx * sumy;
double var_pop = N * sumx2 - sumx * sumx; long double var_pop = N * sumx2 - sumx * sumx;
if (var_pop != 0) if (var_pop != 0)
{ {
double slope = covar_pop / var_pop; long double slope = covar_pop / var_pop;
valOut = slope; valOut = static_cast<double>(slope);
} }
} }
return mcsv1_UDAF::SUCCESS; return mcsv1_UDAF::SUCCESS;

12
utils/regr/regr_sxx.cpp
View File

@ -39,8 +39,8 @@ static Add_regr_sxx_ToUDAFMap addToMap;
struct regr_sxx_data struct regr_sxx_data
{ {
uint64_t cnt; uint64_t cnt;
double sumx; long double sumx;
double sumx2; // sum of (x squared) long double sumx2; // sum of (x squared)
}; };
@ -128,11 +128,11 @@ mcsv1_UDAF::ReturnCode regr_sxx::evaluate(mcsv1Context* context, static_any::any
double N = data->cnt; double N = data->cnt;
if (N > 0) if (N > 0)
{ {
double sumx = data->sumx; long double sumx = data->sumx;
double sumx2 = data->sumx2; long double sumx2 = data->sumx2;
double var_popx = (sumx2 - (sumx * sumx / N)) / N; long double var_popx = (sumx2 - (sumx * sumx / N)) / N;
valOut = data->cnt * var_popx; valOut = static_cast<double>(data->cnt * var_popx);
} }
return mcsv1_UDAF::SUCCESS; return mcsv1_UDAF::SUCCESS;
} }

22
utils/regr/regr_sxy.cpp
View File

@ -39,9 +39,9 @@ static Add_regr_sxy_ToUDAFMap addToMap;
struct regr_sxy_data struct regr_sxy_data
{ {
uint64_t cnt; uint64_t cnt;
double sumx; long double sumx;
double sumy; long double sumy;
double sumxy; // sum of x * y long double sumxy; // sum of x * y
}; };
@ -88,8 +88,8 @@ mcsv1_UDAF::ReturnCode regr_sxy::nextValue(mcsv1Context* context, ColumnDatum* v
static_any::any& valIn_y = valsIn[0].columnData; static_any::any& valIn_y = valsIn[0].columnData;
static_any::any& valIn_x = valsIn[1].columnData; static_any::any& valIn_x = valsIn[1].columnData;
struct regr_sxy_data* data = (struct regr_sxy_data*)context->getUserData()->data; struct regr_sxy_data* data = (struct regr_sxy_data*)context->getUserData()->data;
double valx = 0.0; long double valx = 0.0;
double valy = 0.0; long double valy = 0.0;
valx = convertAnyTo<double>(valIn_x); valx = convertAnyTo<double>(valIn_x);
valy = convertAnyTo<double>(valIn_y); valy = convertAnyTo<double>(valIn_y);
@ -145,13 +145,13 @@ mcsv1_UDAF::ReturnCode regr_sxy::evaluate(mcsv1Context* context, static_any::any
double N = data->cnt; double N = data->cnt;
if (N > 0) if (N > 0)
{ {
double sumx = data->sumx; long double sumx = data->sumx;
double sumy = data->sumy; long double sumy = data->sumy;
double sumxy = data->sumxy; long double sumxy = data->sumxy;
double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N;
double regr_sxy = data->cnt * covar_pop; long double regr_sxy = data->cnt * covar_pop;
valOut = regr_sxy; valOut = static_cast<double>(regr_sxy);
} }
return mcsv1_UDAF::SUCCESS; return mcsv1_UDAF::SUCCESS;
} }

14
utils/regr/regr_syy.cpp
View File

@ -39,8 +39,8 @@ static Add_regr_syy_ToUDAFMap addToMap;
struct regr_syy_data struct regr_syy_data
{ {
uint64_t cnt; uint64_t cnt;
double sumy; long double sumy;
double sumy2; // sum of (y squared) long double sumy2; // sum of (y squared)
}; };
@ -125,14 +125,14 @@ mcsv1_UDAF::ReturnCode regr_syy::subEvaluate(mcsv1Context* context, const UserDa
mcsv1_UDAF::ReturnCode regr_syy::evaluate(mcsv1Context* context, static_any::any& valOut) mcsv1_UDAF::ReturnCode regr_syy::evaluate(mcsv1Context* context, static_any::any& valOut)
{ {
struct regr_syy_data* data = (struct regr_syy_data*)context->getUserData()->data; struct regr_syy_data* data = (struct regr_syy_data*)context->getUserData()->data;
double N = data->cnt; long double N = data->cnt;
if (N > 0) if (N > 0)
{ {
double sumy = data->sumy; long double sumy = data->sumy;
double sumy2 = data->sumy2; long double sumy2 = data->sumy2;
double var_popy = (sumy2 - (sumy * sumy / N)) / N; long double var_popy = (sumy2 - (sumy * sumy / N)) / N;
valOut = data->cnt * var_popy; valOut = static_cast<double>(data->cnt * var_popy);
} }
return mcsv1_UDAF::SUCCESS; return mcsv1_UDAF::SUCCESS;
} }