diff --git a/utils/regr/corr.cpp b/utils/regr/corr.cpp index ac8c090bf..a49baf1c7 100644 --- a/utils/regr/corr.cpp +++ b/utils/regr/corr.cpp @@ -160,13 +160,13 @@ mcsv1_UDAF::ReturnCode corr::evaluate(mcsv1Context* context, static_any::any& va long double sumxy = data->sumxy; long double var_popx = (sumx2 - (sumx * sumx / N)) / N; - if (var_popx == 0) + if (var_popx <= 0) // Catch -0 { // When var_popx is 0, NULL is the result. return mcsv1_UDAF::SUCCESS; } long double var_popy = (sumy2 - (sumy * sumy / N)) / N; - if (var_popy == 0) + if (var_popy <= 0) // Catch -0 { // When var_popy is 0, NULL is the result return mcsv1_UDAF::SUCCESS; diff --git a/utils/regr/covar_pop.cpp b/utils/regr/covar_pop.cpp index f3794f89e..0f0ab8c96 100644 --- a/utils/regr/covar_pop.cpp +++ b/utils/regr/covar_pop.cpp @@ -149,7 +149,7 @@ mcsv1_UDAF::ReturnCode covar_pop::evaluate(mcsv1Context* context, static_any::an long double sumy = data->sumy; long double sumxy = data->sumxy; - long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N ; + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; valOut = static_cast(covar_pop); } return mcsv1_UDAF::SUCCESS; diff --git a/utils/regr/regr_intercept.cpp b/utils/regr/regr_intercept.cpp index 5e05eea19..e649fa55c 100644 --- a/utils/regr/regr_intercept.cpp +++ b/utils/regr/regr_intercept.cpp @@ -145,16 +145,18 @@ mcsv1_UDAF::ReturnCode regr_intercept::subEvaluate(mcsv1Context* context, const mcsv1_UDAF::ReturnCode regr_intercept::evaluate(mcsv1Context* context, static_any::any& valOut) { struct regr_intercept_data* data = (struct regr_intercept_data*)context->getUserData()->data; - double N = data->cnt; + long double N = data->cnt; if (N > 1) { long double sumx = data->sumx; long double sumy = data->sumy; long double sumx2 = data->sumx2; long double sumxy = data->sumxy; + // regr_intercept is AVG(y) - slope(y,x)*avg(x) + // We do some algebra and we can get a slightly smaller calculation long double numerator = sumy * sumx2 - sumx * sumxy; - long double var_pop = (N * sumx2) - (sumx * sumx); - if (var_pop != 0) + long double var_pop = (N * sumx2) - (sumx * sumx); // Not realy var_pop, but sort of after some reductions + if (var_pop > 0) { valOut = static_cast(numerator / var_pop); } diff --git a/utils/regr/regr_r2.cpp b/utils/regr/regr_r2.cpp index 2f28afe3e..b69d9abe0 100644 --- a/utils/regr/regr_r2.cpp +++ b/utils/regr/regr_r2.cpp @@ -160,13 +160,13 @@ mcsv1_UDAF::ReturnCode regr_r2::evaluate(mcsv1Context* context, static_any::any& long double sumxy = data->sumxy; long double var_popx = (sumx2 - (sumx * sumx / N)) / N; - if (var_popx == 0) + if (var_popx <= 0) // Catch -0 { // When var_popx is 0, NULL is the result. return mcsv1_UDAF::SUCCESS; } double var_popy = (sumy2 - (sumy * sumy / N)) / N; - if (var_popy == 0) + if (var_popy <= 0) // Catch -0 { // When var_popy is 0, 1 is the result valOut = 1.0; diff --git a/utils/regr/regr_slope.cpp b/utils/regr/regr_slope.cpp index 2a3445da4..b6aeedbd5 100644 --- a/utils/regr/regr_slope.cpp +++ b/utils/regr/regr_slope.cpp @@ -152,12 +152,14 @@ mcsv1_UDAF::ReturnCode regr_slope::evaluate(mcsv1Context* context, static_any::a long double sumy = data->sumy; long double sumx2 = data->sumx2; long double sumxy = data->sumxy; + // These aren't really covar_pop and var_pop. For the purposes of this calculation + // we multiplied everything by N to reduce calc time and variance. + // It all comes out after the final divide long double covar_pop = N * sumxy - sumx * sumy; long double var_pop = N * sumx2 - sumx * sumx; - if (var_pop != 0) + if (var_pop > 0) { - long double slope = covar_pop / var_pop; - valOut = static_cast(slope); + valOut = static_cast(covar_pop / var_pop); } } return mcsv1_UDAF::SUCCESS; diff --git a/utils/regr/regr_sxx.cpp b/utils/regr/regr_sxx.cpp index beacf14d4..07569abd5 100644 --- a/utils/regr/regr_sxx.cpp +++ b/utils/regr/regr_sxx.cpp @@ -128,8 +128,10 @@ mcsv1_UDAF::ReturnCode regr_sxx::evaluate(mcsv1Context* context, static_any::any long double N = data->cnt; if (N > 0) { - long double var_popx = (data->sumx2 - (data->sumx * data->sumx / N)); - valOut = static_cast(var_popx); + long double regr_sxx = (data->sumx2 - (data->sumx * data->sumx / N)); + if (regr_sxx < 0) // catch -0 + regr_sxx = 0; + valOut = static_cast(regr_sxx); } return mcsv1_UDAF::SUCCESS; } diff --git a/utils/regr/regr_syy.cpp b/utils/regr/regr_syy.cpp index 4f9797663..3c9116b74 100644 --- a/utils/regr/regr_syy.cpp +++ b/utils/regr/regr_syy.cpp @@ -129,6 +129,8 @@ mcsv1_UDAF::ReturnCode regr_syy::evaluate(mcsv1Context* context, static_any::any if (N > 0) { long double var_popy = (data->sumy2 - (data->sumy * data->sumy / N)); + if (var_popy < 0) // might be -0 + var_popy = 0; valOut = static_cast(var_popy); } return mcsv1_UDAF::SUCCESS; diff --git a/utils/regr/regrmysql.cpp b/utils/regr/regrmysql.cpp index a27d94a67..b2aa8af6c 100644 --- a/utils/regr/regrmysql.cpp +++ b/utils/regr/regrmysql.cpp @@ -537,7 +537,7 @@ extern "C" long double sumxy = data->sumxy; long double covar_pop = N * sumxy - sumx * sumy; long double var_pop = N * sumx2 - sumx * sumx; - if (var_pop != 0) + if (var_pop > 0) { valOut = static_cast(covar_pop / var_pop); *is_null = 0; @@ -657,7 +657,7 @@ extern "C" long double sumxy = data->sumxy; long double numerator = sumy * sumx2 - sumx * sumxy; long double var_pop = (N * sumx2) - (sumx * sumx); - if (var_pop != 0) + if (var_pop > 0) { valOut = static_cast(numerator / var_pop); *is_null = 0; @@ -781,14 +781,14 @@ extern "C" long double sumy2 = data->sumy2; long double sumxy = data->sumxy; long double var_popx = (sumx2 - (sumx * sumx / N)) / N; - if (var_popx == 0) + if (var_popx <= 0) // Catch -0 { // When var_popx is 0, NULL is the result. *is_null = 1; return 0; } long double var_popy = (sumy2 - (sumy * sumy / N)) / N; - if (var_popy == 0) + if (var_popy <= 0) // Catch -0 { // When var_popy is 0, 1 is the result return 1; @@ -921,14 +921,14 @@ extern "C" long double sumy2 = data->sumy2; long double sumxy = data->sumxy; long double var_popx = (sumx2 - (sumx * sumx / N)) / N; - if (var_popx == 0) + if (var_popx <= 0) // Catch -0 { // When var_popx is 0, NULL is the result. *is_null = 1; return 0; } long double var_popy = (sumy2 - (sumy * sumy / N)) / N; - if (var_popy == 0) + if (var_popy <= 0) // Catch -0 { // When var_popy is 0, 1 is the result return 1; @@ -1044,8 +1044,10 @@ extern "C" { long double sumx = data->sumx; long double sumx2 = data->sumx2; - long double var_popx = (sumx2 - (sumx * sumx / N)) / N; - valOut = static_cast(N * var_popx); + long double sxx = (sumx2 - (sumx * sumx / N)); + if (sxx < 0) // catch -0 + sxx = 0; + valOut = static_cast(sxx); } else { @@ -1151,8 +1153,10 @@ extern "C" { long double sumy = data->sumy; long double sumy2 = data->sumy2; - long double var_popy = (sumy2 - (sumy * sumy / N)) / N; - valOut = static_cast(N * var_popy); + long double syy = (sumy2 - (sumy * sumy / N)); + if (syy < 0) // might be -0 + syy = 0; + valOut = static_cast(syy); } else { @@ -1265,8 +1269,7 @@ extern "C" long double sumx = data->sumx; long double sumy = data->sumy; long double sumxy = data->sumxy; - long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; - long double regr_sxy = N * covar_pop; + long double regr_sxy = (sumxy - ((sumx * sumy) / N)); valOut = static_cast(regr_sxy); } else @@ -1380,7 +1383,7 @@ extern "C" long double sumx = data->sumx; long double sumy = data->sumy; long double sumxy = data->sumxy; - long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N ; + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; valOut = static_cast(covar_pop); } else