database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-01 06:46:55 +03:00
Code Activity

MCOL-1793, fix REGR_SLOPE calculations, change scale to DECIMAL_NOT_SPECIFIED (variable length) for most REGR_*** functions.

Browse Source
This commit is contained in:
David Hall
2018-11-20 16:29:17 -06:00
parent 06d1c9fcbe
commit 9c1bc910a5
11 changed files with 44 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
utils/regr/regr_intercept.cpp
View File

@ -60,8 +60,8 @@ mcsv1_UDAF::ReturnCode regr_intercept::init(mcsv1Context* context,
context->setUserDataSize(sizeof(regr_intercept_data));
context->setResultType(CalpontSystemCatalog::DOUBLE);
context->setColWidth(8);
context->setScale(colTypes[0].scale + 8);
context->setPrecision(19);
context->setScale(DECIMAL_NOT_SPECIFIED);
context->setPrecision(0);
context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS);
return mcsv1_UDAF::SUCCESS;
@ -145,13 +145,13 @@ mcsv1_UDAF::ReturnCode regr_intercept::evaluate(mcsv1Context* context, static_an
double sumy = data->sumy;
double sumx2 = data->sumx2;
double sumxy = data->sumxy;
double slope = 0.0;
double slope = 0;
double variance = (N * sumx2) - (sumx * sumx);
if (variance != 0)
{
slope = ((N * sumxy) - (sumx * sumy)) / variance;
valOut = (sumy - (slope * sumx)) / N;
}
valOut = (sumy - (slope * sumx)) / N;
}
return mcsv1_UDAF::SUCCESS;
}