database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-01 06:46:55 +03:00
Code Activity

MCOL-1793, fix REGR_SLOPE calculations, change scale to DECIMAL_NOT_SPECIFIED (variable length) for most REGR_*** functions.

Browse Source
This commit is contained in:
David Hall
2018-11-20 16:29:17 -06:00
parent 06d1c9fcbe
commit 9c1bc910a5
11 changed files with 44 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
utils/regr/covar_samp.cpp
View File

@ -59,8 +59,8 @@ mcsv1_UDAF::ReturnCode covar_samp::init(mcsv1Context* context,
context->setUserDataSize(sizeof(covar_samp_data));
context->setResultType(CalpontSystemCatalog::DOUBLE);
context->setColWidth(8);
context->setScale(colTypes[0].scale + 8);
context->setPrecision(19);
context->setScale(DECIMAL_NOT_SPECIFIED);
context->setPrecision(0);
context->setRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS);
return mcsv1_UDAF::SUCCESS;
@ -136,7 +136,7 @@ mcsv1_UDAF::ReturnCode covar_samp::evaluate(mcsv1Context* context, static_any::a
{
struct covar_samp_data* data = (struct covar_samp_data*)context->getUserData()->data;
double N = data->cnt;
if (N > 0)
if (N > 1)
{
double sumx = data->sumx;
double sumy = data->sumy;
@ -145,6 +145,11 @@ mcsv1_UDAF::ReturnCode covar_samp::evaluate(mcsv1Context* context, static_any::a
double covar_samp = (sumxy - ((sumx * sumy) / N)) / (N - 1);
valOut = covar_samp;
}
else
if (N == 1)
{
valOut = 0;
}
return mcsv1_UDAF::SUCCESS;
}