MCOL-1822 finishing up use long double for SUM/AVG

dbcon/execplan/predicateoperator.h

@@ -278,11 +278,42 @@ inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, Retu
                 return false;
 
             long double val1 = lop->getLongDoubleVal(row, isNull);
-
             if (isNull)
                 return false;
 
-            return numericCompare(val1, rop->getLongDoubleVal(row, isNull)) && !isNull;
+            long double val2 = rop->getLongDoubleVal(row, isNull);
+            if (isNull)
+                return false;
+
+            // In many case, rounding error will prevent an eq compare to work
+            // In these cases, use the largest scale of the two items.
+            if (fOp == execplan::OP_EQ)
+            {
+                // In case a val is a representation of a very large integer,
+                // we won't want to just multiply by scale, as it may move
+                // significant digits out of scope. So we break them apart
+                // and compare each separately 
+                int64_t scale = max(lop->resultType().scale, rop->resultType().scale);
+                if (scale)
+                {
+                    long double intpart1;
+                    long double fract1 = modfl(val1, &intpart1);
+                    long double intpart2;
+                    long double fract2 = modfl(val2, &intpart2);
+                    if (numericCompare(intpart1, intpart2))
+                    {
+                        double factor = pow(10.0, (double)scale);
+                        fract1 = roundl(fract1 * factor);
+                        fract2 = roundl(fract2 * factor);
+                        return numericCompare(fract1, fract2);
+                    }
+                    else
+                    {
+                        return false;
+                    }
+                }
+            }
+            return numericCompare(val1, val2);
         }
 
         case execplan::CalpontSystemCatalog::DECIMAL:

dbcon/execplan/windowfunctioncolumn.cpp

@@ -378,6 +378,14 @@ void WindowFunctionColumn::adjustResultType()
             boost::iequals(fFunctionName, "NTH_VALUE")) &&
             !fFunctionParms.empty())
         fResultType = fFunctionParms[0]->resultType();
+
+    if (boost::iequals(fFunctionName, "SUM") ||
+        boost::iequals(fFunctionName, "AVG"))
+    {
+        fResultType.colDataType = CalpontSystemCatalog::LONGDOUBLE;
+        fResultType.colWidth = sizeof(long double);
+        fResultType.precision = -1;
+    }
 }
 
 void WindowFunctionColumn::evaluate(Row& row, bool& isNull)

dbcon/mysql/ha_window_function.cpp

@@ -897,7 +897,13 @@ ReturnedColumn* buildWindowFunctionColumn(Item* item, gp_walk_info& gwi, bool& n
         setError(gwi.thd, ER_CHECK_NOT_IMPLEMENTED, gwi.parseErrorText);
         return NULL;
     }
-
+#if 0
+    if (item_sum->sum_func() != Item_sum::UDF_SUM_FUNC &&
+        item_sum->sum_func() != Item_sum::SUM_FUNC && 
+        item_sum->sum_func() != Item_sum::SUM_DISTINCT_FUNC &&
+        item_sum->sum_func() != Item_sum::AVG_FUNC &&
+        item_sum->sum_func() != Item_sum::AVG_DISTINCT_FUNC)
+#endif
     if (item_sum->sum_func() != Item_sum::UDF_SUM_FUNC)
     {
         ac->resultType(colType_MysqlToIDB(item_sum));

utils/windowfunction/wf_sum_avg.cpp

@@ -49,10 +49,10 @@ using namespace joblist;
 #include "wf_sum_avg.h"
 
 
+#if 0
 namespace
 {
 
-
 template<typename T>
 void checkSumLimit(T sum, T val)
 {
@@ -102,14 +102,12 @@ void checkSumLimit<uint64_t>(uint64_t sum, uint64_t val)
 }
 
 
-template<typename T>
+template<>
-T calculateAvg(T sum, uint64_t count, int s)
+long double calculateAvg(long double sum, uint64_t count, int s)
 {
-    T avg = ((long double) sum) / count;
+   return sum / count;
-    return avg;
 }
 
-
 long double avgWithLimit(long double sum, uint64_t count, int scale, long double u, long double l)
 {
     long double factor = pow(10.0, scale);
@@ -149,8 +147,8 @@ uint64_t calculateAvg<uint64_t>(uint64_t sum, uint64_t count, int scale)
     return t;
 }
 
-
 }
+#endif
 
 namespace windowfunction
 {
@@ -159,7 +157,6 @@ template<typename T>
 boost::shared_ptr<WindowFunctionType> WF_sum_avg<T>::makeFunction(int id, const string& name, int ct)
 {
     boost::shared_ptr<WindowFunctionType> func;
-#if 0
     switch (ct)
     {
         case CalpontSystemCatalog::TINYINT:
@@ -203,30 +200,6 @@ boost::shared_ptr<WindowFunctionType> WF_sum_avg<T>::makeFunction(int id, const
             func.reset(new WF_sum_avg<long double>(id, name));
             break;
         }
-#endif
-    switch (ct)
-    {
-        case CalpontSystemCatalog::TINYINT:
-        case CalpontSystemCatalog::SMALLINT:
-        case CalpontSystemCatalog::MEDINT:
-        case CalpontSystemCatalog::INT:
-        case CalpontSystemCatalog::BIGINT:
-        case CalpontSystemCatalog::DECIMAL:
-        case CalpontSystemCatalog::UTINYINT:
-        case CalpontSystemCatalog::USMALLINT:
-        case CalpontSystemCatalog::UMEDINT:
-        case CalpontSystemCatalog::UINT:
-        case CalpontSystemCatalog::UBIGINT:
-        case CalpontSystemCatalog::UDECIMAL:
-        case CalpontSystemCatalog::DOUBLE:
-        case CalpontSystemCatalog::UDOUBLE:
-        case CalpontSystemCatalog::FLOAT:
-        case CalpontSystemCatalog::UFLOAT:
-        case CalpontSystemCatalog::LONGDOUBLE:
-        {
-            func.reset(new WF_sum_avg<long double>(id, name));
-            break;
-        }
         default:
         {
             string errStr = name + "(" + colType2String[ct] + ")";
@@ -277,7 +250,7 @@ void WF_sum_avg<T>::operator()(int64_t b, int64_t e, int64_t c)
             e = c;
 
         uint64_t colIn = fFieldIndex[1];
-        int scale = fRow.getScale(colOut) - fRow.getScale(colIn);
+        double scale = fRow.getScale(colIn);
 
         for (int64_t i = b; i <= e; i++)
         {
@@ -291,11 +264,16 @@ void WF_sum_avg<T>::operator()(int64_t b, int64_t e, int64_t c)
 
             T valIn;
             getValue(colIn, valIn);
-            checkSumLimit(fSum, valIn);
+//            checkSumLimit(fSum, valIn);
 
             if ((!fDistinct) || (fSet.find(valIn) == fSet.end()))
             {
-                fSum += valIn;
+                long double val = valIn;
+                if (scale)
+                {
+                    val /= pow(10.0, scale);
+                }
+                fSum += val;
                 fCount++;
 
                 if (fDistinct)
@@ -304,10 +282,12 @@ void WF_sum_avg<T>::operator()(int64_t b, int64_t e, int64_t c)
         }
 
         if ((fCount > 0) && (fFunctionId == WF__AVG || fFunctionId == WF__AVG_DISTINCT))
-            fAvg = (T) calculateAvg(fSum, fCount, scale);
+        {
+            fAvg = fSum / fCount;
+        }
     }
 
-    T* v = NULL;
+    long double* v = NULL;
 
     if (fCount > 0)
     {

utils/windowfunction/wf_sum_avg.h

@@ -47,8 +47,8 @@ public:
     static boost::shared_ptr<WindowFunctionType> makeFunction(int, const string&, int);
 
 protected:
-    T           fAvg;
+    long double fAvg;
-    T           fSum;
+    long double fSum;
     uint64_t    fCount;
     bool        fDistinct;
     std::set<T> fSet;