MCOL-641 Initial support for ORDER BY on wide DECIMALs.

datatypes/CMakeLists.txt

@@ -5,5 +5,5 @@ set(datatypes_LIB_SRCS
 
 add_library(datatypes SHARED ${datatypes_LIB_SRCS})
 
-install(TARGETS datatypes DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs)
+install(TARGETS datatypes DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine)
 

tests/CMakeLists.txt

@@ -3,23 +3,29 @@ include_directories( ${ENGINE_COMMON_INCLUDES} )
 if (WITH_ROWGROUP_UT)
     add_executable(rowgroup_tests rowgroup-tests.cpp)
     target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) 
-    install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform)
+    install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)
 endif()
 
 if (WITH_ARITHMETICOPERATOR_UT)
     add_executable(arithmeticoperator_tests arithmeticoperator-tests.cpp)
     target_link_libraries(arithmeticoperator_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) 
-    install(TARGETS arithmeticoperator_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform)
+    install(TARGETS arithmeticoperator_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)
 endif()
 
 if (WITH_CSDECIMAL_UT)
     add_executable(mcs_decimal_tests mcs_decimal-tests.cpp)
     target_link_libraries(mcs_decimal_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) 
-    install(TARGETS mcs_decimal_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform)
+    install(TARGETS mcs_decimal_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)
 endif()
 
 if (WITH_DATACONVERT_UT)
     add_executable(dataconvert_tests dataconvert-tests.cpp)
     target_link_libraries(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) 
-    install(TARGETS dataconvert_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform)
+    install(TARGETS dataconvert_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)
+endif()
+
+if (WITH_SORTING_COMPARATORS_UT)
+    add_executable(comparators_tests comparators-tests.cpp)
+    target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
+    install(TARGETS comparators_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)
 endif()

tests/comparators-tests.cpp

@@ -36,11 +36,13 @@
 #include <boost/any.hpp>
 #include <boost/function.hpp>
 #include "bytestream.h"
-#include "idborderby.h"
+#include "utils/windowfunction/idborderby.h"
+#include "mcs_decimal.h"
 
 #define DEBUG
 #define MEMORY_LIMIT 14983602176
 
+using int128_t = __int128;
 using namespace std;
 using namespace joblist;
 using namespace messageqcpp;
@@ -123,16 +125,18 @@ class FilterDriver : public CppUnit::TestFixture
 
     CPPUNIT_TEST(INT_TEST);
     CPPUNIT_TEST(FLOAT_TEST);
+    CPPUNIT_TEST(WIDEDT_TEST);
 
     CPPUNIT_TEST_SUITE_END();
 
 private:
     // The tests creates an RG with 1 column of the cscDt type 
     // then initialize RGData. After that it adds two numeric values (v1 < v2)and two NULL.
-    // Then creates comparator structores and run a number of tests. v1 < v2
+    // Then creates comparator structures and run a number of tests. v1 < v2
     void testComparatorWithDT(execplan::CalpontSystemCatalog::ColDataType cscDt,
     uint32_t width,
-    bool generateRandValues)
+    bool generateRandValues,
+    uint8_t precision)
     {
         std::cout << std::endl << "------------------------------------------------------------" << std::endl;
         uint32_t oid =3001;
@@ -143,7 +147,7 @@ private:
         tkeys.push_back(1);
         types.push_back(cscDt);
         cscale.push_back(0);
-        cprecision.push_back(20);
+        cprecision.push_back(precision);
         rowgroup::RowGroup inRG(1, //column count
             offsets, //oldOffset
             roids, // column oids
@@ -295,7 +299,7 @@ private:
                         r.setUintField<4>(joblist::INTNULL, 0);
                         break;
                     }
-                    default:
+                    case 8 :
                     {
                         r.setUintField<8>(42, 0);
                         r.nextRow(rowSize);
@@ -304,6 +308,21 @@ private:
                         r.setUintField<8>(joblist::BIGINTNULL, 0);
                         break;
                     }
+                    case 16 :
+                    {
+                        uint128_t dec = 42;
+                        r.setBinaryField(&dec, 0);
+                        r.nextRow(rowSize);
+                        dec++;
+                        r.setBinaryField(&dec, 0);
+                        r.nextRow(rowSize);
+                        r.setBinaryField(
+                            const_cast<int128_t*>(&datatypes::Decimal128Null),
+                            0);
+                        break;
+                    }
+                    default :
+                        std::cout << " This is the end. My only friend the end... " << std::endl;
                 }
                 break;
             }
@@ -387,27 +406,33 @@ private:
     {
         //bool generateValues = true;
         bool fixedValues = false;
-        testComparatorWithDT(execplan::CalpontSystemCatalog::UTINYINT, 1, fixedValues);
+        testComparatorWithDT(execplan::CalpontSystemCatalog::UTINYINT, 1, fixedValues, 20);
-        testComparatorWithDT(execplan::CalpontSystemCatalog::USMALLINT, 2, fixedValues);
+        testComparatorWithDT(execplan::CalpontSystemCatalog::USMALLINT, 2, fixedValues, 20);
-        testComparatorWithDT(execplan::CalpontSystemCatalog::UMEDINT, 4, fixedValues);
+        testComparatorWithDT(execplan::CalpontSystemCatalog::UMEDINT, 4, fixedValues, 20);
-        testComparatorWithDT(execplan::CalpontSystemCatalog::UBIGINT, 8, fixedValues);
+        testComparatorWithDT(execplan::CalpontSystemCatalog::UBIGINT, 8, fixedValues, 20);
-        testComparatorWithDT(execplan::CalpontSystemCatalog::DATETIME, 8, fixedValues);
+        testComparatorWithDT(execplan::CalpontSystemCatalog::DATETIME, 8, fixedValues, 20);
 
-        testComparatorWithDT(execplan::CalpontSystemCatalog::TINYINT, 1, fixedValues);
+        testComparatorWithDT(execplan::CalpontSystemCatalog::TINYINT, 1, fixedValues, 20);
-        testComparatorWithDT(execplan::CalpontSystemCatalog::SMALLINT, 2, fixedValues);
+        testComparatorWithDT(execplan::CalpontSystemCatalog::SMALLINT, 2, fixedValues, 20);
-        testComparatorWithDT(execplan::CalpontSystemCatalog::MEDINT, 4, fixedValues);
+        testComparatorWithDT(execplan::CalpontSystemCatalog::MEDINT, 4, fixedValues, 20);
-        testComparatorWithDT(execplan::CalpontSystemCatalog::DATE, 4, fixedValues);
+        testComparatorWithDT(execplan::CalpontSystemCatalog::DATE, 4, fixedValues, 20);
-        testComparatorWithDT(execplan::CalpontSystemCatalog::BIGINT, 8, fixedValues);
+        testComparatorWithDT(execplan::CalpontSystemCatalog::BIGINT, 8, fixedValues, 20);
-        testComparatorWithDT(execplan::CalpontSystemCatalog::DECIMAL, 8, fixedValues);
+        testComparatorWithDT(execplan::CalpontSystemCatalog::DECIMAL, 8, fixedValues, 20);
 
-        testComparatorWithDT(execplan::CalpontSystemCatalog::FLOAT, 4, fixedValues);
+        testComparatorWithDT(execplan::CalpontSystemCatalog::FLOAT, 4, fixedValues, 20);
-        testComparatorWithDT(execplan::CalpontSystemCatalog::DOUBLE, 8, fixedValues);
+        testComparatorWithDT(execplan::CalpontSystemCatalog::DOUBLE, 8, fixedValues, 20);
-        testComparatorWithDT(execplan::CalpontSystemCatalog::LONGDOUBLE, 8, fixedValues);
+        testComparatorWithDT(execplan::CalpontSystemCatalog::LONGDOUBLE, 8, fixedValues, 20);
     }
 
     void FLOAT_TEST()
     {
     }
+
+    void WIDEDT_TEST()
+    {
+        bool fixedValues = false;
+        testComparatorWithDT(execplan::CalpontSystemCatalog::DECIMAL, 16, fixedValues, 38);
+    }
 };
 
 CPPUNIT_TEST_SUITE_REGISTRATION(FilterDriver);

utils/dataconvert/CMakeLists.txt

@@ -11,9 +11,3 @@ add_library(dataconvert SHARED ${dataconvert_LIB_SRCS})
 add_dependencies(dataconvert loggingcpp)
 
 install(TARGETS dataconvert DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine)
-
-if (WITH_DATACONVERT_UT)
-    add_executable(dataconvert_tests dataconvert-tests.cpp)
-    target_link_libraries(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) 
-    install(TARGETS dataconvert_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)
-endif()

utils/rowgroup/CMakeLists.txt

@@ -15,9 +15,3 @@ add_dependencies(rowgroup loggingcpp)
 target_link_libraries(rowgroup ${NETSNMP_LIBRARIES} funcexp)
 
 install(TARGETS rowgroup DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine)
-
-if (WITH_ROWGROUP_UT)
-    add_executable(rowgroup_tests rowgroup-tests.cpp)
-    target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) 
-    install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)
-endif()

utils/rowgroup/rowgroup.h

@@ -66,7 +66,6 @@ typedef const struct charset_info_st CHARSET_INFO;
 // Workaround for my_global.h #define of isnan(X) causing a std::std namespace
 
 using int128_t = __int128;
-using uint128_t = unsigned __int128;
 
 namespace rowgroup
 {

utils/windowfunction/CMakeLists.txt

@@ -29,9 +29,3 @@ add_library(windowfunction SHARED ${windowfunction_LIB_SRCS})
 add_dependencies(windowfunction loggingcpp)
 
 install(TARGETS windowfunction DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine)
-
-if (WITH_SORTING_COMPARATORS_UT)
-    add_executable(comparators_tests comparators-tests.cpp)
-    target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
-    install(TARGETS comparators_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine)
-endif()

utils/windowfunction/idborderby.cpp

@@ -47,6 +47,9 @@ using namespace rowgroup;
 #include "idborderby.h"
 
 #include "joblisttypes.h"
+#include "mcs_decimal.h"
+
+using int128_t = __int128;
 
 #include "collation.h"
 
@@ -166,6 +169,35 @@ int BigIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
     return ret;
 }
 
+int WideDecimalCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
+{
+    l->row1().setData(r1);
+    l->row2().setData(r2);
+
+    int ret = 0;
+    int128_t v1 = *(l->row1().getBinaryField_offset<int128_t>(keyColumnOffset));
+    int128_t v2 = *(l->row2().getBinaryField_offset<int128_t>(keyColumnOffset));
+    bool v1IsNull = v1 == datatypes::Decimal128Null;
+    bool v2IsNull = v2 == datatypes::Decimal128Null;
+
+    if (v1IsNull || v2IsNull)
+    {
+        if (!v1IsNull && v2IsNull)
+            ret = fSpec.fNf;
+        else if (v1IsNull  && !v2IsNull)
+            ret = -fSpec.fNf;
+    }
+    else
+    {
+        if (v1 > v2)
+            ret = fSpec.fAsc;
+        else if (v1 < v2)
+            ret = -fSpec.fAsc;
+    }
+
+    return ret;
+}
+
 int UTinyIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
 {
     l->row1().setData(r1);
@@ -537,6 +569,7 @@ void CompareRule::revertRules()
 void CompareRule::compileRules(const std::vector<IdbSortSpec>& spec, const rowgroup::RowGroup& rg)
 {
     const vector<CalpontSystemCatalog::ColDataType>& types = rg.getColTypes();
+    const auto& offsets = rg.getOffsets();
 
     for (vector<IdbSortSpec>::const_iterator i = spec.begin(); i != spec.end(); i++)
     {
@@ -574,14 +607,16 @@ void CompareRule::compileRules(const std::vector<IdbSortSpec>& spec, const rowgr
                 uint32_t len = rg.getColumnWidth(i->fIndex);
                 switch (len)
                 {
+                    case datatypes::MAXDECIMALWIDTH:
+                        c = new WideDecimalCompare(*i, offsets[i->fIndex]); break;
+                    case datatypes::MAXLEGACYWIDTH:
+                        c = new BigIntCompare(*i);
                     case 1 :
                         c = new TinyIntCompare(*i); break;
                     case 2 :
                         c = new SmallIntCompare(*i); break;
                     case 4 :
                         c = new IntCompare(*i); break;
-                    default:
-                        c = new BigIntCompare(*i);
                 }
 
                 fCompares.push_back(c);

utils/windowfunction/idborderby.h

@@ -143,6 +143,15 @@ public:
     int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer);
 };
 
+class WideDecimalCompare : public Compare
+{
+    int keyColumnOffset;
+public:
+    WideDecimalCompare(const IdbSortSpec& spec, int offset) : Compare(spec), keyColumnOffset(offset) { }
+
+    int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer);
+};
+
 // End of comparators for signed types
 // Comparators for unsigned types