From 17bad9eb0b8101d579bf955abb8f27f2b474d07a Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Mon, 1 Jun 2020 08:54:30 +0000 Subject: [PATCH] MCOL-641 Initial support for ORDER BY on wide DECIMALs. --- datatypes/CMakeLists.txt | 2 +- tests/CMakeLists.txt | 14 ++-- .../comparators-tests.cpp | 67 +++++++++++++------ utils/dataconvert/CMakeLists.txt | 6 -- utils/rowgroup/CMakeLists.txt | 6 -- utils/rowgroup/rowgroup.h | 1 - utils/windowfunction/CMakeLists.txt | 6 -- utils/windowfunction/idborderby.cpp | 39 ++++++++++- utils/windowfunction/idborderby.h | 9 +++ 9 files changed, 103 insertions(+), 47 deletions(-) rename {utils/windowfunction => tests}/comparators-tests.cpp (90%) diff --git a/datatypes/CMakeLists.txt b/datatypes/CMakeLists.txt index 660ddb15e..b632a9392 100644 --- a/datatypes/CMakeLists.txt +++ b/datatypes/CMakeLists.txt @@ -5,5 +5,5 @@ set(datatypes_LIB_SRCS add_library(datatypes SHARED ${datatypes_LIB_SRCS}) -install(TARGETS datatypes DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) +install(TARGETS datatypes DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d0b88ffd1..1b10181e8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -3,23 +3,29 @@ include_directories( ${ENGINE_COMMON_INCLUDES} ) if (WITH_ROWGROUP_UT) add_executable(rowgroup_tests rowgroup-tests.cpp) target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) - install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) + install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) endif() if (WITH_ARITHMETICOPERATOR_UT) add_executable(arithmeticoperator_tests arithmeticoperator-tests.cpp) target_link_libraries(arithmeticoperator_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) - install(TARGETS arithmeticoperator_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) + install(TARGETS arithmeticoperator_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) endif() if (WITH_CSDECIMAL_UT) add_executable(mcs_decimal_tests mcs_decimal-tests.cpp) target_link_libraries(mcs_decimal_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) - install(TARGETS mcs_decimal_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) + install(TARGETS mcs_decimal_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) endif() if (WITH_DATACONVERT_UT) add_executable(dataconvert_tests dataconvert-tests.cpp) target_link_libraries(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) - install(TARGETS dataconvert_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) + install(TARGETS dataconvert_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) +endif() + +if (WITH_SORTING_COMPARATORS_UT) + add_executable(comparators_tests comparators-tests.cpp) + target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) + install(TARGETS comparators_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) endif() diff --git a/utils/windowfunction/comparators-tests.cpp b/tests/comparators-tests.cpp similarity index 90% rename from utils/windowfunction/comparators-tests.cpp rename to tests/comparators-tests.cpp index d2d5df316..83627fe56 100644 --- a/utils/windowfunction/comparators-tests.cpp +++ b/tests/comparators-tests.cpp @@ -36,11 +36,13 @@ #include #include #include "bytestream.h" -#include "idborderby.h" +#include "utils/windowfunction/idborderby.h" +#include "mcs_decimal.h" #define DEBUG #define MEMORY_LIMIT 14983602176 +using int128_t = __int128; using namespace std; using namespace joblist; using namespace messageqcpp; @@ -123,27 +125,29 @@ class FilterDriver : public CppUnit::TestFixture CPPUNIT_TEST(INT_TEST); CPPUNIT_TEST(FLOAT_TEST); + CPPUNIT_TEST(WIDEDT_TEST); CPPUNIT_TEST_SUITE_END(); private: // The tests creates an RG with 1 column of the cscDt type // then initialize RGData. After that it adds two numeric values (v1 < v2)and two NULL. - // Then creates comparator structores and run a number of tests. v1 < v2 - void testComparatorWithDT(execplan::CalpontSystemCatalog::ColDataType cscDt, + // Then creates comparator structures and run a number of tests. v1 < v2 + void testComparatorWithDT(execplan::CalpontSystemCatalog::ColDataType cscDt, uint32_t width, - bool generateRandValues) + bool generateRandValues, + uint8_t precision) { std::cout << std::endl << "------------------------------------------------------------" << std::endl; uint32_t oid =3001; - std::vector offsets, roids, tkeys, cscale, cprecision; + std::vector offsets, roids, tkeys, cscale, cprecision; std::vector types; offsets.push_back(2); offsets.push_back(2+width); roids.push_back(oid); tkeys.push_back(1); types.push_back(cscDt); cscale.push_back(0); - cprecision.push_back(20); + cprecision.push_back(precision); rowgroup::RowGroup inRG(1, //column count offsets, //oldOffset roids, // column oids @@ -295,7 +299,7 @@ private: r.setUintField<4>(joblist::INTNULL, 0); break; } - default: + case 8 : { r.setUintField<8>(42, 0); r.nextRow(rowSize); @@ -304,6 +308,21 @@ private: r.setUintField<8>(joblist::BIGINTNULL, 0); break; } + case 16 : + { + uint128_t dec = 42; + r.setBinaryField(&dec, 0); + r.nextRow(rowSize); + dec++; + r.setBinaryField(&dec, 0); + r.nextRow(rowSize); + r.setBinaryField( + const_cast(&datatypes::Decimal128Null), + 0); + break; + } + default : + std::cout << " This is the end. My only friend the end... " << std::endl; } break; } @@ -387,27 +406,33 @@ private: { //bool generateValues = true; bool fixedValues = false; - testComparatorWithDT(execplan::CalpontSystemCatalog::UTINYINT, 1, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::USMALLINT, 2, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::UMEDINT, 4, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::UBIGINT, 8, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::DATETIME, 8, fixedValues); + testComparatorWithDT(execplan::CalpontSystemCatalog::UTINYINT, 1, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::USMALLINT, 2, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::UMEDINT, 4, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::UBIGINT, 8, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::DATETIME, 8, fixedValues, 20); - testComparatorWithDT(execplan::CalpontSystemCatalog::TINYINT, 1, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::SMALLINT, 2, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::MEDINT, 4, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::DATE, 4, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::BIGINT, 8, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::DECIMAL, 8, fixedValues); + testComparatorWithDT(execplan::CalpontSystemCatalog::TINYINT, 1, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::SMALLINT, 2, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::MEDINT, 4, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::DATE, 4, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::BIGINT, 8, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::DECIMAL, 8, fixedValues, 20); - testComparatorWithDT(execplan::CalpontSystemCatalog::FLOAT, 4, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::DOUBLE, 8, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::LONGDOUBLE, 8, fixedValues); + testComparatorWithDT(execplan::CalpontSystemCatalog::FLOAT, 4, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::DOUBLE, 8, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::LONGDOUBLE, 8, fixedValues, 20); } void FLOAT_TEST() { } + + void WIDEDT_TEST() + { + bool fixedValues = false; + testComparatorWithDT(execplan::CalpontSystemCatalog::DECIMAL, 16, fixedValues, 38); + } }; CPPUNIT_TEST_SUITE_REGISTRATION(FilterDriver); diff --git a/utils/dataconvert/CMakeLists.txt b/utils/dataconvert/CMakeLists.txt index 9a25dfb1b..3e37b3c0e 100644 --- a/utils/dataconvert/CMakeLists.txt +++ b/utils/dataconvert/CMakeLists.txt @@ -11,9 +11,3 @@ add_library(dataconvert SHARED ${dataconvert_LIB_SRCS}) add_dependencies(dataconvert loggingcpp) install(TARGETS dataconvert DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) - -if (WITH_DATACONVERT_UT) - add_executable(dataconvert_tests dataconvert-tests.cpp) - target_link_libraries(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) - install(TARGETS dataconvert_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) -endif() diff --git a/utils/rowgroup/CMakeLists.txt b/utils/rowgroup/CMakeLists.txt index 3fcde10df..876d620cf 100644 --- a/utils/rowgroup/CMakeLists.txt +++ b/utils/rowgroup/CMakeLists.txt @@ -15,9 +15,3 @@ add_dependencies(rowgroup loggingcpp) target_link_libraries(rowgroup ${NETSNMP_LIBRARIES} funcexp) install(TARGETS rowgroup DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) - -if (WITH_ROWGROUP_UT) - add_executable(rowgroup_tests rowgroup-tests.cpp) - target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) - install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) -endif() diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index da01d0941..a8f84fbc8 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -66,7 +66,6 @@ typedef const struct charset_info_st CHARSET_INFO; // Workaround for my_global.h #define of isnan(X) causing a std::std namespace using int128_t = __int128; -using uint128_t = unsigned __int128; namespace rowgroup { diff --git a/utils/windowfunction/CMakeLists.txt b/utils/windowfunction/CMakeLists.txt index 636f72735..218a43803 100755 --- a/utils/windowfunction/CMakeLists.txt +++ b/utils/windowfunction/CMakeLists.txt @@ -29,9 +29,3 @@ add_library(windowfunction SHARED ${windowfunction_LIB_SRCS}) add_dependencies(windowfunction loggingcpp) install(TARGETS windowfunction DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) - -if (WITH_SORTING_COMPARATORS_UT) - add_executable(comparators_tests comparators-tests.cpp) - target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) - install(TARGETS comparators_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) -endif() diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index f92e1a5c0..58b01ee38 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -47,6 +47,9 @@ using namespace rowgroup; #include "idborderby.h" #include "joblisttypes.h" +#include "mcs_decimal.h" + +using int128_t = __int128; #include "collation.h" @@ -166,6 +169,35 @@ int BigIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) return ret; } +int WideDecimalCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) +{ + l->row1().setData(r1); + l->row2().setData(r2); + + int ret = 0; + int128_t v1 = *(l->row1().getBinaryField_offset(keyColumnOffset)); + int128_t v2 = *(l->row2().getBinaryField_offset(keyColumnOffset)); + bool v1IsNull = v1 == datatypes::Decimal128Null; + bool v2IsNull = v2 == datatypes::Decimal128Null; + + if (v1IsNull || v2IsNull) + { + if (!v1IsNull && v2IsNull) + ret = fSpec.fNf; + else if (v1IsNull && !v2IsNull) + ret = -fSpec.fNf; + } + else + { + if (v1 > v2) + ret = fSpec.fAsc; + else if (v1 < v2) + ret = -fSpec.fAsc; + } + + return ret; +} + int UTinyIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) { l->row1().setData(r1); @@ -537,6 +569,7 @@ void CompareRule::revertRules() void CompareRule::compileRules(const std::vector& spec, const rowgroup::RowGroup& rg) { const vector& types = rg.getColTypes(); + const auto& offsets = rg.getOffsets(); for (vector::const_iterator i = spec.begin(); i != spec.end(); i++) { @@ -574,14 +607,16 @@ void CompareRule::compileRules(const std::vector& spec, const rowgr uint32_t len = rg.getColumnWidth(i->fIndex); switch (len) { + case datatypes::MAXDECIMALWIDTH: + c = new WideDecimalCompare(*i, offsets[i->fIndex]); break; + case datatypes::MAXLEGACYWIDTH: + c = new BigIntCompare(*i); case 1 : c = new TinyIntCompare(*i); break; case 2 : c = new SmallIntCompare(*i); break; case 4 : c = new IntCompare(*i); break; - default: - c = new BigIntCompare(*i); } fCompares.push_back(c); diff --git a/utils/windowfunction/idborderby.h b/utils/windowfunction/idborderby.h index 98dce4b59..30f49c1de 100644 --- a/utils/windowfunction/idborderby.h +++ b/utils/windowfunction/idborderby.h @@ -143,6 +143,15 @@ public: int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); }; +class WideDecimalCompare : public Compare +{ + int keyColumnOffset; +public: + WideDecimalCompare(const IdbSortSpec& spec, int offset) : Compare(spec), keyColumnOffset(offset) { } + + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); +}; + // End of comparators for signed types // Comparators for unsigned types