From 238386bf63db5a2f90af71e98d9e051c9a5e059c Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 3 Mar 2020 15:51:55 +0000 Subject: [PATCH] MCOL-641 Replaced IDB_Decima.__v union with int128_t attribute. Moved all tests into ./test Introduced ./datatypes directory --- CMakeLists.txt | 6 +- datatypes/CMakeLists.txt | 9 + datatypes/decimal.cpp | 20 ++ datatypes/decimal.h | 32 +++ dbcon/execplan/arithmeticoperator.cpp | 64 ----- dbcon/execplan/arithmeticoperator.h | 8 +- dbcon/execplan/simplecolumn.cpp | 12 +- dbcon/execplan/treenode.h | 11 +- dbcon/mysql/ha_mcs_impl.cpp | 1 + tests/CMakeLists.txt | 19 ++ tests/arithmeticoperator-tests.cpp | 27 ++ .../dataconvert-tests.cpp | 25 ++ tests/rowgroup-tests.cpp | 243 ++++++++++++++++++ utils/dataconvert/CMakeLists.txt | 2 +- utils/dataconvert/dataconvert.cpp | 7 +- utils/funcexp/funcexp.cpp | 16 +- utils/rowgroup/CMakeLists.txt | 2 +- 17 files changed, 398 insertions(+), 106 deletions(-) create mode 100644 datatypes/CMakeLists.txt create mode 100644 datatypes/decimal.cpp create mode 100644 datatypes/decimal.h create mode 100644 tests/CMakeLists.txt create mode 100644 tests/arithmeticoperator-tests.cpp rename {utils/dataconvert => tests}/dataconvert-tests.cpp (96%) create mode 100644 tests/rowgroup-tests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 7a084400d..d4fe952ad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -205,6 +205,7 @@ SET (ENGINE_OAM_LIBS oamcpp alarmmanager) SET (ENGINE_BRM_LIBS brm idbdatafile cacheutils rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS}) SET (ENGINE_EXEC_LIBS joblist execplan windowfunction joiner rowgroup funcexp udfsdk regr dataconvert common compress querystats querytele thrift threadpool ${ENGINE_BRM_LIBS}) SET (ENGINE_WRITE_LIBS ddlpackageproc ddlpackage dmlpackageproc dmlpackage writeengine writeengineclient idbdatafile cacheutils ${ENGINE_EXEC_LIBS}) +SET (ENGINE_DATATYPES_LIBS datatypes) SET (ENGINE_COMMON_LDFLAGS "") @@ -284,6 +285,7 @@ SET (ENGINE_UTILS_DDLCLEANUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/ddlclea SET (ENGINE_UTILS_QUERYSTATS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/querystats") SET (ENGINE_UTILS_LIBMYSQL_CL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/libmysql_client") SET (ENGINE_WE_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/xml") +SET (ENGINE_DATATYPES_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/datatypes") SET (ENGINE_SERVER_SQL_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/sql") SET (ENGINE_SERVER_INCLUDE_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/include") IF (PCRE_INCLUDES) @@ -305,7 +307,7 @@ ELSE () SET (ENGINE_READLINE_LIBRARY "readline") ENDIF () -SET (ENGINE_COMMON_INCLUDES ${ENGINE_DEFAULT_INCLUDES} ${Boost_INCLUDE_DIR} ${LIBXML2_INCLUDE_DIR} ${ENGINE_UTILS_MESSAGEQCPP_INCLUDE} ${ENGINE_WE_SHARED_INCLUDE} ${ENGINE_UTILS_IDBDATAFILE_INCLUDE} ${ENGINE_UTILS_LOGGINGCPP_INCLUDE} ${ENGINE_UTILS_CONFIGCPP_INCLUDE} ${ENGINE_UTILS_COMPRESS_INCLUDE} ${ENGINE_VERSIONING_BRM_INCLUDE} ${ENGINE_UTILS_ROWGROUP_INCLUDE} ${ENGINE_UTILS_COMMON_INCLUDE} ${ENGINE_UTILS_DATACONVERT_INCLUDE} ${ENGINE_UTILS_RWLOCK_INCLUDE} ${ENGINE_UTILS_FUNCEXP_INCLUDE} ${ENGINE_OAMAPPS_ALARMMANAGER_INCLUDE} ${ENGINE_UTILS_INCLUDE} ${ENGINE_OAM_OAMCPP_INCLUDE} ${ENGINE_DBCON_DDLPKGPROC_INCLUDE} ${ENGINE_DBCON_DDLPKG_INCLUDE} ${ENGINE_DBCON_EXECPLAN_INCLUDE} ${ENGINE_UTILS_STARTUP_INCLUDE} ${ENGINE_DBCON_JOBLIST_INCLUDE} ${ENGINE_WE_WRAPPER_INCLUDE} ${ENGINE_WE_SERVER_INCLUDE} ${ENGINE_DBCON_DMLPKG_INCLUDE} ${ENGINE_WE_CLIENT_INCLUDE} ${ENGINE_DBCON_DMLPKGPROC_INCLUDE} ${ENGINE_UTILS_CACHEUTILS_INCLUDE} ${ENGINE_UTILS_MYSQLCL_INCLUDE} ${ENGINE_UTILS_QUERYTELE_INCLUDE} ${ENGINE_UTILS_THRIFT_INCLUDE} ${ENGINE_UTILS_JOINER_INCLUDE} ${ENGINE_UTILS_THREADPOOL_INCLUDE} ${ENGINE_UTILS_BATCHLDR_INCLUDE} ${ENGINE_UTILS_DDLCLEANUP_INCLUDE} ${ENGINE_UTILS_QUERYSTATS_INCLUDE} ${ENGINE_WE_CONFIGCPP_INCLUDE} ${ENGINE_SERVER_SQL_INCLUDE} ${ENGINE_SERVER_INCLUDE_INCLUDE} ${ENGINE_SERVER_PCRE_INCLUDE} ${ENGINE_SERVER_WSREP_API_INCLUDE} ${ENGINE_SERVER_WSREP_INCLUDE} ${ENGINE_UTILS_UDFSDK_INCLUDE} ${ENGINE_UTILS_LIBMYSQL_CL_INCLUDE}) +SET (ENGINE_COMMON_INCLUDES ${ENGINE_DEFAULT_INCLUDES} ${Boost_INCLUDE_DIR} ${LIBXML2_INCLUDE_DIR} ${ENGINE_UTILS_MESSAGEQCPP_INCLUDE} ${ENGINE_WE_SHARED_INCLUDE} ${ENGINE_UTILS_IDBDATAFILE_INCLUDE} ${ENGINE_UTILS_LOGGINGCPP_INCLUDE} ${ENGINE_UTILS_CONFIGCPP_INCLUDE} ${ENGINE_UTILS_COMPRESS_INCLUDE} ${ENGINE_VERSIONING_BRM_INCLUDE} ${ENGINE_UTILS_ROWGROUP_INCLUDE} ${ENGINE_UTILS_COMMON_INCLUDE} ${ENGINE_UTILS_DATACONVERT_INCLUDE} ${ENGINE_UTILS_RWLOCK_INCLUDE} ${ENGINE_UTILS_FUNCEXP_INCLUDE} ${ENGINE_OAMAPPS_ALARMMANAGER_INCLUDE} ${ENGINE_UTILS_INCLUDE} ${ENGINE_OAM_OAMCPP_INCLUDE} ${ENGINE_DBCON_DDLPKGPROC_INCLUDE} ${ENGINE_DBCON_DDLPKG_INCLUDE} ${ENGINE_DBCON_EXECPLAN_INCLUDE} ${ENGINE_UTILS_STARTUP_INCLUDE} ${ENGINE_DBCON_JOBLIST_INCLUDE} ${ENGINE_WE_WRAPPER_INCLUDE} ${ENGINE_WE_SERVER_INCLUDE} ${ENGINE_DBCON_DMLPKG_INCLUDE} ${ENGINE_WE_CLIENT_INCLUDE} ${ENGINE_DBCON_DMLPKGPROC_INCLUDE} ${ENGINE_UTILS_CACHEUTILS_INCLUDE} ${ENGINE_UTILS_MYSQLCL_INCLUDE} ${ENGINE_UTILS_QUERYTELE_INCLUDE} ${ENGINE_UTILS_THRIFT_INCLUDE} ${ENGINE_UTILS_JOINER_INCLUDE} ${ENGINE_UTILS_THREADPOOL_INCLUDE} ${ENGINE_UTILS_BATCHLDR_INCLUDE} ${ENGINE_UTILS_DDLCLEANUP_INCLUDE} ${ENGINE_UTILS_QUERYSTATS_INCLUDE} ${ENGINE_WE_CONFIGCPP_INCLUDE} ${ENGINE_SERVER_SQL_INCLUDE} ${ENGINE_SERVER_INCLUDE_INCLUDE} ${ENGINE_SERVER_PCRE_INCLUDE} ${ENGINE_SERVER_WSREP_API_INCLUDE} ${ENGINE_SERVER_WSREP_INCLUDE} ${ENGINE_UTILS_UDFSDK_INCLUDE} ${ENGINE_UTILS_LIBMYSQL_CL_INCLUDE} ${ENGINE_DATATYPES_INCLUDE}) ADD_SUBDIRECTORY(dbcon/mysql) IF(NOT TARGET columnstore) @@ -336,6 +338,8 @@ ADD_SUBDIRECTORY(writeengine/server) ADD_SUBDIRECTORY(writeengine/bulk) ADD_SUBDIRECTORY(writeengine/splitter) ADD_SUBDIRECTORY(storage-manager) +ADD_SUBDIRECTORY(datatypes) +ADD_SUBDIRECTORY(tests) # WriteEngine component tests IF( WITH_SHARED_COMP_TESTS ) diff --git a/datatypes/CMakeLists.txt b/datatypes/CMakeLists.txt new file mode 100644 index 000000000..433cefa5c --- /dev/null +++ b/datatypes/CMakeLists.txt @@ -0,0 +1,9 @@ + +include_directories( ${ENGINE_COMMON_INCLUDES} ) +set(datatypes_LIB_SRCS + decimal.cpp) + +add_library(datatypes SHARED ${datatypes_LIB_SRCS}) + +install(TARGETS datatypes DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) + diff --git a/datatypes/decimal.cpp b/datatypes/decimal.cpp new file mode 100644 index 000000000..e5c44d395 --- /dev/null +++ b/datatypes/decimal.cpp @@ -0,0 +1,20 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include "decimal.h" + +const datatypes::Decimal someDecimal; diff --git a/datatypes/decimal.h b/datatypes/decimal.h new file mode 100644 index 000000000..104c4db5c --- /dev/null +++ b/datatypes/decimal.h @@ -0,0 +1,32 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#ifndef H_DECIMALDATATYPE +#define H_DECIMALDATATYPE + +namespace datatypes +{ + +class Decimal +{ + public: + Decimal() { }; + ~Decimal() { }; +}; + +} //end of namespace +#endif diff --git a/dbcon/execplan/arithmeticoperator.cpp b/dbcon/execplan/arithmeticoperator.cpp index 230786800..d968ceb5b 100644 --- a/dbcon/execplan/arithmeticoperator.cpp +++ b/dbcon/execplan/arithmeticoperator.cpp @@ -116,70 +116,6 @@ bool ArithmeticOperator::operator!=(const TreeNode* t) const return (!(*this == t)); } -#if 0 -void ArithmeticOperator::operationType(const Type& l, const Type& r) -{ - if (l.colDataType == execplan::CalpontSystemCatalog::DECIMAL) - { - switch (r.colDataType) - { - case execplan::CalpontSystemCatalog::DECIMAL: - { - // should follow the result type that MySQL gives - fOperationType = fResultType; - break; - } - - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::BIGINT: - fOperationType.colDataType = execplan::CalpontSystemCatalog::DECIMAL; - fOperationType.scale = l.scale; - break; - - default: - fOperationType.colDataType = execplan::CalpontSystemCatalog::DOUBLE; - } - } - else if (r.colDataType == execplan::CalpontSystemCatalog::DECIMAL) - { - switch (l.colDataType) - { - case execplan::CalpontSystemCatalog::DECIMAL: - { - // should following the result type that MySQL gives based on the following logic? - // @NOTE is this trustable? - fOperationType = fResultType; - break; - } - - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::BIGINT: - fOperationType.colDataType = execplan::CalpontSystemCatalog::DECIMAL; - fOperationType.scale = r.scale; - break; - - default: - fOperationType.colDataType = execplan::CalpontSystemCatalog::DOUBLE; - } - } - else if ((l.colDataType == execplan::CalpontSystemCatalog::INT || - l.colDataType == execplan::CalpontSystemCatalog::MEDINT || - l.colDataType == execplan::CalpontSystemCatalog::TINYINT || - l.colDataType == execplan::CalpontSystemCatalog::BIGINT) && - (r.colDataType == execplan::CalpontSystemCatalog::INT || - r.colDataType == execplan::CalpontSystemCatalog::MEDINT || - r.colDataType == execplan::CalpontSystemCatalog::TINYINT || - r.colDataType == execplan::CalpontSystemCatalog::BIGINT)) - fOperationType.colDataType = execplan::CalpontSystemCatalog::BIGINT; - else - fOperationType.colDataType = execplan::CalpontSystemCatalog::DOUBLE; -} -#endif - void ArithmeticOperator::adjustResultType(const CalpontSystemCatalog::ColType& m) { if (m.colDataType != CalpontSystemCatalog::DECIMAL) diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index 4660387fe..479b9e645 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -288,13 +288,9 @@ inline void ArithmeticOperator::execute(IDB_Decimal& result, IDB_Decimal op1, ID switch (fOp) { case OP_ADD: - if (resultCscType.precision > 18) + if (resultCscType.colWidth == 16) { - // WIP make this a separate function w and w/o overflow check - if (resultCscType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) - result.__v.__s128 = op1.__v.__s128 + op2.__v.__s128; - else - result.__v.__u128 = op1.__v.__u128 + op2.__v.__u128; + result.s128Value = op1.s128Value + op2.s128Value; break; } diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index eeb287e53..34e02a4df 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -635,16 +635,8 @@ void SimpleColumn::evaluate(Row& row, bool& isNull) { case 16: { - if (fResultType.colDataType == CalpontSystemCatalog::UDECIMAL) - { - fResult.decimalVal.__v.__u128 = - *row.getBinaryField_offset(fInputOffset); - } - else - { - fResult.decimalVal.__v.__s128 = - *row.getBinaryField_offset(fInputOffset); - } + fResult.decimalVal.s128Value = + *row.getBinaryField_offset(fInputOffset); fResult.decimalVal.scale = (unsigned)fResultType.scale; break; } diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 760760a9d..6448d12e2 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -37,7 +37,6 @@ #include "dataconvert.h" using int128_t = __int128; -using uint128_t = unsigned __int128; namespace messageqcpp { @@ -66,14 +65,14 @@ struct IDB_Decimal { IDB_Decimal(): value(0), scale(0), precision(0) { - __v.__s128 = 0; + s128Value = 0; } IDB_Decimal(int64_t val, int8_t s, uint8_t p) : value (val), scale(s), precision(p) { - __v.__s128 = 0; + s128Value = 0; } int decimalComp(const IDB_Decimal& d) const @@ -158,11 +157,7 @@ struct IDB_Decimal return (decimalComp(rhs) != 0); } - union { - uint128_t __u128; - int128_t __s128; - int64_t __s64[2]; - } __v; + int128_t s128Value; int64_t value; int8_t scale; // 0~38 uint8_t precision; // 1~38 diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 0529523cc..fa2c4a77d 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -820,6 +820,7 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h dataconvert::DataConvert::decimalToString(dec, (unsigned)colType.scale, buf, sizeof(buf), colType.colDataType); + std::cout << buf << std::endl; Field_new_decimal* f2 = (Field_new_decimal*)*f; f2->store(buf, strlen(buf), f2->charset()); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 000000000..5bc8f600a --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,19 @@ +include_directories( ${ENGINE_COMMON_INCLUDES} ) + +if (WITH_ROWGROUP_UT) + add_executable(rowgroup_tests rowgroup-tests.cpp) + target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) + install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) +endif() + +if (WITH_ARITHMETICOPERATOR_UT) + add_executable(arithmeticoperator_tests arithmeticoperator-tests.cpp) + target_link_libraries(arithmeticoperator_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) + install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) +endif() + +if (WITH_DATACONVERT_UT) + add_executable(dataconvert_tests dataconvert-tests.cpp) + target_link_libraries(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) + install(TARGETS dataconvert_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) +endif() diff --git a/tests/arithmeticoperator-tests.cpp b/tests/arithmeticoperator-tests.cpp new file mode 100644 index 000000000..dc192fbd1 --- /dev/null +++ b/tests/arithmeticoperator-tests.cpp @@ -0,0 +1,27 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include // googletest header file + +using int128_t = __int128; +using uint128_t = unsigned __int128; +//using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; + +TEST(SimpleCheck, check1) +{ + EXPECT_EQ(true, true); +} diff --git a/utils/dataconvert/dataconvert-tests.cpp b/tests/dataconvert-tests.cpp similarity index 96% rename from utils/dataconvert/dataconvert-tests.cpp rename to tests/dataconvert-tests.cpp index b184030cb..461ca28f1 100644 --- a/utils/dataconvert/dataconvert-tests.cpp +++ b/tests/dataconvert-tests.cpp @@ -590,6 +590,31 @@ TEST(DataConvertTest, DecimalToStringCheckScale3) { } } +TEST(DataConvertTest, DecimalToStringCheckScale10) { + std::vector expected; + std::vector decimalAsStringVec; + // scale 3 + uint8_t scale3 = 10; + decimalAsStringVec.push_back("10000000009"); + decimalAsStringVec.push_back("-10000000009"); + decimalAsStringVec.push_back("0"); + decimalAsStringVec.push_back("-10000000010"); + expected.push_back("1.0000000009"); + expected.push_back("-1.0000000009"); + expected.push_back("0.0000000000"); + expected.push_back("-1.0000000010"); + char buf[42]; + CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL; + for (int i=0; i < expected.size(); i++) { + int128_t value = -4; + memset(buf, 0, 42); + dataconvert::atoi128(decimalAsStringVec[i], value); + dataconvert::DataConvert::decimalToString(&value, scale3, buf, + utils::precisionByWidth(sizeof(value))+4, type); + EXPECT_EQ(expected[i], std::string(buf)); + } +} + TEST(DataConvertTest, DecimalToStringCheckScale37) { std::vector expected; std::vector decimalAsStringVec; diff --git a/tests/rowgroup-tests.cpp b/tests/rowgroup-tests.cpp new file mode 100644 index 000000000..8fbbe22fc --- /dev/null +++ b/tests/rowgroup-tests.cpp @@ -0,0 +1,243 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include // googletest header file +#include + +#include "rowgroup.h" +#include "columnwidth.h" +#include "joblisttypes.h" +#include "dataconvert.h" + +#define WIDE_DEC_PRECISION 38U +#define INITIAL_ROW_OFFSET 2 + +using int128_t = __int128; +using uint128_t = unsigned __int128; +using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; + +class RowDecimalTest : public ::testing::Test { + protected: + void SetUp() override { + uint32_t precision = WIDE_DEC_PRECISION; + uint32_t oid =3001; + + std::vectortypes; + std::vectorprecisionVec; + std::vector roids, tkeys, cscale; + types.push_back(execplan::CalpontSystemCatalog::DECIMAL); + types.push_back(execplan::CalpontSystemCatalog::UDECIMAL); + for (size_t i=0; i <= 3; i++) { + types.push_back(execplan::CalpontSystemCatalog::DECIMAL); + } + precisionVec.push_back(precision); + precisionVec.push_back(precision); + precisionVec.push_back(18); + precisionVec.push_back(9); + precisionVec.push_back(4); + precisionVec.push_back(2); + std::vectorwidthVec; + uint32_t offset = INITIAL_ROW_OFFSET; + offsets.push_back(offset); + for (size_t i=0; i < types.size(); i++) { + uint8_t width = utils::widthByPrecision(precisionVec[i]); + widthVec.push_back(width); + offset += width; + offsets.push_back(offset); + roids.push_back(oid+i); + tkeys.push_back(i+1); cscale.push_back(0); + } + /*offsets.push_back(INITIAL_ROW_OFFSET); + offsets.push_back(16+INITIAL_ROW_OFFSET); + offsets.push_back(16*2+INITIAL_ROW_OFFSET); + roids.push_back(oid); roids.push_back(oid+1); + tkeys.push_back(1); tkeys.push_back(1); + cscale.push_back(0); cscale.push_back(0);*/ + + rowgroup::RowGroup inRG(roids.size(), //column count + offsets, //oldOffset + roids, // column oids + tkeys, //keys + types, // types + cscale, //scale + precisionVec, // precision + 20, // sTableThreshold + false //useStringTable + ); + + rg = inRG; + rgD.reinit(rg); + rg.setData(&rgD); + + rg.initRow(&r); + rg.initRow(&rOutMappingCheck); + rowSize = r.getSize(); + rg.getRow(0, &r); + + int128_t nullValue = 0; + int128_t bigValue = 0; + uint64_t* uint128_pod = reinterpret_cast(&nullValue); + uint128_pod[0] = joblist::UBIGINTNULL; + uint128_pod[1] = joblist::UBIGINTEMPTYROW; + bigValue = -static_cast(0xFFFFFFFF)*0xFFFFFFFFFFFFFFFF; + sValueVector.push_back(nullValue); + sValueVector.push_back(-42); + sValueVector.push_back(bigValue); + sValueVector.push_back(0); + sValueVector.push_back(nullValue-1); + + uValueVector.push_back(nullValue); + uValueVector.push_back(42); + uValueVector.push_back(bigValue); + uValueVector.push_back(0); + uValueVector.push_back(nullValue-1); + + s8ValueVector.push_back(joblist::TINYINTNULL); + s8ValueVector.push_back(-0x79); + s8ValueVector.push_back(0); + s8ValueVector.push_back(0x81); + s8ValueVector.push_back(joblist::TINYINTNULL-1); + + s16ValueVector.push_back(joblist::SMALLINTNULL); + s16ValueVector.push_back(-0x79); + s16ValueVector.push_back(0); + s16ValueVector.push_back(0x81); + s16ValueVector.push_back(joblist::SMALLINTNULL-1); + + s32ValueVector.push_back(joblist::INTNULL); + s32ValueVector.push_back(-0x79); + s32ValueVector.push_back(0); + s32ValueVector.push_back(0x81); + s32ValueVector.push_back(joblist::INTNULL-1); + + s64ValueVector.push_back(joblist::BIGINTNULL); + s64ValueVector.push_back(-0x79); + s64ValueVector.push_back(0); + s64ValueVector.push_back(0x81); + s64ValueVector.push_back(joblist::BIGINTNULL-1); + + for(size_t i = 0; i < sValueVector.size(); i++) { + r.setBinaryField_offset(&sValueVector[i], + sizeof(sValueVector[0]), offsets[0]); + r.setBinaryField_offset(&uValueVector[i], + sizeof(uValueVector[0]), offsets[1]); + r.setIntField(s64ValueVector[i], 2); + r.setIntField(s32ValueVector[i], 3); + r.setIntField(s16ValueVector[i], 4); + r.setIntField(s8ValueVector[i], 5); + r.nextRow(rowSize); + } + rowCount = sValueVector.size(); + } + // void TearDown() override {} + + rowgroup::Row r; + rowgroup::Row rOutMappingCheck; + rowgroup::RowGroup rg; + rowgroup::RGData rgD; + uint32_t rowSize; + size_t rowCount; + std::vector sValueVector; + std::vector uValueVector; + std::vector s8ValueVector; + std::vector s16ValueVector; + std::vector s32ValueVector; + std::vector s64ValueVector; + std::vector offsets; +}; + +TEST_F(RowDecimalTest, NonNULLValuesCheck) { + rg.getRow(1, &r); + for (size_t i = 1; i <= sValueVector.size(); i++) { + EXPECT_FALSE(r.isNullValue(0)); + EXPECT_FALSE(r.isNullValue(1)); + EXPECT_FALSE(r.isNullValue(2)); + EXPECT_FALSE(r.isNullValue(3)); + EXPECT_FALSE(r.isNullValue(4)); + EXPECT_FALSE(r.isNullValue(5)); + r.nextRow(rowSize); + } +} + +TEST_F(RowDecimalTest, initToNullANDisNullValueValueCheck) { + rg.getRow(0, &r); + r.initToNull(); + EXPECT_TRUE(r.isNullValue(0)); + EXPECT_TRUE(r.isNullValue(1)); + EXPECT_TRUE(r.isNullValue(2)); + EXPECT_TRUE(r.isNullValue(3)); + EXPECT_TRUE(r.isNullValue(4)); + EXPECT_TRUE(r.isNullValue(5)); +} + +TEST_F(RowDecimalTest, getBinaryFieldCheck) { + rg.getRow(0, &r); + uint128_t* u128Value; + int128_t* s128Value; +// std::remove_reference::type uType; +// std::remove_reference::type sType; + + for (size_t i = 0; i < sValueVector.size(); i++) { + s128Value = r.getBinaryField(0); + EXPECT_EQ(sValueVector[i], *s128Value); + u128Value = r.getBinaryField(1); + EXPECT_EQ(uValueVector[i], *u128Value); + //EXPECT_EQ(s64ValueVector[i], r.getIntField(2)); + //EXPECT_EQ(s32ValueVector[i],r.getIntField(3)); + //EXPECT_EQ(r.getIntField(4),s16ValueVector[i]); + //EXPECT_EQ(r.getIntField(5),s8ValueVector[i]); + r.nextRow(rowSize); + } +} + +TEST_F(RowDecimalTest, toStringCheck) { + std::vector exemplarVector; + exemplarVector.push_back(std::string("0: NULL NULL NULL NULL NULL NULL ")); + exemplarVector.push_back(std::string("0: -42 42 -121 -121 -121 -121 ")); + exemplarVector.push_back(std::string("0: -79228162495817593515539431425 -79228162495817593515539431425 0 0 0 0 ")); + exemplarVector.push_back(std::string("0: 0 0 129 129 129 -127 ")); + exemplarVector.push_back(std::string("0: -3 -3 9223372036854775807 2147483647 32767 127 ")); + + rg.getRow(0, &r); + r.initToNull(); + for (auto &el: exemplarVector) { + EXPECT_EQ(el, r.toString()); + r.nextRow(rowSize); + } +} + +TEST_F(RowDecimalTest, toCSVCheck) { +} + +TEST_F(RowDecimalTest, applyMappingCheck) { + int mapping[] = {0, 1, -1, -1, -1, -1}; + rg.getRow(1, &r); + rg.getRow(2, &rOutMappingCheck); + int128_t* s128Value = rOutMappingCheck.getBinaryField(0); + uint128_t* u128Value = rOutMappingCheck.getBinaryField(1); + EXPECT_NE(sValueVector[1], *s128Value); + EXPECT_NE(uValueVector[1], *u128Value); + applyMapping(mapping, r, &rOutMappingCheck); + s128Value = rOutMappingCheck.getBinaryField(0); + EXPECT_EQ(sValueVector[1], *s128Value); + u128Value = rOutMappingCheck.getBinaryField(1); + EXPECT_EQ(uValueVector[1], *u128Value); +} + +// WIP +TEST_F(RowDecimalTest, RowEqualsCheck) { +} diff --git a/utils/dataconvert/CMakeLists.txt b/utils/dataconvert/CMakeLists.txt index 168efdf80..9a25dfb1b 100644 --- a/utils/dataconvert/CMakeLists.txt +++ b/utils/dataconvert/CMakeLists.txt @@ -15,5 +15,5 @@ install(TARGETS dataconvert DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-e if (WITH_DATACONVERT_UT) add_executable(dataconvert_tests dataconvert-tests.cpp) target_link_libraries(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) - install(TARGETS dataconvert_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) + install(TARGETS dataconvert_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) endif() diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index dac9e521e..cb4d7fe27 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1334,14 +1334,11 @@ size_t DataConvert::writeFractionalPart(int128_t* dec, char* p, scaleDivisor *= columnstore_pow_10[scale%maxPowOf10]; } - //for (size_t i = 1; i < scale; i++) - // scaleDivisor *= 10; - int128_t fractionalPart = *dec % scaleDivisor; // divide by the base untill we have non-zero quotinent size_t written = 0; scaleDivisor /= 10; - while (scaleDivisor > 1 && *dec / scaleDivisor == 0) + while (scaleDivisor > 1 && fractionalPart/scaleDivisor == 0) { *p++ = '0'; written++; @@ -1385,6 +1382,8 @@ void DataConvert::toString(int128_t* dec, uint8_t scale, p += writeFractionalPart(dec, p, buflen-(p-original_p), scale); } + *p = '\0'; + if (buflen <= p-original_p) { throw QueryDataExcept("toString() char buffer overflow.", formatErr); diff --git a/utils/funcexp/funcexp.cpp b/utils/funcexp/funcexp.cpp index d20382d30..1499c9744 100644 --- a/utils/funcexp/funcexp.cpp +++ b/utils/funcexp/funcexp.cpp @@ -472,18 +472,12 @@ void FuncExp::evaluate(rowgroup::Row& row, std::vector& expressi { IDB_Decimal val = expression[i]->getDecimalVal(row, isNull); - // WIP - if (expression[i]->resultType().precision > 18) + // WIP check for null and overflow here. + if (expression[i]->resultType().colWidth == 16) { - // WIP make this a separate function w and w/o overflow check - if (expression[i]->resultType().colDataType == execplan::CalpontSystemCatalog::DECIMAL) - row.setBinaryField_offset(&val.__v.__s128, - expression[i]->resultType().colWidth, - row.getOffset(expression[i]->outputIndex())); - else - row.setBinaryField_offset(&val.__v.__u128, - expression[i]->resultType().colWidth, - row.getOffset(expression[i]->outputIndex())); + row.setBinaryField_offset(&val.s128Value, + expression[i]->resultType().colWidth, + row.getOffset(expression[i]->outputIndex())); } else { diff --git a/utils/rowgroup/CMakeLists.txt b/utils/rowgroup/CMakeLists.txt index 86c5fa9db..3fcde10df 100644 --- a/utils/rowgroup/CMakeLists.txt +++ b/utils/rowgroup/CMakeLists.txt @@ -19,5 +19,5 @@ install(TARGETS rowgroup DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engi if (WITH_ROWGROUP_UT) add_executable(rowgroup_tests rowgroup-tests.cpp) target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) - install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) + install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) endif()