diff --git a/.drone.jsonnet b/.drone.jsonnet index 882c3d70a..399ba1549 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -25,9 +25,9 @@ local platformMap(branch, platform) = }; local platform_map = { - 'opensuse/leap:15': 'zypper ' + rpm_build_deps + ' cmake libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel libquadmath0 && cmake ' + cmakeflags + branch_cmakeflags_map[branch] + ' -DRPM=sles15 && make -j$(nproc) package', - 'centos:7': 'yum install -y epel-release && yum install -y cmake3 && ln -s /usr/bin/cmake3 /usr/bin/cmake && yum ' + rpm_build_deps + ' libquadmath && cmake ' + cmakeflags + branch_cmakeflags_map[branch] + ' -DRPM=centos7 && make -j$(nproc) package', - 'centos:8': "yum install -y libgcc && sed -i 's/enabled=0/enabled=1/' /etc/yum.repos.d/CentOS-PowerTools.repo && yum " + rpm_build_deps + ' cmake libquadmath && cmake ' + cmakeflags + branch_cmakeflags_map[branch] + ' -DRPM=centos8 && make -j$(nproc) package', + 'opensuse/leap:15': 'zypper ' + rpm_build_deps + ' cmake libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel gcc-fortran && cmake ' + cmakeflags + branch_cmakeflags_map[branch] + ' -DRPM=sles15 && make -j$(nproc) package', + 'centos:7': 'yum install -y epel-release && yum install -y cmake3 && ln -s /usr/bin/cmake3 /usr/bin/cmake && yum ' + rpm_build_deps + ' libquadmath libquadmath-devel && cmake ' + cmakeflags + branch_cmakeflags_map[branch] + ' -DRPM=centos7 && make -j$(nproc) package', + 'centos:8': "yum install -y libgcc && sed -i 's/enabled=0/enabled=1/' /etc/yum.repos.d/CentOS-PowerTools.repo && yum " + rpm_build_deps + ' cmake libquadmath libquadmath-devel && cmake ' + cmakeflags + branch_cmakeflags_map[branch] + ' -DRPM=centos8 && make -j$(nproc) package', 'debian:9': deb_build_deps + " && CMAKEFLAGS='" + cmakeflags + branch_cmakeflags_map[branch] + " -DDEB=stretch' debian/autobake-deb.sh", 'debian:10': deb_build_deps + " && CMAKEFLAGS='" + cmakeflags + branch_cmakeflags_map[branch] + " -DDEB=buster' debian/autobake-deb.sh", 'ubuntu:16.04': deb_build_deps + " && CMAKEFLAGS='" + cmakeflags + branch_cmakeflags_map[branch] + " -DDEB=xenial' debian/autobake-deb.sh", diff --git a/CMakeLists.txt b/CMakeLists.txt index bc7033ccb..e0a3fd636 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -148,6 +148,13 @@ if (NOT CURL_FOUND) return() endif() +IF (WITH_GTEST) + INCLUDE (FindGTest) + IF (NOT GTEST_FOUND) + MESSAGE(FATAL_ERROR "GSuite libs not found but are requested. Please install them or build.") + ENDIF() + SET (GTEST_LIBRARIES ${GTEST_LIBRARY} ${GTESTMAIN_LIBRARY} ${PTHREAD_LIBRARY}) +ENDIF() FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable") if(NOT AWK_EXECUTABLE) @@ -193,7 +200,8 @@ ENDIF() SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed") -SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client) +SET (ENGINE_DT_LIB datatypes) +SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client ${ENGINE_DT_LIB}) SET (ENGINE_OAM_LIBS oamcpp alarmmanager) SET (ENGINE_BRM_LIBS brm idbdatafile cacheutils rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS}) SET (ENGINE_EXEC_LIBS joblist execplan windowfunction joiner rowgroup funcexp udfsdk regr dataconvert common compress querystats querytele thrift threadpool ${ENGINE_BRM_LIBS}) @@ -277,6 +285,7 @@ SET (ENGINE_UTILS_DDLCLEANUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/ddlclea SET (ENGINE_UTILS_QUERYSTATS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/querystats") SET (ENGINE_UTILS_LIBMYSQL_CL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/libmysql_client") SET (ENGINE_WE_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/xml") +SET (ENGINE_DATATYPES_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/datatypes") SET (ENGINE_SERVER_SQL_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/sql") SET (ENGINE_SERVER_INCLUDE_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/include") IF (PCRE_INCLUDES) @@ -298,7 +307,7 @@ ELSE () SET (ENGINE_READLINE_LIBRARY "readline") ENDIF () -SET (ENGINE_COMMON_INCLUDES ${ENGINE_DEFAULT_INCLUDES} ${Boost_INCLUDE_DIR} ${LIBXML2_INCLUDE_DIR} ${ENGINE_UTILS_MESSAGEQCPP_INCLUDE} ${ENGINE_WE_SHARED_INCLUDE} ${ENGINE_UTILS_IDBDATAFILE_INCLUDE} ${ENGINE_UTILS_LOGGINGCPP_INCLUDE} ${ENGINE_UTILS_CONFIGCPP_INCLUDE} ${ENGINE_UTILS_COMPRESS_INCLUDE} ${ENGINE_VERSIONING_BRM_INCLUDE} ${ENGINE_UTILS_ROWGROUP_INCLUDE} ${ENGINE_UTILS_COMMON_INCLUDE} ${ENGINE_UTILS_DATACONVERT_INCLUDE} ${ENGINE_UTILS_RWLOCK_INCLUDE} ${ENGINE_UTILS_FUNCEXP_INCLUDE} ${ENGINE_OAMAPPS_ALARMMANAGER_INCLUDE} ${ENGINE_UTILS_INCLUDE} ${ENGINE_OAM_OAMCPP_INCLUDE} ${ENGINE_DBCON_DDLPKGPROC_INCLUDE} ${ENGINE_DBCON_DDLPKG_INCLUDE} ${ENGINE_DBCON_EXECPLAN_INCLUDE} ${ENGINE_UTILS_STARTUP_INCLUDE} ${ENGINE_DBCON_JOBLIST_INCLUDE} ${ENGINE_WE_WRAPPER_INCLUDE} ${ENGINE_WE_SERVER_INCLUDE} ${ENGINE_DBCON_DMLPKG_INCLUDE} ${ENGINE_WE_CLIENT_INCLUDE} ${ENGINE_DBCON_DMLPKGPROC_INCLUDE} ${ENGINE_UTILS_CACHEUTILS_INCLUDE} ${ENGINE_UTILS_MYSQLCL_INCLUDE} ${ENGINE_UTILS_QUERYTELE_INCLUDE} ${ENGINE_UTILS_THRIFT_INCLUDE} ${ENGINE_UTILS_JOINER_INCLUDE} ${ENGINE_UTILS_THREADPOOL_INCLUDE} ${ENGINE_UTILS_BATCHLDR_INCLUDE} ${ENGINE_UTILS_DDLCLEANUP_INCLUDE} ${ENGINE_UTILS_QUERYSTATS_INCLUDE} ${ENGINE_WE_CONFIGCPP_INCLUDE} ${ENGINE_SERVER_SQL_INCLUDE} ${ENGINE_SERVER_INCLUDE_INCLUDE} ${ENGINE_SERVER_PCRE_INCLUDE} ${ENGINE_SERVER_WSREP_API_INCLUDE} ${ENGINE_SERVER_WSREP_INCLUDE} ${ENGINE_UTILS_UDFSDK_INCLUDE} ${ENGINE_UTILS_LIBMYSQL_CL_INCLUDE}) +SET (ENGINE_COMMON_INCLUDES ${ENGINE_DEFAULT_INCLUDES} ${Boost_INCLUDE_DIR} ${LIBXML2_INCLUDE_DIR} ${ENGINE_UTILS_MESSAGEQCPP_INCLUDE} ${ENGINE_WE_SHARED_INCLUDE} ${ENGINE_UTILS_IDBDATAFILE_INCLUDE} ${ENGINE_UTILS_LOGGINGCPP_INCLUDE} ${ENGINE_UTILS_CONFIGCPP_INCLUDE} ${ENGINE_UTILS_COMPRESS_INCLUDE} ${ENGINE_VERSIONING_BRM_INCLUDE} ${ENGINE_UTILS_ROWGROUP_INCLUDE} ${ENGINE_UTILS_COMMON_INCLUDE} ${ENGINE_UTILS_DATACONVERT_INCLUDE} ${ENGINE_UTILS_RWLOCK_INCLUDE} ${ENGINE_UTILS_FUNCEXP_INCLUDE} ${ENGINE_OAMAPPS_ALARMMANAGER_INCLUDE} ${ENGINE_UTILS_INCLUDE} ${ENGINE_OAM_OAMCPP_INCLUDE} ${ENGINE_DBCON_DDLPKGPROC_INCLUDE} ${ENGINE_DBCON_DDLPKG_INCLUDE} ${ENGINE_DBCON_EXECPLAN_INCLUDE} ${ENGINE_UTILS_STARTUP_INCLUDE} ${ENGINE_DBCON_JOBLIST_INCLUDE} ${ENGINE_WE_WRAPPER_INCLUDE} ${ENGINE_WE_SERVER_INCLUDE} ${ENGINE_DBCON_DMLPKG_INCLUDE} ${ENGINE_WE_CLIENT_INCLUDE} ${ENGINE_DBCON_DMLPKGPROC_INCLUDE} ${ENGINE_UTILS_CACHEUTILS_INCLUDE} ${ENGINE_UTILS_MYSQLCL_INCLUDE} ${ENGINE_UTILS_QUERYTELE_INCLUDE} ${ENGINE_UTILS_THRIFT_INCLUDE} ${ENGINE_UTILS_JOINER_INCLUDE} ${ENGINE_UTILS_THREADPOOL_INCLUDE} ${ENGINE_UTILS_BATCHLDR_INCLUDE} ${ENGINE_UTILS_DDLCLEANUP_INCLUDE} ${ENGINE_UTILS_QUERYSTATS_INCLUDE} ${ENGINE_WE_CONFIGCPP_INCLUDE} ${ENGINE_SERVER_SQL_INCLUDE} ${ENGINE_SERVER_INCLUDE_INCLUDE} ${ENGINE_SERVER_PCRE_INCLUDE} ${ENGINE_SERVER_WSREP_API_INCLUDE} ${ENGINE_SERVER_WSREP_INCLUDE} ${ENGINE_UTILS_UDFSDK_INCLUDE} ${ENGINE_UTILS_LIBMYSQL_CL_INCLUDE} ${ENGINE_DATATYPES_INCLUDE}) ADD_SUBDIRECTORY(dbcon/mysql) IF(NOT TARGET columnstore) @@ -329,6 +338,8 @@ ADD_SUBDIRECTORY(writeengine/server) ADD_SUBDIRECTORY(writeengine/bulk) ADD_SUBDIRECTORY(writeengine/splitter) ADD_SUBDIRECTORY(storage-manager) +ADD_SUBDIRECTORY(datatypes) +ADD_SUBDIRECTORY(tests) # WriteEngine component tests IF( WITH_SHARED_COMP_TESTS ) diff --git a/cmake/FindGTest.cmake b/cmake/FindGTest.cmake new file mode 100644 index 000000000..5860ca415 --- /dev/null +++ b/cmake/FindGTest.cmake @@ -0,0 +1,37 @@ +find_path(GTEST_ROOT_DIR + NAMES include/gtest/gtest.h +) + +find_library(GTEST_LIBRARY + NAMES gtest + HINTS ${GTEST_ROOT_DIR}/lib +) + +find_library(GTESTMAIN_LIBRARY + NAMES gtest_main + HINTS ${GTEST_ROOT_DIR}/lib +) + +find_library(PTHREAD_LIBRARY + NAMES pthread + HINTS ${GTEST_ROOT_DIR}/lib +) + +find_path(GTEST_INCLUDE_DIR + NAMES gtest.h + HINTS ${GTEST_ROOT_DIR}/include/gtest +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(GTest DEFAULT_MSG + GTEST_LIBRARY + GTESTMAIN_LIBRARY + PTHREAD_LIBRARY + GTEST_INCLUDE_DIR +) + +mark_as_advanced( + GTEST_ROOT_DIR + GTEST_LIBRARIES + GTEST_INCLUDE_DIR +) diff --git a/datatypes/CMakeLists.txt b/datatypes/CMakeLists.txt new file mode 100644 index 000000000..c74eaf7fc --- /dev/null +++ b/datatypes/CMakeLists.txt @@ -0,0 +1,10 @@ + +include_directories( ${ENGINE_COMMON_INCLUDES} ) +set(datatypes_LIB_SRCS + mcs_int128.cpp + mcs_decimal.cpp) + +add_library(datatypes SHARED ${datatypes_LIB_SRCS}) + +install(TARGETS datatypes DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) + diff --git a/datatypes/mcs_datatype.cpp b/datatypes/mcs_datatype.cpp new file mode 100644 index 000000000..493d64fd9 --- /dev/null +++ b/datatypes/mcs_datatype.cpp @@ -0,0 +1,2080 @@ +/* + Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include +#include +#include +#ifdef _MSC_VER +#include +#else +#include +#endif +#include +#include +#include +#include +#ifdef _MSC_VER +#include +#else +#include +#endif +#include +#include +using namespace std; + +#include + + +#include "dataconvert.h" +using namespace dataconvert; + +#include "simplecolumn.h" +#include "simplecolumn_int.h" +#include "simplecolumn_uint.h" +#include "simplecolumn_decimal.h" + +#include "rowgroup.h" +#include "ddlpkg.h" +#include "dbrm.h" +#include "we_typeext.h" +#include "joblisttypes.h" + +namespace datatypes +{ + +int128_t SystemCatalog::TypeAttributesStd::decimal128FromString(const std::string& value) const +{ + int128_t result = 0; + bool pushWarning = false; + bool noRoundup = false; + dataconvert::number_int_value(value, + SystemCatalog::DECIMAL, + *this, + pushWarning, + noRoundup, + result); + return result; +} + + +const string & TypeHandlerSInt8::name() const +{ + static const string xname= "TINYINT"; + return xname; +} + + +const string & TypeHandlerUInt8::name() const +{ + static const string xname= "UTINYINT"; + return xname; +} + + +const string & TypeHandlerSInt16::name() const +{ + static const string xname= "SMALLINT"; + return xname; +} + + +const string & TypeHandlerUInt16::name() const +{ + static const string xname= "USMALLINT"; + return xname; +} + + +const string & TypeHandlerSInt24::name() const +{ + static const string xname= "MEDINT"; + return xname; +} + + +const string & TypeHandlerUInt24::name() const +{ + static const string xname= "UMEDINT"; + return xname; +} + + +const string & TypeHandlerSInt32::name() const +{ + static const string xname= "INT"; + return xname; +} + + +const string & TypeHandlerUInt32::name() const +{ + static const string xname= "UINT"; + return xname; +} + + +const string & TypeHandlerSInt64::name() const +{ + static const string xname= "BIGINT"; + return xname; +} + + +const string & TypeHandlerUInt64::name() const +{ + static const string xname= "UBIGINT"; + return xname; +} + + +const string & TypeHandlerSFloat::name() const +{ + static const string xname= "FLOAT"; + return xname; +} + + +const string & TypeHandlerUFloat::name() const +{ + static const string xname= "UFLOAT"; + return xname; +} + + +const string & TypeHandlerSDouble::name() const +{ + static const string xname= "DOUBLE"; + return xname; +} + + +const string & TypeHandlerUDouble::name() const +{ + static const string xname= "UDOUBLE"; + return xname; +} + + +const string & TypeHandlerSLongDouble::name() const +{ + static const string xname= "LONGDOUBLE"; + return xname; +} + + +const string & TypeHandlerSDecimal64::name() const +{ + static const string xname= "DECIMAL"; + return xname; +} + + +const string & TypeHandlerUDecimal64::name() const +{ + static const string xname= "UDECIMAL"; + return xname; +} + + +const string & TypeHandlerSDecimal128::name() const +{ + static const string xname= "DECIMAL"; + return xname; +} + + +const string & TypeHandlerUDecimal128::name() const +{ + static const string xname= "UDECIMAL"; + return xname; +} + + +const string & TypeHandlerDate::name() const +{ + static const string xname= "DATE"; + return xname; +} + + +const string & TypeHandlerDatetime::name() const +{ + static const string xname= "DATETIME"; + return xname; +} + + +const string & TypeHandlerTime::name() const +{ + static const string xname= "TIME"; + return xname; +} + + +const string & TypeHandlerTimestamp::name() const +{ + static const string xname= "TIMESTAMP"; + return xname; +} + + +const string & TypeHandlerChar::name() const +{ + static const string xname= "CHAR"; + return xname; +} + + +const string & TypeHandlerVarchar::name() const +{ + static const string xname= "VARCHAR"; + return xname; +} + + +const string & TypeHandlerVarbinary::name() const +{ + static const string xname= "VARBINARY"; + return xname; +} + + +const string & TypeHandlerBlob::name() const +{ + static const string xname= "BLOB"; + return xname; +} + + +const string & TypeHandlerClob::name() const +{ + static const string xname= "CLOB"; + return xname; +} + + +const string & TypeHandlerText::name() const +{ + static const string xname= "TEXT"; + return xname; +} + + +const string & TypeHandlerBit::name() const +{ + static const string xname= "BIT"; + return xname; +} + + +TypeHandlerBit mcs_type_handler_bit; + +TypeHandlerSInt8 mcs_type_handler_sint8; +TypeHandlerSInt16 mcs_type_handler_sint16; +TypeHandlerSInt24 mcs_type_handler_sint24; +TypeHandlerSInt32 mcs_type_handler_sint32; +TypeHandlerSInt64 mcs_type_handler_sint64; + +TypeHandlerUInt8 mcs_type_handler_uint8; +TypeHandlerUInt16 mcs_type_handler_uint16; +TypeHandlerUInt24 mcs_type_handler_uint24; +TypeHandlerUInt32 mcs_type_handler_uint32; +TypeHandlerUInt64 mcs_type_handler_uint64; + +TypeHandlerSFloat mcs_type_handler_sfloat; +TypeHandlerSDouble mcs_type_handler_sdouble; +TypeHandlerSLongDouble mcs_type_handler_slongdouble; + +TypeHandlerUFloat mcs_type_handler_ufloat; +TypeHandlerUDouble mcs_type_handler_udouble; + +TypeHandlerSDecimal64 mcs_type_handler_sdecimal64; +TypeHandlerUDecimal64 mcs_type_handler_udecimal64; + +TypeHandlerSDecimal128 mcs_type_handler_sdecimal128; +TypeHandlerUDecimal128 mcs_type_handler_udecimal128; + +TypeHandlerDate mcs_type_handler_date; +TypeHandlerTime mcs_type_handler_time; +TypeHandlerDatetime mcs_type_handler_datetime; +TypeHandlerTimestamp mcs_type_handler_timestamp; + + +TypeHandlerChar mcs_type_handler_char; +TypeHandlerVarchar mcs_type_handler_varchar; +TypeHandlerText mcs_type_handler_text; +TypeHandlerClob mcs_type_handler_clob; +TypeHandlerVarbinary mcs_type_handler_varbinary; +TypeHandlerBlob mcs_type_handler_blob; + + +const TypeHandler * +TypeHandler::find(SystemCatalog::ColDataType typeCode, + const SystemCatalog::TypeAttributesStd &ct) +{ + switch (typeCode) { + case SystemCatalog::BIT: return &mcs_type_handler_bit; + case SystemCatalog::TINYINT: return &mcs_type_handler_sint8; + case SystemCatalog::SMALLINT: return &mcs_type_handler_sint16; + case SystemCatalog::MEDINT: return &mcs_type_handler_sint24; + case SystemCatalog::INT: return &mcs_type_handler_sint32; + case SystemCatalog::BIGINT: return &mcs_type_handler_sint64; + case SystemCatalog::UTINYINT: return &mcs_type_handler_uint8; + case SystemCatalog::USMALLINT: return &mcs_type_handler_uint16; + case SystemCatalog::UMEDINT: return &mcs_type_handler_uint24; + case SystemCatalog::UINT: return &mcs_type_handler_uint32; + case SystemCatalog::UBIGINT: return &mcs_type_handler_uint64; + case SystemCatalog::FLOAT: return &mcs_type_handler_sfloat; + case SystemCatalog::DOUBLE: return &mcs_type_handler_sdouble; + case SystemCatalog::LONGDOUBLE: return &mcs_type_handler_slongdouble; + case SystemCatalog::UFLOAT: return &mcs_type_handler_ufloat; + case SystemCatalog::UDOUBLE: return &mcs_type_handler_udouble; + + case SystemCatalog::DECIMAL: + if (static_cast(ct.colWidth) < datatypes::MAXDECIMALWIDTH) + return &mcs_type_handler_sdecimal64; + else + return &mcs_type_handler_sdecimal128; + + case SystemCatalog::UDECIMAL: + if (static_cast(ct.colWidth) < datatypes::MAXDECIMALWIDTH) + return &mcs_type_handler_udecimal64; + else + return &mcs_type_handler_udecimal128; + + case SystemCatalog::TIME: return &mcs_type_handler_time; + case SystemCatalog::DATE: return &mcs_type_handler_date; + case SystemCatalog::DATETIME: return &mcs_type_handler_datetime; + case SystemCatalog::TIMESTAMP: return &mcs_type_handler_timestamp; + case SystemCatalog::CHAR: return &mcs_type_handler_char; + case SystemCatalog::VARCHAR: return &mcs_type_handler_varchar; + case SystemCatalog::TEXT: return &mcs_type_handler_text; + case SystemCatalog::CLOB: return &mcs_type_handler_clob; + case SystemCatalog::VARBINARY: return &mcs_type_handler_varbinary; + case SystemCatalog::BLOB: return &mcs_type_handler_blob; + + case SystemCatalog::NUM_OF_COL_DATA_TYPE: + case SystemCatalog::STRINT: + case SystemCatalog::UNDEFINED: + case SystemCatalog::BINARY: + break; + } + return NULL; +} + + +const TypeHandler * +SystemCatalog::TypeHolderStd::typeHandler() const +{ + return TypeHandler::find(colDataType, *this); +} + + +boost::any +SystemCatalog::TypeHolderStd::getNullValueForType() const +{ + const TypeHandler *h= typeHandler(); + if (!h) + { + throw std::runtime_error("getNullValueForType: unkown column data type"); + return boost::any(); + } + return h->getNullValueForType(*this); +} + + +const TypeHandler * +TypeHandler::find_by_ddltype(const ddlpackage::ColumnType &ct) +{ + switch (ct.fType) { + case ddlpackage::DDL_CHAR: return &mcs_type_handler_char; + case ddlpackage::DDL_VARCHAR: return &mcs_type_handler_varchar; + case ddlpackage::DDL_VARBINARY: return &mcs_type_handler_varbinary; + case ddlpackage::DDL_BIT: return &mcs_type_handler_bit; + + case ddlpackage::DDL_REAL: + case ddlpackage::DDL_DECIMAL: + case ddlpackage::DDL_NUMERIC: + case ddlpackage::DDL_NUMBER: + + if (ct.fLength < datatypes::MAXDECIMALWIDTH) + return &mcs_type_handler_sdecimal64; + return &mcs_type_handler_sdecimal128; + + case ddlpackage::DDL_FLOAT: return &mcs_type_handler_sfloat; + case ddlpackage::DDL_DOUBLE: return &mcs_type_handler_sdouble; + + case ddlpackage::DDL_INT: + case ddlpackage::DDL_INTEGER: return &mcs_type_handler_sint32; + + case ddlpackage::DDL_BIGINT: return &mcs_type_handler_sint64; + case ddlpackage::DDL_MEDINT: return &mcs_type_handler_sint24; + case ddlpackage::DDL_SMALLINT: return &mcs_type_handler_sint16; + case ddlpackage::DDL_TINYINT: return &mcs_type_handler_sint8; + + case ddlpackage::DDL_DATE: return &mcs_type_handler_date; + case ddlpackage::DDL_DATETIME: return &mcs_type_handler_datetime; + case ddlpackage::DDL_TIME: return &mcs_type_handler_time; + case ddlpackage::DDL_TIMESTAMP: return &mcs_type_handler_timestamp; + + case ddlpackage::DDL_CLOB: return &mcs_type_handler_clob; + case ddlpackage::DDL_BLOB: return &mcs_type_handler_blob; + case ddlpackage::DDL_TEXT: return &mcs_type_handler_text; + + case ddlpackage::DDL_UNSIGNED_TINYINT: return &mcs_type_handler_uint8; + case ddlpackage::DDL_UNSIGNED_SMALLINT: return &mcs_type_handler_uint16; + case ddlpackage::DDL_UNSIGNED_MEDINT: return &mcs_type_handler_uint24; + case ddlpackage::DDL_UNSIGNED_INT: return &mcs_type_handler_uint32; + case ddlpackage::DDL_UNSIGNED_BIGINT: return &mcs_type_handler_uint64; + + case ddlpackage::DDL_UNSIGNED_DECIMAL: + case ddlpackage::DDL_UNSIGNED_NUMERIC: + + if (ct.fLength < datatypes::MAXDECIMALWIDTH) + return &mcs_type_handler_udecimal64; + return &mcs_type_handler_udecimal128; + + case ddlpackage::DDL_UNSIGNED_FLOAT: return &mcs_type_handler_ufloat; + case ddlpackage::DDL_UNSIGNED_DOUBLE: return &mcs_type_handler_udouble; + + case ddlpackage::DDL_BINARY: //return &mcs_type_handler_binary; + case ddlpackage::DDL_INVALID_DATATYPE: + break; + } + return NULL; +} + + +/****************************************************************************/ + + +int TypeHandlerDate::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + int64_t intColVal = row.getUintField<4>(pos); + return f->store_date(intColVal); +} + + +int TypeHandlerDatetime::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + int64_t intColVal = row.getUintField<8>(pos); + return f->store_datetime(intColVal); +} + + +int TypeHandlerTime::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + int64_t intColVal = row.getUintField<8>(pos); + return f->store_time(intColVal); +} + + +int TypeHandlerTimestamp::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + int64_t intColVal = row.getUintField<8>(pos); + return f->store_timestamp(intColVal); +} + + +int TypeHandlerStr::storeValueToFieldCharVarchar(rowgroup::Row &row, int pos, + StoreField *f) const +{ + int64_t intColVal; + switch (f->colWidth()) + { + case 1: + intColVal = row.getUintField<1>(pos); + return f->store_string((const char*)(&intColVal), strlen((char*)(&intColVal))); + + case 2: + intColVal = row.getUintField<2>(pos); + return f->store_string((char*)(&intColVal), strlen((char*)(&intColVal))); + + case 4: + intColVal = row.getUintField<4>(pos); + return f->store_string((char*)(&intColVal), strlen((char*)(&intColVal))); + + case 8: + { + //make sure we don't send strlen off into the weeds... + intColVal = row.getUintField<8>(pos); + char tmp[256]; + memcpy(tmp, &intColVal, 8); + tmp[8] = 0; + return f->store_string(tmp, strlen(tmp)); + } + default: + return f->store_string((const char*)row.getStringPointer(pos), row.getStringLength(pos)); + } +} + + +int TypeHandlerVarbinary::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + uint32_t l; + const uint8_t* p = row.getVarBinaryField(l, pos); + return f->store_varbinary((const char *) p, l); +} + + +int TypeHandlerSInt64::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + int64_t val = row.getIntField<8>(pos); + return f->store_xlonglong(val); +} + + +int TypeHandlerUInt64::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + uint64_t val = row.getUintField<8>(pos); + return f->store_xlonglong(static_cast(val)); +} + + +int TypeHandlerInt::storeValueToFieldSInt32(rowgroup::Row &row, int pos, + StoreField *f) const +{ + int64_t val = row.getIntField<4>(pos); + return f->store_xlonglong(val); +} + + +int TypeHandlerInt::storeValueToFieldUInt32(rowgroup::Row &row, int pos, + StoreField *f) const +{ + uint64_t val = row.getUintField<4>(pos); + return f->store_xlonglong(static_cast(val)); +} + + +int TypeHandlerSInt16::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + int64_t val = row.getIntField<2>(pos); + return f->store_xlonglong(val); +} + + +int TypeHandlerUInt16::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + uint64_t val = row.getUintField<2>(pos); + return f->store_xlonglong(static_cast(val)); +} + + +int TypeHandlerSInt8::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + int64_t val = row.getIntField<1>(pos); + return f->store_xlonglong(val); +} + + +int TypeHandlerUInt8::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + uint64_t val = row.getUintField<1>(pos); + return f->store_xlonglong(static_cast(val)); +} + + +/* + In this case, we're trying to load a double output column with float data. This is the + case when you do sum(floatcol), e.g. +*/ +int TypeHandlerReal::storeValueToFieldXFloat(rowgroup::Row &row, int pos, + StoreField *f) const +{ + float dl = row.getFloatField(pos); + return f->store_float(dl); +} + + +int TypeHandlerReal::storeValueToFieldXDouble(rowgroup::Row &row, int pos, + StoreField *f) const +{ + double dl = row.getDoubleField(pos); + return f->store_double(dl); +} + + +int TypeHandlerSLongDouble::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + long double dl = row.getLongDoubleField(pos); + return f->store_long_double(dl); +} + + +int TypeHandlerXDecimal::storeValueToField64(rowgroup::Row &row, int pos, + StoreField *f) const +{ + return f->store_decimal64(datatypes::VDecimal(row.getIntField(pos), + f->scale(), + f->precision())); +} + + +int TypeHandlerXDecimal::storeValueToField128(rowgroup::Row &row, int pos, + StoreField *f) const +{ + int128_t* decPtr = row.getBinaryField(pos); + return f->store_decimal128(datatypes::VDecimal(0, + f->scale(), + f->precision(), + decPtr)); +} + + +int TypeHandlerStr::storeValueToFieldBlobText(rowgroup::Row &row, int pos, + StoreField *f) const +{ + return f->store_lob((const char*) row.getVarBinaryField(pos), + row.getVarBinaryLength(pos)); +} + + +/* +int TypeHandlerBinary::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + Field_varstring* f2 = static_cast(f); + // WIP MCOL-641 Binary representation could contain \0. + char* binaryString = row.getBinaryField(pos); + return f2->store(binaryString, colType.colWidth, f2->charset()); +} +*/ + +/* + Default behaviour: treat as int64 + int64_t intColVal = row.getUintField<8>(pos); + storeNumericField(f, intColVal, colType); +*/ + + +/****************************************************************************/ + +string TypeHandlerDate::format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + return DataConvert::dateToString(v.toSInt64()); +} + + +string TypeHandlerDatetime::format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + return DataConvert::datetimeToString(v.toSInt64()); +} + +string TypeHandlerTimestamp::format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + return DataConvert::timestampToString(v.toSInt64(), v.tzname()); +} + + +string TypeHandlerTime::format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + return DataConvert::timeToString(v.toSInt64()); +} + + +string TypeHandlerChar::format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + // swap again to retain the string byte order + ostringstream oss; + uint64_t tmp = uint64ToStr(v.toSInt64()); + oss << (char*)(&tmp); + return oss.str(); +} + + +string TypeHandlerVarchar::format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + // swap again to retain the string byte order + ostringstream oss; + uint64_t tmp = uint64ToStr(v.toSInt64()); + oss << (char*)(&tmp); + return oss.str(); +} + + +string TypeHandlerInt::formatSInt64(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + ostringstream oss; + oss << v.toSInt64(); + return oss.str(); +} + + +string TypeHandlerInt::formatUInt64(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + ostringstream oss; + oss << static_cast(v.toSInt64()); + return oss.str(); +} + + +string TypeHandlerVarbinary::format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + return "N/A"; +} + + +string +TypeHandlerXDecimal::format64(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + idbassert(isValidXDecimal64(attr)); + ostringstream oss; + if (attr.scale > 0) + { + double d = ((double)(v.toSInt64()) / (double)pow((double)10, attr.scale)); + oss << setprecision(attr.scale) << fixed << d; + } + else + oss << v.toSInt64(); + return oss.str(); +} + + +string +TypeHandlerXDecimal::format128(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + idbassert(isValidXDecimal128(attr)); + datatypes::VDecimal dec(0, attr.scale, attr.precision, v.toSInt128()); + return dec.toString(true); +} + + +/****************************************************************************/ + +string TypeHandler::formatPartitionInfoSInt64( + const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &pi) const +{ + ostringstream output; + if (pi.isEmptyOrNullSInt64()) + output << setw(30) << "Empty/Null" + << setw(30) << "Empty/Null"; + else + output << setw(30) << format(SimpleValueSInt64(pi.min), attr) + << setw(30) << format(SimpleValueSInt64(pi.max), attr); + return output.str(); +} + + +string TypeHandler::formatPartitionInfoUInt64( + const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &pi) const +{ + ostringstream output; + if (pi.isEmptyOrNullUInt64()) + output << setw(30) << "Empty/Null" + << setw(30) << "Empty/Null"; + else + output << setw(30) << format(SimpleValueSInt64(pi.min), attr) + << setw(30) << format(SimpleValueSInt64(pi.max), attr); + return output.str(); +} + + + +string +TypeHandlerXDecimal::formatPartitionInfo128( + const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &pi) const +{ + ostringstream output; + if (pi.isEmptyOrNullSInt128()) + output << setw(datatypes::Decimal::MAXLENGTH16BYTES) << "Empty/Null" + << setw(datatypes::Decimal::MAXLENGTH16BYTES) << "Empty/Null"; + else + output << setw(datatypes::Decimal::MAXLENGTH16BYTES) << format(SimpleValueSInt128(pi.int128Min), attr) + << setw(datatypes::Decimal::MAXLENGTH16BYTES) << format(SimpleValueSInt128(pi.int128Max), attr); + return output.str(); +} + + +string +TypeHandlerStr::formatPartitionInfoSmallCharVarchar( + const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &pi) const +{ + ostringstream output; + int64_t maxLimit = numeric_limits::max(); + int64_t minLimit = numeric_limits::min(); + maxLimit = uint64ToStr(maxLimit); + minLimit = uint64ToStr(minLimit); + if (pi.min == maxLimit && pi.max == minLimit) + output << setw(30) << "Empty/Null" + << setw(30) << "Empty/Null"; + else + output << setw(30) << format(SimpleValueSInt64(pi.min), attr) + << setw(30) << format(SimpleValueSInt64(pi.max), attr); + return output.str(); +} + + +string +TypeHandlerChar::formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &pi) const +{ + // char column order swap for compare in subsequent loop + if (attr.colWidth <= 8) + return formatPartitionInfoSmallCharVarchar(attr, pi); + return formatPartitionInfoSInt64(attr, pi); +} + + +string +TypeHandlerVarchar::formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &pi) const +{ + // varchar column order swap for compare in subsequent loop + if (attr.colWidth <= 7) + return formatPartitionInfoSmallCharVarchar(attr, pi); + return formatPartitionInfoSInt64(attr, pi); +} + + +/****************************************************************************/ + +execplan::SimpleColumn * +TypeHandlerSInt8::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + if (ct.scale == 0) + return new execplan::SimpleColumn_INT<1>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + ct.colDataType = SystemCatalog::DECIMAL; + return new execplan::SimpleColumn_Decimal<1>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerSInt16::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + if (ct.scale == 0) + return new execplan::SimpleColumn_INT<2>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + ct.colDataType = SystemCatalog::DECIMAL; + return new execplan::SimpleColumn_Decimal<2>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerSInt24::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + if (ct.scale == 0) + return new execplan::SimpleColumn_INT<4>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + ct.colDataType = SystemCatalog::DECIMAL; + return new execplan::SimpleColumn_Decimal<4>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerSInt32::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + if (ct.scale == 0) + return new execplan::SimpleColumn_INT<4>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + ct.colDataType = SystemCatalog::DECIMAL; + return new execplan::SimpleColumn_Decimal<4>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerSInt64::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + if (ct.scale == 0) + return new execplan::SimpleColumn_INT<8>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + ct.colDataType = SystemCatalog::DECIMAL; + return new execplan::SimpleColumn_Decimal<8>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerUInt8::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + // QQ: why scale is not checked (unlike SInt1)? + return new execplan::SimpleColumn_UINT<1>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerUInt16::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + return new execplan::SimpleColumn_UINT<2>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerUInt24::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + return new execplan::SimpleColumn_UINT<4>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerUInt32::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + return new execplan::SimpleColumn_UINT<4>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerUInt64::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + return new execplan::SimpleColumn_UINT<8>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerReal::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + // QQ + return new execplan::SimpleColumn(name.db(), name.table(), name.column(), + prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerXDecimal::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + // QQ + return new execplan::SimpleColumn(name.db(), name.table(), name.column(), + prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerStr::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + // QQ + return new execplan::SimpleColumn(name.db(), name.table(), name.column(), + prm.columnStore(), prm.sessionid()); +} + + + +execplan::SimpleColumn * +TypeHandlerTemporal::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + // QQ + return new execplan::SimpleColumn(name.db(), name.table(), name.column(), + prm.columnStore(), prm.sessionid()); +} + + +/****************************************************************************/ + +class SimpleConverter: public boost::any +{ + bool &initPushWarning() + { + m_pushWarning= false; + return m_pushWarning; + } + bool m_pushWarning; +public: + SimpleConverter(const SessionParam &sp, + const TypeHandler *h, + const SystemCatalog::TypeAttributesStd &attr, + const char *str) + :boost::any(h->convertFromString(attr, + ConvertFromStringParam(sp.tzname(), true, false), + str, initPushWarning())) + { } + round_style_t roundStyle() const + { + return m_pushWarning ? round_style_t::POS : round_style_t::NONE; + } + round_style_t roundStyle(const char *str) const + { + return m_pushWarning ? roundStyleDetect(str) : round_style_t::NONE; + } + static round_style_t roundStyleDetect(const char *str) + { + // get rid of leading white spaces and parentheses + string data(str); + size_t fpos = data.find_first_of(" \t()"); + while (string::npos != fpos) + { + data.erase(fpos, 1); + fpos = data.find_first_of(" \t()"); + } + return (data[0] == '-') ? round_style_t::NEG : round_style_t::POS; + } + int64_t to_sint64() const { return boost::any_cast(*this); } + uint64_t to_uint64() const { return boost::any_cast(*this); } + uint32_t to_uint32() const { return boost::any_cast(*this); } + int128_t to_sint128() const { return boost::any_cast(*this); } +}; + + +class SimpleConverterSNumeric: public SimpleConverter +{ +public: + SimpleConverterSNumeric(const SessionParam &sp, + const TypeHandler *h, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, + round_style_t &rf) + :SimpleConverter(sp, h, attr, str) + { + rf = roundStyle(str); + } +}; + + +template +SimpleValue +toSimpleValueSInt(const SessionParam &sp, + const TypeHandler *h, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) +{ + idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); + SimpleConverterSNumeric anyVal(sp, h, attr, str, rf); + return SimpleValueSInt64(static_cast(boost::any_cast(anyVal))); +} + + +SimpleValue +TypeHandlerSInt8::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + return toSimpleValueSInt(sp, this, attr, str, rf); +} + + +SimpleValue +TypeHandlerSInt16::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + return toSimpleValueSInt(sp, this, attr, str, rf); +} + + +SimpleValue +TypeHandlerSInt24::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + return toSimpleValueSInt(sp, this, attr, str, rf); +} + + +SimpleValue +TypeHandlerSInt32::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + return toSimpleValueSInt(sp, this, attr, str, rf); +} + + +SimpleValue +TypeHandlerSInt64::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + return toSimpleValueSInt(sp, this, attr, str, rf); +} + + +template +SimpleValue toSimpleValueUInt(const SessionParam &sp, + const TypeHandler *h, + const SystemCatalog::TypeAttributesStd &attr, + const char *str) +{ + idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); + SimpleConverter anyVal(sp, h, attr, str); + return SimpleValueSInt64(static_cast(boost::any_cast(anyVal))); +} + + +SimpleValue +TypeHandlerUInt8::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + return toSimpleValueUInt(sp, this, attr, str); +} + + +SimpleValue +TypeHandlerUInt16::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + return toSimpleValueUInt(sp, this, attr, str); +} + + +SimpleValue +TypeHandlerUInt24::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + return toSimpleValueUInt(sp, this, attr, str); +} + + +SimpleValue +TypeHandlerUInt32::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + return toSimpleValueUInt(sp, this, attr, str); +} + + +SimpleValue +TypeHandlerUInt64::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + return toSimpleValueUInt(sp, this, attr, str); +} + + +SimpleValue +TypeHandlerDate::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); + SimpleConverter anyVal(sp, this, attr, str); + return SimpleValueSInt64(static_cast(anyVal.to_uint32())); +} + + +SimpleValue +TypeHandlerDatetime::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); + SimpleConverter anyVal(sp, this, attr, str); + return SimpleValueSInt64(static_cast(anyVal.to_uint64())); +} + + +SimpleValue +TypeHandlerTimestamp::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); + SimpleConverter anyVal(sp, this, attr, str); + return SimpleValueTimestamp(anyVal.to_uint64(), sp.tzname()); +} + + +SimpleValue +TypeHandlerTime::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); + SimpleConverter anyVal(sp, this, attr, str); + return SimpleValueSInt64(anyVal.to_sint64()); +} + + +SimpleValue +TypeHandlerXDecimal::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + if (attr.colWidth <= SystemCatalog::EIGHT_BYTE) + { + SimpleConverterSNumeric anyVal(sp, this, attr, str, rf); + int64_t v; + if (attr.colWidth == SystemCatalog::ONE_BYTE) + v = boost::any_cast(anyVal); + else if (attr.colWidth == SystemCatalog::TWO_BYTE) + v = boost::any_cast(anyVal); + else if (attr.colWidth == SystemCatalog::FOUR_BYTE) + v = boost::any_cast(anyVal); + else if (attr.colWidth == SystemCatalog::EIGHT_BYTE) + v = anyVal.to_sint64(); + else + { + idbassert(0); + v = 0; + } + return SimpleValueSInt64(v); + } + else + { + idbassert(attr.colWidth == datatypes::MAXDECIMALWIDTH); + SimpleConverterSNumeric anyVal(sp, this, attr, str, rf); + return SimpleValueSInt128(anyVal.to_sint128()); + } +} + + +SimpleValue +TypeHandlerStr::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + SimpleConverter anyVal(sp, this, attr, str); + rf= anyVal.roundStyle(); + string i = boost::any_cast(anyVal); + // bug 1932, pad nulls up to the size of v + i.resize(sizeof(int64_t), 0); + return SimpleValueSInt64(static_cast(uint64ToStr(*((uint64_t*) i.data())))); +} + + +/****************************************************************************/ + +MinMaxPartitionInfo::MinMaxPartitionInfo(const BRM::EMEntry &entry) + :m_status(entry.status == BRM::EXTENTOUTOFSERVICE ? ET_DISABLED : EXPL_NULL) +{ } + + +MinMaxPartitionInfo +TypeHandler::getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const +{ + int32_t seqNum; + MinMaxPartitionInfo partInfo(entry); + *state = em.getExtentMaxMin(entry.range.start, partInfo.max, partInfo.min, seqNum); + return partInfo; +} + + +MinMaxPartitionInfo +TypeHandlerXDecimal::getExtentPartitionInfo64(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const +{ + int32_t seqNum; + MinMaxPartitionInfo partInfo(entry); + *state = em.getExtentMaxMin(entry.range.start, partInfo.max, partInfo.min, seqNum); + return partInfo; +} + + +MinMaxPartitionInfo +TypeHandlerXDecimal::getExtentPartitionInfo128(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const +{ + int32_t seqNum; + MinMaxPartitionInfo partInfo(entry); + *state = em.getExtentMaxMin(entry.range.start, partInfo.int128Max, partInfo.int128Min, seqNum); + return partInfo; +} + + +MinMaxPartitionInfo +TypeHandlerChar::getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const +{ + int32_t seqNum; + MinMaxPartitionInfo partInfo(entry); + *state = em.getExtentMaxMin(entry.range.start, partInfo.max, partInfo.min, seqNum); + // char column order swap + if (attr.colWidth <= 8) + { + partInfo.max = uint64ToStr(partInfo.max); + partInfo.min = uint64ToStr(partInfo.min); + } + return partInfo; +} + + +MinMaxPartitionInfo +TypeHandlerVarchar::getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const +{ + int32_t seqNum; + MinMaxPartitionInfo partInfo(entry); + *state = em.getExtentMaxMin(entry.range.start, partInfo.max, partInfo.min, seqNum); + // char column order swap + if (attr.colWidth <= 7) + { + partInfo.max = uint64ToStr(partInfo.max); + partInfo.min = uint64ToStr(partInfo.min); + } + return partInfo; +} + + +/****************************************************************************/ + + +string TypeHandler::PrintPartitionValueSInt64( + const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const +{ + if (!partInfo.isSuitableSInt64(startVal, rfMin, endVal, rfMax)) + return ""; + + ostringstream oss; + if (partInfo.min > partInfo.max) + oss << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; + else + oss << setw(30) << format(SimpleValueSInt64(partInfo.min), attr) + << setw(30) << format(SimpleValueSInt64(partInfo.max), attr); + return oss.str(); +} + + +string TypeHandler::PrintPartitionValueUInt64( + const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const +{ + if (!partInfo.isSuitableUInt64(startVal, rfMin, endVal, rfMax)) + return ""; + + ostringstream oss; + if (static_cast(partInfo.min) > static_cast(partInfo.max)) + oss << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; + else + oss << setw(30) << format(SimpleValueSInt64(partInfo.min), attr) + << setw(30) << format(SimpleValueSInt64(partInfo.max), attr); + return oss.str(); +} + + +string TypeHandlerXDecimal::PrintPartitionValue128( + const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const +{ + if (!partInfo.isSuitableSInt128(startVal, rfMin, endVal, rfMax)) + return ""; + + ostringstream oss; + if (partInfo.int128Min > partInfo.int128Max) + oss << setw(datatypes::Decimal::MAXLENGTH16BYTES) << "Empty/Null" + << setw(datatypes::Decimal::MAXLENGTH16BYTES) << "Empty/Null"; + else + oss << setw(datatypes::Decimal::MAXLENGTH16BYTES) << format(SimpleValueSInt128(partInfo.int128Min), attr) + << setw(datatypes::Decimal::MAXLENGTH16BYTES) << format(SimpleValueSInt128(partInfo.int128Max), attr); + return oss.str(); +} + +/****************************************************************************/ + + +boost::any +TypeHandlerSInt8::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + char tinyintvalue = joblist::TINYINTNULL; + boost::any value = tinyintvalue; + return value; +} + + +boost::any +TypeHandlerUInt8::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint8_t utinyintvalue = joblist::UTINYINTNULL; + boost::any value = utinyintvalue; + return value; +} + + +boost::any +TypeHandlerSInt16::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + short smallintvalue = joblist::SMALLINTNULL; + boost::any value = smallintvalue; + return value; +} + + +boost::any +TypeHandlerUInt16::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint16_t usmallintvalue = joblist::USMALLINTNULL; + boost::any value = usmallintvalue; + return value; +} + + +boost::any +TypeHandlerSInt24::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + int intvalue = joblist::INTNULL; + boost::any value = intvalue; + return value; +} + + +boost::any +TypeHandlerSInt32::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + int intvalue = joblist::INTNULL; + boost::any value = intvalue; + return value; +} + + +boost::any +TypeHandlerUInt24::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint32_t uintvalue = joblist::UINTNULL; + boost::any value = uintvalue; + return value; +} + + +boost::any +TypeHandlerUInt32::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint32_t uintvalue = joblist::UINTNULL; + boost::any value = uintvalue; + return value; +} + + +boost::any +TypeHandlerSInt64::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + long long bigint = joblist::BIGINTNULL; + boost::any value = bigint; + return value; +} + + +boost::any +TypeHandlerUInt64::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint64_t ubigint = joblist::UBIGINTNULL; + boost::any value = ubigint; + return value; +} + + +boost::any +TypeHandlerXDecimal::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + if (LIKELY(attr.colWidth == 16)) + { + int128_t val; + datatypes::Decimal::setWideDecimalNullValue(val); + boost::any value = val; + return value; + } + if (attr.colWidth == SystemCatalog::EIGHT_BYTE) + { + long long eightbyte = joblist::BIGINTNULL; + boost::any value = eightbyte; + return value; + } + if (attr.colWidth == SystemCatalog::FOUR_BYTE) + { + int intvalue = joblist::INTNULL; + boost::any value = intvalue; + return value; + } + if (attr.colWidth == SystemCatalog::TWO_BYTE) + { + short smallintvalue = joblist::SMALLINTNULL; + boost::any value = smallintvalue; + return value; + } + if (attr.colWidth == SystemCatalog::ONE_BYTE) + { + char tinyintvalue = joblist::TINYINTNULL; + boost::any value = tinyintvalue; + return value; + } + WriteEngine::Token nullToken; + boost::any value = nullToken; + return value; +} + + +boost::any +TypeHandlerSFloat::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint32_t jlfloatnull = joblist::FLOATNULL; + float* fp = reinterpret_cast(&jlfloatnull); + boost::any value = *fp; + return value; +} + + +boost::any +TypeHandlerUFloat::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint32_t jlfloatnull = joblist::FLOATNULL; + float* fp = reinterpret_cast(&jlfloatnull); + boost::any value = *fp; + return value; +} + + +boost::any +TypeHandlerSDouble::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint64_t jldoublenull = joblist::DOUBLENULL; + double* dp = reinterpret_cast(&jldoublenull); + boost::any value = *dp; + return value; +} + + +boost::any +TypeHandlerUDouble::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint64_t jldoublenull = joblist::DOUBLENULL; + double* dp = reinterpret_cast(&jldoublenull); + boost::any value = *dp; + return value; +} + + +boost::any +TypeHandlerDate::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint32_t d = joblist::DATENULL; + boost::any value = d; + return value; +} + + +boost::any +TypeHandlerDatetime::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint64_t d = joblist::DATETIMENULL; + boost::any value = d; + return value; +} + + +boost::any +TypeHandlerTime::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + int64_t d = joblist::TIMENULL; + boost::any value = d; + return value; +} + + +boost::any +TypeHandlerTimestamp::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint64_t d = joblist::TIMESTAMPNULL; + boost::any value = d; + return value; +} + + +boost::any +TypeHandlerChar::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + switch (attr.colWidth) + { + case 1: + { + //charnull = joblist::CHAR1NULL; + std::string charnull = "\376"; + boost::any value = charnull; + return value; + } + case 2: + { + //charnull = joblist::CHAR2NULL; + std::string charnull = "\377\376"; + boost::any value = charnull; + return value; + } + case 3: + case 4: + { + //charnull = joblist::CHAR4NULL; + std::string charnull = "\377\377\377\376"; + boost::any value = charnull; + return value; + } + case 5: + case 6: + case 7: + case 8: + { + //charnull = joblist::CHAR8NULL; + std::string charnull = "\377\377\377\377\377\377\377\376"; + boost::any value = charnull; + return value; + } + } + WriteEngine::Token nullToken; + boost::any value = nullToken; + return value; +} + + +boost::any +TypeHandlerStr::getNullValueForTypeVarcharText(const SystemCatalog::TypeAttributesStd &attr) const +{ + switch (attr.colWidth) + { + case 1: + { + //charnull = joblist::CHAR2NULL; + std::string charnull = "\377\376"; + boost::any value = charnull; + return value; + } + case 2: + case 3: + { + //charnull = joblist::CHAR4NULL; + std::string charnull = "\377\377\377\376"; + boost::any value = charnull; + return value; + } + case 4: + case 5: + case 6: + case 7: + { + //charnull = joblist::CHAR8NULL; + std::string charnull = "\377\377\377\377\377\377\377\376"; + boost::any value = charnull; + return value; + } + } + WriteEngine::Token nullToken; + boost::any value = nullToken; + return value; +} + + +boost::any +TypeHandlerBlob::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + WriteEngine::Token nullToken; + boost::any value = nullToken; + return value; +} + + +boost::any +TypeHandlerVarbinary::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + WriteEngine::Token nullToken; + boost::any value = nullToken; + return value; +} + +/****************************************************************************/ + +boost::any +TypeHandlerBit::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToBit(colType, prm, data, pushWarning); +} + + +boost::any +TypeHandlerSInt8::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + int64_t val64; + dataconvert::number_int_value(data, SystemCatalog::TINYINT, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (char) val64; + return value; +} + + +boost::any +TypeHandlerSInt16::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + int64_t val64; + dataconvert::number_int_value(data, SystemCatalog::SMALLINT, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (short) val64; + return value; +} + + +boost::any +TypeHandlerSInt24::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + int64_t val64; + dataconvert::number_int_value(data, SystemCatalog::MEDINT, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (int) val64; + return value; +} + + +boost::any +TypeHandlerSInt32::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + int64_t val64; + dataconvert::number_int_value(data, SystemCatalog::INT, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (int) val64; + return value; +} + + +boost::any +TypeHandlerSInt64::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + int64_t val64; + dataconvert::number_int_value(data, SystemCatalog::BIGINT, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (long long) val64; + return value; +} + + +boost::any +TypeHandlerUInt8::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const + +{ + boost::any value = (uint8_t)dataconvert::number_uint_value(data, SystemCatalog::UTINYINT, colType, pushWarning, prm.noRoundup()); + return value; +} + + +boost::any +TypeHandlerUInt16::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + boost::any value = (uint16_t)dataconvert::number_uint_value(data, SystemCatalog::USMALLINT, colType, pushWarning, prm.noRoundup()); + return value; +} + + +boost::any +TypeHandlerUInt24::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + boost::any value = (uint32_t)dataconvert::number_uint_value(data, SystemCatalog::UMEDINT, colType, pushWarning, prm.noRoundup()); + return value; +} + + +boost::any +TypeHandlerUInt32::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + boost::any value = (uint32_t)dataconvert::number_uint_value(data, SystemCatalog::UINT, colType, pushWarning, prm.noRoundup()); + return value; +} + + +boost::any +TypeHandlerUInt64::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + boost::any value = (uint64_t)dataconvert::number_uint_value(data, SystemCatalog::UBIGINT, colType, pushWarning, prm.noRoundup()); + return value; +} + + +boost::any +TypeHandlerSFloat::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToFloat(SystemCatalog::FLOAT, data, pushWarning); +} + + +boost::any +TypeHandlerUFloat::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToFloat(SystemCatalog::UFLOAT, data, pushWarning); +} + + +boost::any +TypeHandlerSDouble::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToDouble(SystemCatalog::DOUBLE, data, pushWarning); +} + + +boost::any +TypeHandlerUDouble::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToDouble(SystemCatalog::UDOUBLE, data, pushWarning); +} + + +boost::any +TypeHandlerDate::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToDate(data, pushWarning); +} + + +boost::any +TypeHandlerDatetime::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToDatetime(data, pushWarning); +} + + +boost::any +TypeHandlerTime::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToTime(colType, data, pushWarning); +} + +boost::any +TypeHandlerTimestamp::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToTimestamp(prm, data, pushWarning); +} + + +boost::any +TypeHandlerChar::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToString(colType, data, pushWarning); +} + + +boost::any +TypeHandlerVarchar::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToString(colType, data, pushWarning); +} + + +boost::any +TypeHandlerText::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToString(colType, data, pushWarning); +} + + +boost::any +TypeHandlerClob::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + boost::any value = data; + return value; +} + + +boost::any +TypeHandlerBlob::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + boost::any value = data; + return value; +} + + +boost::any +TypeHandlerVarbinary::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + boost::any value = data; + return value; +} + + +boost::any +TypeHandlerSDecimal64::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToSDecimal(colType, prm, data, pushWarning); +} + + +boost::any +TypeHandlerUDecimal64::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToUDecimal(colType, prm, data, pushWarning); +} + + +boost::any +TypeHandlerSDecimal128::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToSDecimal(colType, prm, data, pushWarning); +} + + +boost::any +TypeHandlerUDecimal128::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToUDecimal(colType, prm, data, pushWarning); +} + + +/****************************************************************************/ + +} // end of namespace datatypes + + +// vim:ts=2 sw=2: diff --git a/datatypes/mcs_datatype.h b/datatypes/mcs_datatype.h new file mode 100644 index 000000000..470950960 --- /dev/null +++ b/datatypes/mcs_datatype.h @@ -0,0 +1,2653 @@ +/* + Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ +#ifndef MCS_DATATYPE_H_INCLUDED +#define MCS_DATATYPE_H_INCLUDED + +#include +#include +#include +#include "exceptclasses.h" +#include "mcs_decimal.h" + + +#ifdef _MSC_VER +typedef int mcs_sint32_t; +#else +typedef int32_t mcs_sint32_t; +#endif + + +// Because including my_sys.h in a Columnstore header causes too many conflicts +struct charset_info_st; +typedef const struct charset_info_st CHARSET_INFO; + + +#ifdef _MSC_VER +#define __attribute__(x) +#endif + +namespace +{ +const int64_t MIN_TINYINT __attribute__ ((unused)) = std::numeric_limits::min() + 2; // -126; +const int64_t MAX_TINYINT __attribute__ ((unused)) = std::numeric_limits::max(); // 127; +const int64_t MIN_SMALLINT __attribute__ ((unused)) = std::numeric_limits::min() + 2; // -32766; +const int64_t MAX_SMALLINT __attribute__ ((unused)) = std::numeric_limits::max(); // 32767; +const int64_t MIN_MEDINT __attribute__ ((unused)) = -(1ULL << 23); // -8388608; +const int64_t MAX_MEDINT __attribute__ ((unused)) = (1ULL << 23) - 1; // 8388607; +const int64_t MIN_INT __attribute__ ((unused)) = std::numeric_limits::min() + 2; // -2147483646; +const int64_t MAX_INT __attribute__ ((unused)) = std::numeric_limits::max(); // 2147483647; +const int64_t MIN_BIGINT __attribute__ ((unused)) = std::numeric_limits::min() + 2; // -9223372036854775806LL; +const int64_t MAX_BIGINT __attribute__ ((unused)) = std::numeric_limits::max(); // 9223372036854775807 + +const uint64_t MIN_UINT __attribute__ ((unused)) = 0; +const uint64_t MIN_UTINYINT __attribute__ ((unused)) = 0; +const uint64_t MIN_USMALLINT __attribute__ ((unused)) = 0; +const uint64_t MIN_UMEDINT __attribute__ ((unused)) = 0; +const uint64_t MIN_UBIGINT __attribute__ ((unused)) = 0; +const uint64_t MAX_UINT __attribute__ ((unused)) = std::numeric_limits::max() - 2; // 4294967293 +const uint64_t MAX_UTINYINT __attribute__ ((unused)) = std::numeric_limits::max() - 2; // 253; +const uint64_t MAX_USMALLINT __attribute__ ((unused)) = std::numeric_limits::max() - 2; // 65533; +const uint64_t MAX_UMEDINT __attribute__ ((unused)) = (1ULL << 24) - 1; // 16777215 +const uint64_t MAX_UBIGINT __attribute__ ((unused)) = std::numeric_limits::max() - 2; // 18446744073709551613 + +const float MAX_FLOAT __attribute__ ((unused)) = std::numeric_limits::max(); // 3.402823466385289e+38 +const float MIN_FLOAT __attribute__ ((unused)) = -std::numeric_limits::max(); +const double MAX_DOUBLE __attribute__ ((unused)) = std::numeric_limits::max(); // 1.7976931348623157e+308 +const double MIN_DOUBLE __attribute__ ((unused)) = -std::numeric_limits::max(); +const long double MAX_LONGDOUBLE __attribute__ ((unused)) = std::numeric_limits::max(); // 1.7976931348623157e+308 +const long double MIN_LONGDOUBLE __attribute__ ((unused)) = -std::numeric_limits::max(); + +const uint64_t AUTOINCR_SATURATED __attribute__ ((unused)) = std::numeric_limits::max(); +} + + +using namespace std; // e.g. string + + +namespace ddlpackage +{ + struct ColumnType; +}; + + +namespace BRM +{ + struct EMEntry; + class DBRM; +}; + + +namespace rowgroup +{ + struct Row; +}; + + +namespace execplan +{ + struct SimpleColumn; +}; + + + +namespace datatypes +{ + +class SystemCatalog +{ +public: + + /** the set of Calpont column widths + * + */ + enum ColWidth { ONE_BIT, ONE_BYTE, TWO_BYTE, THREE_BYTE, FOUR_BYTE, FIVE_BYTE, SIX_BYTE, SEVEN_BYTE, EIGHT_BYTE }; + + + /** the set of Calpont column data types + * + */ + enum ColDataType + { + BIT, /*!< BIT type */ + TINYINT, /*!< TINYINT type */ + CHAR, /*!< CHAR type */ + SMALLINT, /*!< SMALLINT type */ + DECIMAL, /*!< DECIMAL type */ + MEDINT, /*!< MEDINT type */ + INT, /*!< INT type */ + FLOAT, /*!< FLOAT type */ + DATE, /*!< DATE type */ + BIGINT, /*!< BIGINT type */ + DOUBLE, /*!< DOUBLE type */ + DATETIME, /*!< DATETIME type */ + VARCHAR, /*!< VARCHAR type */ + VARBINARY, /*!< VARBINARY type */ + CLOB, /*!< CLOB type */ + BLOB, /*!< BLOB type */ + UTINYINT, /*!< Unsigned TINYINT type */ + USMALLINT, /*!< Unsigned SMALLINT type */ + UDECIMAL, /*!< Unsigned DECIMAL type */ + UMEDINT, /*!< Unsigned MEDINT type */ + UINT, /*!< Unsigned INT type */ + UFLOAT, /*!< Unsigned FLOAT type */ + UBIGINT, /*!< Unsigned BIGINT type */ + UDOUBLE, /*!< Unsigned DOUBLE type */ + TEXT, /*!< TEXT type */ + TIME, /*!< TIME type */ + TIMESTAMP, /*!< TIMESTAMP type */ + NUM_OF_COL_DATA_TYPE, /* NEW TYPES ABOVE HERE */ + LONGDOUBLE, /* @bug3241, dev and variance calculation only */ + STRINT, /* @bug3532, string as int for fast comparison */ + UNDEFINED, /*!< Undefined - used in UDAF API */ + BINARY, /*!< BINARY type */ + }; + + class TypeAttributesStd + { + public: + int32_t colWidth; + int32_t scale; //number after decimal points + int32_t precision; + TypeAttributesStd() + :colWidth(0), + scale(0), + precision(-1) + {} + /** + @brief Convenience method to get int128 from a std::string. + */ + int128_t decimal128FromString(const std::string& value) const; + + /** + @brief The method sets the legacy scale and precision of a wide decimal + column which is the result of an arithmetic operation. + */ + inline void setDecimalScalePrecisionLegacy(unsigned int p, unsigned int s) + { + scale = s; + + if (s == 0) + precision = p - 1; + else + precision = p - s; + } + + /** + @brief The method sets the scale and precision of a wide decimal + column which is the result of an arithmetic operation. + */ + inline void setDecimalScalePrecision(unsigned int p, unsigned int s) + { + colWidth = (p > INT64MAXPRECISION) ? + MAXDECIMALWIDTH : MAXLEGACYWIDTH; + + precision = (p > INT128MAXPRECISION) ? + INT128MAXPRECISION : p; + + scale = s; + } + + /** + @brief The method sets the scale and precision of a wide decimal + column which is the result of an arithmetic operation, based on a heuristic. + */ + inline void setDecimalScalePrecisionHeuristic(unsigned int p, unsigned int s) + { + unsigned int diff = 0; + + if (p > INT128MAXPRECISION) + { + precision = INT128MAXPRECISION; + diff = p - INT128MAXPRECISION; + } + else + { + precision = p; + } + + scale = s; + + if (diff != 0) + { + scale = s - (int)(diff * (38.0/65.0)); + + if (scale < 0) + scale = 0; + } + } + + }; + + class TypeHolderStd: public TypeAttributesStd + { + public: + ColDataType colDataType; + TypeHolderStd() + :colDataType(MEDINT) + { } + const class TypeHandler *typeHandler() const; + boost::any getNullValueForType() const; + + /** + @brief The method detects whether decimal type is wide + using csc colType. + */ + inline bool isWideDecimalType() const + { + return (colDataType == DECIMAL || + colDataType == UDECIMAL) && + colWidth == MAXDECIMALWIDTH; + } + }; + +}; + + +/** + @brief The method detects whether decimal type is wide + using datatype and width. +*/ +static inline bool isWideDecimalType(const datatypes::SystemCatalog::ColDataType &dt, + const int32_t width) +{ + return width == MAXDECIMALWIDTH && + (dt == SystemCatalog::DECIMAL || + dt == SystemCatalog::UDECIMAL); +} + + +/** convenience function to determine if column type is a char + * type + */ +inline bool isCharType(const datatypes::SystemCatalog::ColDataType type) +{ + return (datatypes::SystemCatalog::VARCHAR == type || + datatypes::SystemCatalog::CHAR == type || + datatypes::SystemCatalog::BLOB == type || + datatypes::SystemCatalog::TEXT == type); +} + +/** convenience function to determine if column type is a + * numeric type + */ +inline bool isNumeric(const datatypes::SystemCatalog::ColDataType type) +{ + switch (type) + { + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + case datatypes::SystemCatalog::UDECIMAL: + return true; + + default: + return false; + } +} + +inline bool isDecimal(const datatypes::SystemCatalog::ColDataType type) +{ + return (type == datatypes::SystemCatalog::DECIMAL || + type == datatypes::SystemCatalog::UDECIMAL); +} + +/** convenience function to determine if column type is an + * unsigned type + */ +inline bool isUnsigned(const datatypes::SystemCatalog::ColDataType type) +{ + switch (type) + { + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + return true; + + default: + return false; + } +} + +inline bool isSignedInteger(const datatypes::SystemCatalog::ColDataType type) +{ + switch (type) + { + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + return true; + + default: + return false; + } +} + + +/** + @brief Returns true if all arguments have a DECIMAL/UDECIMAL type +*/ +static inline bool isDecimalOperands(const SystemCatalog::ColDataType resultDataType, + const SystemCatalog::ColDataType leftColDataType, + const SystemCatalog::ColDataType rightColDataType) +{ + return ((resultDataType == SystemCatalog::DECIMAL || + resultDataType == SystemCatalog::UDECIMAL) && + (leftColDataType == SystemCatalog::DECIMAL || + leftColDataType == SystemCatalog::UDECIMAL) && + (rightColDataType == SystemCatalog::DECIMAL || + rightColDataType == SystemCatalog::UDECIMAL)); +} + +} // end of namespace datatypes + + + + +namespace datatypes +{ + +static constexpr int128_t minInt128 = int128_t(0x8000000000000000LL) << 64; +static constexpr int128_t maxInt128 = (int128_t(0x7FFFFFFFFFFFFFFFLL) << 64) + 0xFFFFFFFFFFFFFFFFLL; + +class ConstString +{ + const char *m_str; + size_t m_length; +public: + ConstString(const char *str, size_t length) + :m_str(str), m_length(length) + { } + const char *str() const { return m_str; } + const char *end() const { return m_str + m_length; } + size_t length() const { return m_length; } + void bin2hex(char *o) + { + static const char hexdig[] = { '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}; + const char *e= end(); + for (const char *s= m_str; s < e; s++) + { + *o++ = hexdig[*s >> 4]; + *o++ = hexdig[*s & 0xf]; + } + } +}; + + +enum class round_style_t: uint8_t +{ + NONE= 0, + POS= 0x01, + NEG= 0x80 +}; + + +class SessionParam +{ + const char *m_tzname; +public: + SessionParam(const char *tzname) + :m_tzname(tzname) + { } + const char *tzname() const + { + return m_tzname; + } +}; + + +class ConvertFromStringParam +{ + const std::string& m_timeZone; + const bool m_noRoundup; + const bool m_isUpdate; +public: + ConvertFromStringParam(const std::string& timeZone, + bool noRoundup, bool isUpdate) + :m_timeZone(timeZone), + m_noRoundup(noRoundup), + m_isUpdate(isUpdate) + { } + const std::string& timeZone() const { return m_timeZone; } + bool noRoundup() const { return m_noRoundup; } + bool isUpdate() const { return m_isUpdate; } +}; + + +class SimpleValue +{ + int64_t m_sint64; + int128_t m_sint128; + const char *m_tzname; +public: + SimpleValue(const int64_t sint64, + const int128_t &sint128, + const char *tzname) + :m_sint64(sint64), m_sint128(sint128), m_tzname(tzname) + { } + SimpleValue() + :m_sint64(0), m_sint128(0), m_tzname(0) + { } + int64_t toSInt64() const { return m_sint64; } + uint64_t toUInt64() const { return static_cast(m_sint64); } + int128_t toSInt128() const { return m_sint128; } + const char *tzname() const { return m_tzname; } +}; + + +class SimpleValueSInt64: public SimpleValue +{ +public: + SimpleValueSInt64(int64_t value) + :SimpleValue(value, 0, NULL) + { } +}; + + +class SimpleValueUInt64: public SimpleValue +{ +public: + SimpleValueUInt64(uint64_t value) + :SimpleValue(static_cast(value), 0, NULL) + { } +}; + + +class SimpleValueSInt128: public SimpleValue +{ +public: + SimpleValueSInt128(int128_t value) + :SimpleValue(0, value, NULL) + { } +}; + + +class SimpleValueTimestamp: public SimpleValue +{ +public: + SimpleValueTimestamp(uint64_t value, const char *tzname) + :SimpleValue(static_cast(value), 0, tzname) + { } +}; + + +class MinMaxInfo +{ +public: + int64_t min; + int64_t max; + union + { + int128_t int128Min; + int64_t min_; + }; + union + { + int128_t int128Max; + int64_t max_; + }; + MinMaxInfo() + :min((uint64_t)0x8000000000000001ULL), + max((uint64_t) - 0x8000000000000001LL) + { + int128Min = datatypes::minInt128; + int128Max = datatypes::maxInt128; + }; + bool isEmptyOrNullSInt64() const + { + return min == std::numeric_limits::max() && + max == std::numeric_limits::min(); + } + bool isEmptyOrNullUInt64() const + { + return static_cast(min) == std::numeric_limits::max() && + static_cast(max) == std::numeric_limits::min(); + } + bool isEmptyOrNullSInt128() const + { + return int128Min == datatypes::maxInt128 && + int128Max == datatypes::minInt128; + } + void widenSInt64(const MinMaxInfo &partInfo) + { + min = partInfo.min < min ? partInfo.min : min; + max = partInfo.max > max ? partInfo.max : max; + } + void widenUInt64(const MinMaxInfo &partInfo) + { + min = static_cast(static_cast(partInfo.min) < static_cast(min) ? partInfo.min : min); + max = static_cast(static_cast(partInfo.max) > static_cast(max) ? partInfo.max : max); + } + void widenSInt128(const MinMaxInfo &partInfo) + { + int128Min = partInfo.int128Min < int128Min ? partInfo.int128Min : int128Min; + int128Max = partInfo.int128Max > int128Max ? partInfo.int128Max : int128Max; + } + static MinMaxInfo widenSInt64(const MinMaxInfo &a, + const MinMaxInfo &b) + { + MinMaxInfo tmp= a; + tmp.widenSInt64(b); + return tmp; + } + static MinMaxInfo widenUInt64(const MinMaxInfo &a, + const MinMaxInfo &b) + { + MinMaxInfo tmp(a); + tmp.widenUInt64(b); + return tmp; + } + static MinMaxInfo widenSInt128(const MinMaxInfo &a, + const MinMaxInfo &b) + { + MinMaxInfo tmp= a; + tmp.widenSInt128(b); + return tmp; + } +}; + + +class MinMaxPartitionInfo: public MinMaxInfo +{ + enum status_flag_t : uint64_t + { + EXPL_NULL = 0x0000, + ET_DISABLED = 0x0002, + CPINVALID = 0x0004 + }; + uint64_t m_status; +public: + MinMaxPartitionInfo() + :m_status(0) + { }; + MinMaxPartitionInfo(const BRM::EMEntry &entry); + void set_invalid() { m_status|= CPINVALID; } + bool is_invalid() const { return m_status & CPINVALID; } + bool is_disabled() const { return m_status & ET_DISABLED; } + + bool isSuitableSInt64(const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const + { + if (min >= startVal.toSInt64() && + max <= endVal.toSInt64() && + !(min == std::numeric_limits::max() && + max == std::numeric_limits::min())) + { + if (rfMin == round_style_t::POS && min == startVal.toSInt64()) + return false; + + if (rfMax == round_style_t::NEG && max == endVal.toSInt64()) + return false; + + return true; + } + return false; + } + bool isSuitableUInt64(const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const + { + if (static_cast(min) >= startVal.toUInt64() && + static_cast(max) <= endVal.toUInt64() && + !(static_cast(min) == std::numeric_limits::max() && + max == 0)) + { + if (rfMin == round_style_t::POS && min == startVal.toSInt64()) + return false; + + if (rfMax == round_style_t::NEG && max == endVal.toSInt64()) + return false; + return true; + } + return false; + } + bool isSuitableSInt128(const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const + { + if (int128Min >= startVal.toSInt128() && + int128Max <= endVal.toSInt128() && + !(int128Min == datatypes::maxInt128 && + int128Max == datatypes::minInt128)) + { + if (rfMin == round_style_t::POS && int128Min == startVal.toSInt128()) + return false; + + if (rfMax == round_style_t::NEG && int128Max == endVal.toSInt128()) + return false; + + return true; + } + return false; + } +}; + + +class ColBatchWriter +{ + FILE *m_filePtr; + char m_delimiter; + char m_enclosed_by; +public: + ColBatchWriter(FILE *f, + char delimiter, + char enclosed_by) + :m_filePtr(f), + m_delimiter(delimiter), + m_enclosed_by(enclosed_by) + { } + FILE *filePtr() const { return m_filePtr; } + char delimiter() const { return m_delimiter; } + char enclosed_by() const { return m_enclosed_by; } +}; + + +class SimpleColumnParam +{ + uint32_t m_sessionid; + bool m_columnStore; +public: + SimpleColumnParam(uint32_t sessionid, bool columnStore) + :m_sessionid(sessionid), + m_columnStore(columnStore) + { } + uint32_t sessionid() const { return m_sessionid; } + bool columnStore() const { return m_columnStore; } + void columnStore(bool value) { m_columnStore= value; } +}; + + +class DatabaseQualifiedColumnName +{ + std::string m_db; + std::string m_table; + std::string m_column; +public: + DatabaseQualifiedColumnName(const std::string &db, + const std::string &table, + const std::string &column) + :m_db(db), + m_table(table), + m_column(column) + { } + const std::string &db() const { return m_db; } + const std::string &table() const { return m_table; } + const std::string &column() const { return m_column; } +}; + + +class StoreField +{ +public: + virtual ~StoreField() {} + virtual int32_t colWidth() const = 0; + virtual int32_t precision() const = 0; + virtual int32_t scale() const = 0; + virtual int store_date(int64_t val) = 0; + virtual int store_datetime(int64_t val) = 0; + virtual int store_time(int64_t val) = 0; + virtual int store_timestamp(int64_t val) = 0; + virtual int store_string(const char *str, size_t length) = 0; + virtual int store_varbinary(const char *str, size_t length) = 0; + virtual int store_xlonglong(int64_t val) = 0; + virtual int store_float(float val) = 0; + virtual int store_double(double val) = 0; + virtual int store_long_double(long double val) = 0; + virtual int store_decimal64(const datatypes::VDecimal& dec) = 0; + virtual int store_decimal128(const datatypes::VDecimal& dec) = 0; + virtual int store_lob(const char *str, size_t length) = 0; +}; + + +class WriteBatchField +{ +public: + virtual ~WriteBatchField() { } + virtual size_t ColWriteBatchDate(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchDatetime(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchTime(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchTimestamp(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchChar(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchVarchar(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchSInt64(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchUInt64(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchSInt32(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchUInt32(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchSInt16(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchUInt16(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchSInt8(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchUInt8(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchXFloat(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchXDouble(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchSLongDouble(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchXDecimal(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchVarbinary(const unsigned char *buf0, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchBlob(const unsigned char *buf0, bool nullVal, ColBatchWriter &ci) = 0; +}; + + +class TypeHandler +{ +public: + using code_t = datatypes::SystemCatalog::ColDataType; +protected: + std::string formatPartitionInfoSInt64(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const; + std::string formatPartitionInfoUInt64(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const; + + std::string PrintPartitionValueSInt64(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const; + + std::string PrintPartitionValueUInt64(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const; + +public: + static const TypeHandler *find(SystemCatalog::ColDataType typeCode, + const SystemCatalog::TypeAttributesStd &attr); + static const TypeHandler *find_by_ddltype(const ddlpackage::ColumnType &ct); + virtual ~TypeHandler() { } + virtual const string & name() const= 0; + virtual const string print(const SystemCatalog::TypeAttributesStd &attr) const + { + return name(); + } + virtual code_t code() const= 0; + virtual bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const + { + return false; + } + virtual uint8_t PartitionValueCharLength(const SystemCatalog::TypeAttributesStd &attr) + const + { + return 30; + } + virtual size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const= 0; + virtual int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const= 0; + + virtual std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const = 0; + + virtual std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const= 0; + virtual execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const= 0; + virtual SimpleValue getMinValueSimple() const + { + return SimpleValue(std::numeric_limits::min(), + std::numeric_limits::min(), + 0); + } + virtual SimpleValue getMaxValueSimple() const + { + return SimpleValue(std::numeric_limits::max(), + std::numeric_limits::max(), + 0); + } + virtual SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const= 0; + virtual MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &a, + const MinMaxInfo &b) const + { + return MinMaxInfo::widenSInt64(a, b); + } + virtual MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const; + virtual string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const + { + return PrintPartitionValueSInt64(attr, partInfo, + startVal, rfMin, + endVal, rfMax); + } + virtual bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &part, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const + { + return part.isSuitableSInt64(startVal, rfMin, endVal, rfMax); + } + virtual boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const = 0; + + virtual boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const = 0; +}; + + +// QQ: perhaps not needed yet +class TypeHandlerBit: public TypeHandler +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::BIT; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + idbassert(0); // QQ + return 0; + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + idbassert(0); // QQ + return 1; + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return "0"; // QQ + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) const override + { + idbassert(0); + return "Error"; + } + + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override + { + idbassert(0); + return NULL; + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override + { + idbassert(0); + return SimpleValue(); + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override + { + //TODO: How to communicate with write engine? + return boost::any(); + } + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerInt: public TypeHandler +{ +protected: + int storeValueToFieldSInt32(rowgroup::Row &row, int pos, + StoreField *f) const; + int storeValueToFieldUInt32(rowgroup::Row &row, int pos, + StoreField *f) const; + std::string formatSInt64(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const; + std::string formatUInt64(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const; + +}; + + +class TypeHandlerSInt8: public TypeHandlerInt +{ +public: + const string &name() const override; + code_t code() const override + { + return SystemCatalog::TINYINT; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchSInt8(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return formatSInt64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoSInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue getMinValueSimple() const override + { + return SimpleValueSInt64(std::numeric_limits::min()); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueSInt64(std::numeric_limits::max()); + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerSInt16: public TypeHandlerInt +{ +public: + const string &name() const override; + code_t code() const override + { + return SystemCatalog::SMALLINT; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchSInt16(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return formatSInt64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoSInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue getMinValueSimple() const override + { + return SimpleValueSInt64(std::numeric_limits::min()); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueSInt64(std::numeric_limits::max()); + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerSInt24: public TypeHandlerInt +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::MEDINT; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchSInt32(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldSInt32(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return formatSInt64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoSInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue getMinValueSimple() const override + { + return SimpleValueSInt64(MIN_MEDINT); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueSInt64(MAX_MEDINT); + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerSInt32: public TypeHandlerInt +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::INT; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchSInt32(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldSInt32(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return formatSInt64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoSInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue getMinValueSimple() const override + { + return SimpleValueSInt64(std::numeric_limits::min()); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueSInt64(std::numeric_limits::max()); + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerSInt64: public TypeHandlerInt +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::BIGINT; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchSInt64(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return formatSInt64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoSInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue getMinValueSimple() const override + { + return SimpleValueSInt64(std::numeric_limits::min()); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueSInt64(std::numeric_limits::max()); + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerUInt8: public TypeHandlerInt +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::UTINYINT; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchUInt8(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return formatUInt64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoUInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + + SimpleValue getMinValueSimple() const override + { + return SimpleValueUInt64(0); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueUInt64(std::numeric_limits::max()); + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &a, + const MinMaxInfo &b) const override + { + return MinMaxInfo::widenUInt64(a, b); + } + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return PrintPartitionValueUInt64(attr, partInfo, + startVal, rfMin, + endVal, rfMax); + } + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &part, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return part.isSuitableUInt64(startVal, rfMin, endVal, rfMax); + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerUInt16: public TypeHandlerInt +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::USMALLINT; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchUInt16(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return formatUInt64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoUInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue getMinValueSimple() const override + { + return SimpleValueUInt64(0); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueUInt64(std::numeric_limits::max()); + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &a, + const MinMaxInfo &b) const override + { + return MinMaxInfo::widenUInt64(a, b); + } + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return PrintPartitionValueUInt64(attr, partInfo, + startVal, rfMin, + endVal, rfMax); + } + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &part, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return part.isSuitableUInt64(startVal, rfMin, endVal, rfMax); + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerUInt24: public TypeHandlerInt +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::UMEDINT; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchUInt32(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldUInt32(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return formatUInt64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoUInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue getMinValueSimple() const override + { + return SimpleValueUInt64(0); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueUInt64(MAX_UMEDINT); + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &a, + const MinMaxInfo &b) const override + { + return MinMaxInfo::widenUInt64(a, b); + } + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return PrintPartitionValueUInt64(attr, partInfo, + startVal, rfMin, + endVal, rfMax); + } + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &part, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return part.isSuitableUInt64(startVal, rfMin, endVal, rfMax); + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerUInt32: public TypeHandlerInt +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::UINT; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchUInt32(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldUInt32(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return formatUInt64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoUInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue getMinValueSimple() const override + { + return SimpleValueUInt64(0); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueUInt64(std::numeric_limits::max()); + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &a, + const MinMaxInfo &b) const override + { + return MinMaxInfo::widenUInt64(a, b); + } + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return PrintPartitionValueUInt64(attr, partInfo, + startVal, rfMin, + endVal, rfMax); + } + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &part, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return part.isSuitableUInt64(startVal, rfMin, endVal, rfMax); + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerUInt64: public TypeHandlerInt +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::BIGINT; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchUInt64(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return formatUInt64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoUInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue getMinValueSimple() const override + { + return SimpleValueUInt64(0); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueUInt64(std::numeric_limits::max()); + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &a, + const MinMaxInfo &b) const override + { + return MinMaxInfo::widenUInt64(a, b); + } + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return PrintPartitionValueUInt64(attr, partInfo, + startVal, rfMin, + endVal, rfMax); + } + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &part, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return part.isSuitableUInt64(startVal, rfMin, endVal, rfMax); + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerXDecimal: public TypeHandler +{ +protected: + + static bool isValidXDecimal64(const SystemCatalog::TypeAttributesStd &attr) + { + return attr.colWidth <= 8; + } + static bool isValidXDecimal128(const SystemCatalog::TypeAttributesStd &attr) + { + return attr.colWidth == 16; + } + int storeValueToField64(rowgroup::Row &row, int pos, + StoreField *f) const; + int storeValueToField128(rowgroup::Row &row, int pos, + StoreField *f) const; + + std::string format64(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const; + std::string format128(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const; + std::string formatPartitionInfo128(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const; + + MinMaxPartitionInfo getExtentPartitionInfo64(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const; + MinMaxPartitionInfo getExtentPartitionInfo128(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const; + string PrintPartitionValue128(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const; + +public: + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchXDecimal(buf, nullVal, writer); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerSDecimal64: public TypeHandlerXDecimal +{ +public: + const string &name() const override; + code_t code() const override + { + return SystemCatalog::DECIMAL; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + uint8_t PartitionValueCharLength(const SystemCatalog::TypeAttributesStd &attr) + const override + { + return 30; + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToField64(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return format64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) const override + { + return formatPartitionInfoSInt64(attr, i); + } + SimpleValue getMinValueSimple() const override + { + return SimpleValueSInt64(std::numeric_limits::min()); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueSInt64(std::numeric_limits::max()); + } + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &a, + const MinMaxInfo &b) const override + { + return MinMaxInfo::widenSInt64(a, b); + } + MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const override + { + return getExtentPartitionInfo64(attr, em, entry, state); + } + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return PrintPartitionValueSInt64(attr, partInfo, + startVal, rfMin, + endVal, rfMax); + } + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &part, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return part.isSuitableSInt64(startVal, rfMin, endVal, rfMax); + } + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerUDecimal64: public TypeHandlerXDecimal +{ +public: + const string &name() const override; + code_t code() const override + { + return SystemCatalog::UDECIMAL; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + uint8_t PartitionValueCharLength(const SystemCatalog::TypeAttributesStd &attr) + const override + { + return 30; + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToField64(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return format64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) const override + { + return formatPartitionInfoSInt64(attr, i); + } + SimpleValue getMinValueSimple() const override + { + return SimpleValueUInt64(0); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueUInt64(std::numeric_limits::max()); + } + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &a, + const MinMaxInfo &b) const override + { + return MinMaxInfo::widenSInt64(a, b); + } + MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const override + { + return getExtentPartitionInfo64(attr, em, entry, state); + } + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return PrintPartitionValueSInt64(attr, partInfo, + startVal, rfMin, + endVal, rfMax); + } + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &part, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return part.isSuitableSInt64(startVal, rfMin, endVal, rfMax); + } + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerSDecimal128: public TypeHandlerXDecimal +{ +public: + const string &name() const override; + code_t code() const override + { + return SystemCatalog::DECIMAL; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + uint8_t PartitionValueCharLength(const SystemCatalog::TypeAttributesStd &attr) + const override + { + return Decimal::MAXLENGTH16BYTES; + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToField128(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return format128(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) const override + { + return formatPartitionInfo128(attr, i); + } + SimpleValue getMinValueSimple() const override + { + return SimpleValue(std::numeric_limits::min(), + datatypes::minInt128, + 0); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValue(std::numeric_limits::max(), + datatypes::maxInt128, + 0); + } + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &a, + const MinMaxInfo &b) const override + { + return MinMaxInfo::widenSInt128(a, b); + } + MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const override + { + return getExtentPartitionInfo128(attr, em, entry, state); + } + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return PrintPartitionValue128(attr, partInfo, startVal, rfMin, endVal, rfMax); + } + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &part, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return part.isSuitableSInt128(startVal, rfMin, endVal, rfMax); + } + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerUDecimal128: public TypeHandlerXDecimal +{ +public: + const string &name() const override; + code_t code() const override + { + return SystemCatalog::UDECIMAL; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + uint8_t PartitionValueCharLength(const SystemCatalog::TypeAttributesStd &attr) + const override + { + return Decimal::MAXLENGTH16BYTES; + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToField128(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return format128(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) const override + { + return formatPartitionInfo128(attr, i); + } + SimpleValue getMinValueSimple() const override + { + return SimpleValueSInt128(0); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueSInt128(-1); + } + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &a, + const MinMaxInfo &b) const override + { + return MinMaxInfo::widenSInt128(a, b); + } + MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const override + { + return getExtentPartitionInfo128(attr, em, entry, state); + } + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return PrintPartitionValue128(attr, partInfo, startVal, rfMin, endVal, rfMax); + } + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &part, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return part.isSuitableSInt128(startVal, rfMin, endVal, rfMax); + } + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerReal: public TypeHandler +{ +public: + int storeValueToFieldXFloat(rowgroup::Row &row, int pos, + StoreField *f) const; + int storeValueToFieldXDouble(rowgroup::Row &row, int pos, + StoreField *f) const; + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override + { + return SimpleValue(); // QQ: real types were not handled in IDB_format() + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return "0"; // QQ + } +}; + + +class TypeHandlerSFloat: public TypeHandlerReal +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::FLOAT; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchXFloat(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldXFloat(row, pos, f); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoSInt64(attr, i); + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerSDouble: public TypeHandlerReal +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::DOUBLE; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchXDouble(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldXDouble(row, pos, f); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoSInt64(attr, i); + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerUFloat: public TypeHandlerReal +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::UFLOAT; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchXFloat(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldXFloat(row, pos, f); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoSInt64(attr, i); + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerUDouble: public TypeHandlerReal +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::UDOUBLE; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchXDouble(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldXDouble(row, pos, f); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoSInt64(attr, i); + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerSLongDouble: public TypeHandlerReal +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::LONGDOUBLE; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchSLongDouble(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + idbassert(0); + return "Error"; + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override + { + // QQ: DDLPackageProcessor::getNullValueForType() did not handle LONGDOUBLE + return boost::any(); + } + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override + { + throw logging::QueryDataExcept("convertColumnData: unknown column data type.", logging::dataTypeErr); + return boost::any(); + } +}; + + +class TypeHandlerTemporal: public TypeHandler +{ +public: + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoSInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; +}; + + +class TypeHandlerDate: public TypeHandlerTemporal +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::DATE; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchDate(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override; + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerDatetime: public TypeHandlerTemporal +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::DATETIME; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchDatetime(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override; + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerTime: public TypeHandlerTemporal +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::TIME; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchTime(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override; + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerTimestamp: public TypeHandlerTemporal +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::TIMESTAMP; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchTimestamp(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override; + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerStr: public TypeHandler +{ +protected: + std::string formatPartitionInfoSmallCharVarchar( + const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const; + boost::any getNullValueForTypeVarcharText(const SystemCatalog::TypeAttributesStd &attr) + const; +public: + int storeValueToFieldCharVarchar(rowgroup::Row &row, int pos, + StoreField *f) const; + int storeValueToFieldBlobText(rowgroup::Row &row, int pos, + StoreField *f) const; + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + // QQ: Check with Roman if this correct: + return formatPartitionInfoSInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; +}; + + +class TypeHandlerChar: public TypeHandlerStr +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::CHAR; + } + const string print(const SystemCatalog::TypeAttributesStd &attr) const override + { + ostringstream oss; + oss << name () << "(" << attr.colWidth << ")"; + return oss.str(); + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return attr.colWidth <= 8; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchChar(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldCharVarchar(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override; + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override; + MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerVarchar: public TypeHandlerStr +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::VARCHAR; + } + const string print(const SystemCatalog::TypeAttributesStd &attr) const override + { + ostringstream oss; + oss << name () << "(" << attr.colWidth << ")"; + return oss.str(); + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return attr.colWidth <= 7; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchVarchar(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldCharVarchar(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override; + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override; + MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override + { + return getNullValueForTypeVarcharText(attr); + } + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerVarbinary: public TypeHandlerStr +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::VARBINARY; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchVarbinary(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerBlob: public TypeHandlerStr +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::BLOB; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchBlob(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldBlobText(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return "0"; // QQ + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerText: public TypeHandlerStr +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::TEXT; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchBlob(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldBlobText(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return "0"; // QQ + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override + { + return getNullValueForTypeVarcharText(attr); + } + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +class TypeHandlerClob: public TypeHandlerStr +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::CLOB; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + idbassert(0); // QQ + return 0; + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + idbassert(0); // QQ + return 1; + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return "0"; // QQ + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override + { + return boost::any(); // QQ + } + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; +}; + + +}// end of namespace datatypes + +#endif //MCS_DATATYPE_H_INCLUDED + +// vim:ts=2 sw=2: diff --git a/datatypes/mcs_decimal.cpp b/datatypes/mcs_decimal.cpp new file mode 100644 index 000000000..b2563251f --- /dev/null +++ b/datatypes/mcs_decimal.cpp @@ -0,0 +1,740 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include +#include + +#include "utils/common/branchpred.h" +#include "mcs_decimal.h" +#include "exceptclasses.h" +#include "dataconvert.h" + +namespace datatypes +{ + + struct lldiv_t_128 + { + int128_t quot; + int128_t rem; + lldiv_t_128() : quot(0), rem(0) {} + }; + + inline lldiv_t_128 lldiv128(const int128_t& dividend, const int128_t& divisor) + { + lldiv_t_128 res; + + if (UNLIKELY(divisor == 0) || UNLIKELY(dividend == 0)) + return res; + + res.quot = dividend / divisor; + res.rem = dividend % divisor; + + return res; + } + + template + void addSubtractExecute(const VDecimal& l, + const VDecimal& r, + VDecimal& result, + BinaryOperation op, + OpOverflowCheck opOverflowCheck, + MultiplicationOverflowCheck mulOverflowCheck) + { + int128_t lValue = Decimal::isWideDecimalTypeByPrecision(l.precision) + ? l.s128Value : l.value; + int128_t rValue = Decimal::isWideDecimalTypeByPrecision(r.precision) + ? r.s128Value : r.value; + + if (result.scale == l.scale && result.scale == r.scale) + { + opOverflowCheck(lValue, rValue); + result.s128Value = op(lValue, rValue); + return; + } + + if (result.scale > l.scale) + { + int128_t scaleMultiplier; + getScaleDivisor(scaleMultiplier, result.scale - l.scale); + mulOverflowCheck(lValue, scaleMultiplier); + lValue *= scaleMultiplier; + } + else if (result.scale < l.scale) + { + int128_t scaleMultiplier; + getScaleDivisor(scaleMultiplier, l.scale - result.scale); + lValue = (int128_t) (lValue > 0 ? + (__float128)lValue / scaleMultiplier + 0.5 : + (__float128)lValue / scaleMultiplier - 0.5); + } + + if (result.scale > r.scale) + { + int128_t scaleMultiplier; + getScaleDivisor(scaleMultiplier, result.scale - r.scale); + mulOverflowCheck(rValue, scaleMultiplier); + rValue *= scaleMultiplier; + } + else if (result.scale < r.scale) + { + int128_t scaleMultiplier; + getScaleDivisor(scaleMultiplier, r.scale - result.scale); + rValue = (int128_t) (rValue > 0 ? + (__float128)rValue / scaleMultiplier + 0.5 : + (__float128)rValue / scaleMultiplier - 0.5); + } + + // We assume there is no way that lValue or rValue calculations + // give an overflow and this is an incorrect assumption. + opOverflowCheck(lValue, rValue); + + result.s128Value = op(lValue, rValue); + } + + template + void divisionExecute(const VDecimal& l, + const VDecimal& r, + VDecimal& result, + OpOverflowCheck opOverflowCheck, + MultiplicationOverflowCheck mulOverflowCheck) + { + int128_t lValue = Decimal::isWideDecimalTypeByPrecision(l.precision) + ? l.s128Value : l.value; + int128_t rValue = Decimal::isWideDecimalTypeByPrecision(r.precision) + ? r.s128Value : r.value; + + opOverflowCheck(lValue, rValue); + + if (result.scale >= l.scale - r.scale) + { + int128_t scaleMultiplier; + + getScaleDivisor(scaleMultiplier, result.scale - (l.scale - r.scale)); + + // TODO How do we check overflow of (int128_t)((__float128)lValue / rValue * scaleMultiplier) ? + + result.s128Value = (int128_t)(( (lValue > 0 && rValue > 0) || (lValue < 0 && rValue < 0) ? + (__float128)lValue / rValue * scaleMultiplier + 0.5 : + (__float128)lValue / rValue * scaleMultiplier - 0.5)); + } + else + { + int128_t scaleMultiplier; + + getScaleDivisor(scaleMultiplier, (l.scale - r.scale) - result.scale); + + result.s128Value = (int128_t)(( (lValue > 0 && rValue > 0) || (lValue < 0 && rValue < 0) ? + (__float128)lValue / rValue / scaleMultiplier + 0.5 : + (__float128)lValue / rValue / scaleMultiplier - 0.5)); + } + } + + template + void multiplicationExecute(const VDecimal& l, + const VDecimal& r, + VDecimal& result, + OpOverflowCheck opOverflowCheck, + MultiplicationOverflowCheck mulOverflowCheck) + { + int128_t lValue = Decimal::isWideDecimalTypeByPrecision(l.precision) + ? l.s128Value : l.value; + int128_t rValue = Decimal::isWideDecimalTypeByPrecision(r.precision) + ? r.s128Value : r.value; + + if (lValue == 0 || rValue == 0) + { + result.s128Value = 0; + return; + } + + if (result.scale >= l.scale + r.scale) + { + int128_t scaleMultiplier; + + getScaleDivisor(scaleMultiplier, result.scale - (l.scale + r.scale)); + + opOverflowCheck(lValue, rValue, result.s128Value); + opOverflowCheck(result.s128Value, scaleMultiplier, result.s128Value); + } + else + { + unsigned int diff = l.scale + r.scale - result.scale; + + int128_t scaleMultiplierL, scaleMultiplierR; + + getScaleDivisor(scaleMultiplierL, diff / 2); + getScaleDivisor(scaleMultiplierR, diff - (diff / 2)); + + lValue = (int128_t)(( (lValue > 0) ? + (__float128)lValue / scaleMultiplierL + 0.5 : + (__float128)lValue / scaleMultiplierL - 0.5)); + + rValue = (int128_t)(( (rValue > 0) ? + (__float128)rValue / scaleMultiplierR + 0.5 : + (__float128)rValue / scaleMultiplierR - 0.5)); + + opOverflowCheck(lValue, rValue, result.s128Value);; + } + } + + int Decimal::compare(const VDecimal& l, const VDecimal& r) + { + int128_t divisorL, divisorR; + getScaleDivisor(divisorL, l.scale); + getScaleDivisor(divisorR, r.scale); + + lldiv_t_128 d1 = lldiv128(l.s128Value, divisorL); + lldiv_t_128 d2 = lldiv128(r.s128Value, divisorR); + + int ret = 0; + + if (d1.quot > d2.quot) + { + ret = 1; + } + else if (d1.quot < d2.quot) + { + ret = -1; + } + else + { + // rem carries the value's sign, but needs to be normalized. + int64_t s = l.scale - r.scale; + int128_t divisor; + getScaleDivisor(divisor, abs(s)); + + if (s < 0) + { + if ((d1.rem * divisor) > d2.rem) + ret = 1; + else if ((d1.rem * divisor) < d2.rem) + ret = -1; + } + else + { + if (d1.rem > (d2.rem * divisor)) + ret = 1; + else if (d1.rem < (d2.rem * divisor)) + ret = -1; + } + } + + return ret; + } + + // no overflow check + template<> + void Decimal::addition(const VDecimal& l, + const VDecimal& r, VDecimal& result) + { + std::plus add; + NoOverflowCheck noOverflowCheck; + addSubtractExecute(l, r, result, add, noOverflowCheck, noOverflowCheck); + } + + // with overflow check + template<> + void Decimal::addition(const VDecimal& l, + const VDecimal& r, VDecimal& result) + { + std::plus add; + AdditionOverflowCheck overflowCheck; + MultiplicationOverflowCheck mulOverflowCheck; + addSubtractExecute(l, r, result, add, overflowCheck, mulOverflowCheck); + } + + // no overflow check + template<> + void Decimal::addition(const VDecimal& l, + const VDecimal& r, VDecimal& result) + { + if (result.scale == l.scale && result.scale == r.scale) + { + result.value = l.value + r.value; + return; + } + + int64_t lValue = l.value, rValue = r.value; + + if (result.scale > l.scale) + lValue *= mcs_pow_10[result.scale - l.scale]; + else if (result.scale < l.scale) + lValue = (int64_t)(lValue > 0 ? + (double)lValue / mcs_pow_10[l.scale - result.scale] + 0.5 : + (double)lValue / mcs_pow_10[l.scale - result.scale] - 0.5); + + if (result.scale > r.scale) + rValue *= mcs_pow_10[result.scale - r.scale]; + else if (result.scale < r.scale) + rValue = (int64_t)(rValue > 0 ? + (double)rValue / mcs_pow_10[r.scale - result.scale] + 0.5 : + (double)rValue / mcs_pow_10[r.scale - result.scale] - 0.5); + + result.value = lValue + rValue; + } + + // with overflow check + template<> + void Decimal::addition(const VDecimal& l, + const VDecimal& r, VDecimal& result) + { + AdditionOverflowCheck additionOverflowCheck; + MultiplicationOverflowCheck mulOverflowCheck; + + if (result.scale == l.scale && result.scale == r.scale) + { + additionOverflowCheck(l.value, r.value); + result.value = l.value + r.value; + return; + } + + int64_t lValue = l.value, rValue = r.value; + + if (result.scale > l.scale) + mulOverflowCheck(lValue, mcs_pow_10[result.scale - l.scale], lValue); + else if (result.scale < l.scale) + lValue = (int64_t)(lValue > 0 ? + (double)lValue / mcs_pow_10[l.scale - result.scale] + 0.5 : + (double)lValue / mcs_pow_10[l.scale - result.scale] - 0.5); + + if (result.scale > r.scale) + mulOverflowCheck(rValue, mcs_pow_10[result.scale - r.scale], rValue); + else if (result.scale < r.scale) + rValue = (int64_t)(rValue > 0 ? + (double)rValue / mcs_pow_10[r.scale - result.scale] + 0.5 : + (double)rValue / mcs_pow_10[r.scale - result.scale] - 0.5); + + additionOverflowCheck(lValue, rValue); + result.value = lValue + rValue; + } + + // no overflow check + template<> + void Decimal::subtraction(const VDecimal& l, + const VDecimal& r, VDecimal& result) + { + std::minus subtract; + NoOverflowCheck noOverflowCheck; + addSubtractExecute(l, r, result, subtract, noOverflowCheck, noOverflowCheck); + } + + // with overflow check + template<> + void Decimal::subtraction(const VDecimal& l, + const VDecimal& r, VDecimal& result) + { + std::minus subtract; + SubtractionOverflowCheck overflowCheck; + MultiplicationOverflowCheck mulOverflowCheck; + addSubtractExecute(l, r, result, subtract, overflowCheck, mulOverflowCheck); + } + + // no overflow check + template<> + void Decimal::subtraction(const VDecimal& l, + const VDecimal& r, VDecimal& result) + { + if (result.scale == l.scale && result.scale == r.scale) + { + result.value = l.value - r.value; + return; + } + + int64_t lValue = l.value, rValue = r.value; + + if (result.scale > l.scale) + lValue *= mcs_pow_10[result.scale - l.scale]; + else if (result.scale < l.scale) + lValue = (int64_t)(lValue > 0 ? + (double)lValue / mcs_pow_10[l.scale - result.scale] + 0.5 : + (double)lValue / mcs_pow_10[l.scale - result.scale] - 0.5); + + if (result.scale > r.scale) + rValue *= mcs_pow_10[result.scale - r.scale]; + else if (result.scale < r.scale) + rValue = (int64_t)(rValue > 0 ? + (double)rValue / mcs_pow_10[r.scale - result.scale] + 0.5 : + (double)rValue / mcs_pow_10[r.scale - result.scale] - 0.5); + + result.value = lValue - rValue; + } + + // with overflow check + template<> + void Decimal::subtraction(const VDecimal& l, + const VDecimal& r, VDecimal& result) + { + SubtractionOverflowCheck subtractionOverflowCheck; + MultiplicationOverflowCheck mulOverflowCheck; + + if (result.scale == l.scale && result.scale == r.scale) + { + subtractionOverflowCheck(l.value, r.value); + result.value = l.value - r.value; + return; + } + + int64_t lValue = l.value, rValue = r.value; + + if (result.scale > l.scale) + mulOverflowCheck(lValue, mcs_pow_10[result.scale - l.scale], lValue); + else if (result.scale < l.scale) + lValue = (int64_t)(lValue > 0 ? + (double)lValue / mcs_pow_10[l.scale - result.scale] + 0.5 : + (double)lValue / mcs_pow_10[l.scale - result.scale] - 0.5); + + if (result.scale > r.scale) + mulOverflowCheck(rValue, mcs_pow_10[result.scale - r.scale], rValue); + else if (result.scale < r.scale) + rValue = (int64_t)(rValue > 0 ? + (double)rValue / mcs_pow_10[r.scale - result.scale] + 0.5 : + (double)rValue / mcs_pow_10[r.scale - result.scale] - 0.5); + + subtractionOverflowCheck(lValue, rValue); + result.value = lValue - rValue; + } + + // no overflow check + template<> + void Decimal::division(const VDecimal& l, + const VDecimal& r, VDecimal& result) + { + NoOverflowCheck noOverflowCheck; + divisionExecute(l, r, result, noOverflowCheck, noOverflowCheck); + } + + // With overflow check + template<> + void Decimal::division(const VDecimal& l, + const VDecimal& r, VDecimal& result) + { + DivisionOverflowCheck overflowCheck; + MultiplicationOverflowCheck mulOverflowCheck; + divisionExecute(l, r, result, overflowCheck, mulOverflowCheck); + } + + // no overflow check + // We rely on the zero check from ArithmeticOperator::execute + template<> + void Decimal::division(const VDecimal& l, + const VDecimal& r, VDecimal& result) + { + if (result.scale >= l.scale - r.scale) + result.value = (int64_t)(( (l.value > 0 && r.value > 0) || (l.value < 0 && r.value < 0) ? + (long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)] + 0.5 : + (long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)] - 0.5)); + else + result.value = (int64_t)(( (l.value > 0 && r.value > 0) || (l.value < 0 && r.value < 0) ? + (long double)l.value / r.value / mcs_pow_10[l.scale - r.scale - result.scale] + 0.5 : + (long double)l.value / r.value / mcs_pow_10[l.scale - r.scale - result.scale] - 0.5)); + } + + // With overflow check + template<> + void Decimal::division(const VDecimal& l, + const VDecimal& r, VDecimal& result) + { + DivisionOverflowCheck divisionOverflowCheck; + + divisionOverflowCheck(l.value, r.value); + + if (result.scale >= l.scale - r.scale) + // TODO How do we check overflow of (int64_t)((long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)]) ? + result.value = (int64_t)(( (l.value > 0 && r.value > 0) || (l.value < 0 && r.value < 0) ? + (long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)] + 0.5 : + (long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)] - 0.5)); + else + result.value = (int64_t)(( (l.value > 0 && r.value > 0) || (l.value < 0 && r.value < 0) ? + (long double)l.value / r.value / mcs_pow_10[l.scale - r.scale - result.scale] + 0.5 : + (long double)l.value / r.value / mcs_pow_10[l.scale - r.scale - result.scale] - 0.5)); + } + + // no overflow check + template<> + void Decimal::multiplication(const VDecimal& l, + const VDecimal& r, VDecimal& result) + { + MultiplicationNoOverflowCheck noOverflowCheck; + multiplicationExecute(l, r, result, noOverflowCheck, noOverflowCheck); + } + + // With overflow check + template<> + void Decimal::multiplication(const VDecimal& l, + const VDecimal& r, VDecimal& result) + { + MultiplicationOverflowCheck mulOverflowCheck; + multiplicationExecute(l, r, result, mulOverflowCheck, mulOverflowCheck); + } + + // no overflow check + template<> + void Decimal::multiplication(const VDecimal& l, + const VDecimal& r, VDecimal& result) + { + if (result.scale >= l.scale + r.scale) + result.value = l.value * r.value * mcs_pow_10[result.scale - (l.scale + r.scale)]; + else + result.value = (int64_t)(( (l.value > 0 && r.value > 0) || (l.value < 0 && r.value < 0) ? + (double)l.value * r.value / mcs_pow_10[l.scale + r.scale - result.scale] + 0.5 : + (double)l.value * r.value / mcs_pow_10[l.scale + r.scale - result.scale] - 0.5)); + } + + // With overflow check + template<> + void Decimal::multiplication(const VDecimal& l, + const VDecimal& r, VDecimal& result) + { + MultiplicationOverflowCheck mulOverflowCheck; + + if (result.scale >= l.scale + r.scale) + { + mulOverflowCheck(l.value, r.value, result.value); + mulOverflowCheck(result.value, mcs_pow_10[result.scale - (l.scale + r.scale)], result.value); + } + else + { + mulOverflowCheck(l.value, r.value, result.value); + + result.value = (int64_t)(( (result.value > 0) ? + (double)result.value / mcs_pow_10[l.scale + r.scale - result.scale] + 0.5 : + (double)result.value / mcs_pow_10[l.scale + r.scale - result.scale] - 0.5)); + } + } + + // Writes integer part of a Decimal using int128 argument provided + uint8_t VDecimal::writeIntPart(const int128_t& x, + char* buf, + const uint8_t buflen) const + { + char* p = buf; + int128_t intPart = x; + int128_t high = 0, mid = 0, low = 0; + uint64_t maxUint64divisor = 10000000000000000000ULL; + + // Assuming scale = [0, 56] + switch (scale / datatypes::maxPowOf10) + { + case 2: // scale = [38, 56] + intPart /= datatypes::mcs_pow_10[datatypes::maxPowOf10]; + intPart /= datatypes::mcs_pow_10[datatypes::maxPowOf10]; + low = intPart; + break; + case 1: // scale = [19, 37] + intPart /= datatypes::mcs_pow_10[datatypes::maxPowOf10]; + intPart /= datatypes::mcs_pow_10[scale % datatypes::maxPowOf10]; + low = intPart % maxUint64divisor; + mid = intPart / maxUint64divisor; + break; + case 0: // scale = [0, 18] + intPart /= datatypes::mcs_pow_10[scale % datatypes::maxPowOf10]; + low = intPart % maxUint64divisor; + intPart /= maxUint64divisor; + mid = intPart % maxUint64divisor; + high = intPart / maxUint64divisor; + break; + default: + throw logging::QueryDataExcept("VDecimal::writeIntPart() bad scale", + logging::formatErr); + } + + p += printPodParts(p, high, mid, low); + uint8_t written = p - buf; + if (buflen <= written) + { + throw logging::QueryDataExcept("VDecimal::writeIntPart() char buffer overflow.", + logging::formatErr); + } + + return written; + } + + uint8_t VDecimal::writeFractionalPart(const int128_t& x, + char* buf, + const uint8_t buflen) const + { + int128_t scaleDivisor = 1; + char* p = buf; + + switch (scale / datatypes::maxPowOf10) + { + case 2: + scaleDivisor *= datatypes::mcs_pow_10[datatypes::maxPowOf10]; + scaleDivisor *= datatypes::mcs_pow_10[datatypes::maxPowOf10]; + break; + case 1: + scaleDivisor *= datatypes::mcs_pow_10[datatypes::maxPowOf10]; + //fallthrough + case 0: + scaleDivisor *= datatypes::mcs_pow_10[scale % datatypes::maxPowOf10]; + } + + int128_t fractionalPart = x % scaleDivisor; + + // divide by the base until we have non-zero quotient + scaleDivisor /= 10; + + while (scaleDivisor > 1 && fractionalPart / scaleDivisor == 0) + { + *p++ = '0'; + scaleDivisor /= 10; + } + size_t written = p - buf;; + p += TSInt128::writeIntPart(fractionalPart, p, buflen - written); + return p - buf; + } + + // The method writes Decimal based on TSInt128 with scale provided. + // It first writes sign, then extracts integer part + // prints delimiter and then decimal part. + std::string VDecimal::toStringTSInt128WithScale() const + { + char buf[Decimal::MAXLENGTH16BYTES]; + uint8_t left = sizeof(buf); + char* p = buf; + int128_t tempValue = s128Value; + // sign + if (tempValue < static_cast(0)) + { + *p++ = '-'; + tempValue *= -1; + left--; + } + + // integer part + p += writeIntPart(tempValue, p, left); + + // decimal delimiter + *p++ = '.'; + // decimal part + left = sizeof(buf) - (p - buf); + p += writeFractionalPart(tempValue, p, left); + + *p = '\0'; + + uint8_t written = p - buf; + if (sizeof(buf) <= written) + { + throw logging::QueryDataExcept("VDecimal::toString() char buffer overflow.", + logging::formatErr); + } + return std::string(buf); + } + + std::string VDecimal::toStringTSInt64() const + { + char buf[Decimal::MAXLENGTH8BYTES]; + // Need 19 digits maxium to hold a sum result of 18 digits decimal column. + // We don't make a copy of value b/c we mutate its string + // representation. +#ifndef __LP64__ + snprintf(buf, sizeof(buf), "%lld", value); +#else + snprintf(buf, sizeof(buf), "%ld", value); +#endif + + //we want to move the last dt_scale chars right by one spot + // to insert the dp we want to move the trailing null as well, + // so it's really dt_scale+1 chars + size_t l1 = strlen(buf); + char* ptr = &buf[0]; + + if (value < 0) + { + ptr++; + idbassert(l1 >= 2); + l1--; + } + + //need to make sure we have enough leading zeros for this to work. + //at this point scale is always > 0 + size_t l2 = 1; + + if ((unsigned)scale > l1) + { + const char* zeros = "00000000000000000000"; //20 0's + size_t diff = 0; + + if (value != 0) + diff = scale - l1; //this will always be > 0 + else + diff = scale; + + memmove((ptr + diff), ptr, l1 + 1); //also move null + memcpy(ptr, zeros, diff); + + if (value != 0) + l1 = 0; + else + l1 = 1; + } + else if ((unsigned)scale == l1) + { + l1 = 0; + l2 = 2; + } + else + { + l1 -= scale; + } + + memmove((ptr + l1 + l2), (ptr + l1), scale + 1); //also move null + + if (l2 == 2) + *(ptr + l1++) = '0'; + + *(ptr + l1) = '.'; + return std::string(buf); + } + + // Dispatcher method for toString() implementations + std::string VDecimal::toString(bool hasTSInt128) const + { + // There must be no empty at this point though + if (isNull()) + { + return std::string("NULL"); + } + + if(LIKELY(hasTSInt128 || isTSInt128ByPrecision())) + { + if (scale) + { + return toStringTSInt128WithScale(); + } + return TSInt128::toString(); + } + // TSInt64 Decimal + if (scale) + { + return toStringTSInt64(); + } + return std::to_string(value); + } + + std::ostream& operator<<(std::ostream& os, const VDecimal& dec) + { + os << dec.toString(); + return os; + } + + +} // end of namespace diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h new file mode 100644 index 000000000..387c4f662 --- /dev/null +++ b/datatypes/mcs_decimal.h @@ -0,0 +1,855 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#ifndef H_DECIMALDATATYPE +#define H_DECIMALDATATYPE + +#include +#include +#include +#include "mcs_basic_types.h" +#include "exceptclasses.h" +#include "widedecimalutils.h" +#include "mcs_int128.h" + + +namespace datatypes +{ + class VDecimal; +} + +// A class by Fabio Fernandes pulled off of stackoverflow +// Creates a type _xxl that can be used to create 128bit constant values +// Ex: int128_t i128 = 12345678901234567890123456789_xxl +namespace detail_xxl +{ + constexpr uint8_t hexval(char c) + { return c>='a' ? (10+c-'a') : c>='A' ? (10+c-'A') : c-'0'; } + + template + constexpr uint128_t lit_eval() { return V; } + + template + constexpr uint128_t lit_eval() { + static_assert( BASE!=16 || sizeof...(Cs) <= 32-1, "Literal too large for BASE=16"); + static_assert( BASE!=10 || sizeof...(Cs) <= 39-1, "Literal too large for BASE=10"); + static_assert( BASE!=8 || sizeof...(Cs) <= 44-1, "Literal too large for BASE=8"); + static_assert( BASE!=2 || sizeof...(Cs) <= 128-1, "Literal too large for BASE=2"); + return lit_eval(); + } + + template struct LitEval + {static constexpr uint128_t eval() {return lit_eval<10,0,Cs...>();} }; + + template struct LitEval<'0','x',Cs...> + {static constexpr uint128_t eval() {return lit_eval<16,0,Cs...>();} }; + + template struct LitEval<'0','b',Cs...> + {static constexpr uint128_t eval() {return lit_eval<2,0,Cs...>();} }; + + template struct LitEval<'0',Cs...> + {static constexpr uint128_t eval() {return lit_eval<8,0,Cs...>();} }; + + template + constexpr uint128_t operator "" _xxl() {return LitEval::eval();} +} + +template +constexpr uint128_t operator "" _xxl() {return ::detail_xxl::operator "" _xxl();} + +namespace datatypes +{ + +constexpr uint32_t MAXDECIMALWIDTH = 16U; +constexpr uint8_t INT64MAXPRECISION = 18U; +constexpr uint8_t INT128MAXPRECISION = 38U; +constexpr uint8_t MAXLEGACYWIDTH = 8U; +constexpr uint8_t MAXSCALEINC4AVG = 4U; +constexpr int8_t IGNOREPRECISION = -1; + + + +const uint64_t mcs_pow_10[20] = +{ + 1ULL, + 10ULL, + 100ULL, + 1000ULL, + 10000ULL, + 100000ULL, + 1000000ULL, + 10000000ULL, + 100000000ULL, + 1000000000ULL, + 10000000000ULL, + 100000000000ULL, + 1000000000000ULL, + 10000000000000ULL, + 100000000000000ULL, + 1000000000000000ULL, + 10000000000000000ULL, + 100000000000000000ULL, + 1000000000000000000ULL, + 10000000000000000000ULL, +}; +const int128_t mcs_pow_10_128[20] = +{ + 10000000000000000000_xxl, + 100000000000000000000_xxl, + 1000000000000000000000_xxl, + 10000000000000000000000_xxl, + 100000000000000000000000_xxl, + 1000000000000000000000000_xxl, + 10000000000000000000000000_xxl, + 100000000000000000000000000_xxl, + 1000000000000000000000000000_xxl, + 10000000000000000000000000000_xxl, + 100000000000000000000000000000_xxl, + 1000000000000000000000000000000_xxl, + 10000000000000000000000000000000_xxl, + 100000000000000000000000000000000_xxl, + 1000000000000000000000000000000000_xxl, + 10000000000000000000000000000000000_xxl, + 100000000000000000000000000000000000_xxl, + 1000000000000000000000000000000000000_xxl, + 10000000000000000000000000000000000000_xxl, + 100000000000000000000000000000000000000_xxl, +}; + +constexpr uint32_t maxPowOf10 = sizeof(mcs_pow_10)/sizeof(mcs_pow_10[0])-1; +constexpr int128_t Decimal128Null = TSInt128::NullValue; +constexpr int128_t Decimal128Empty = TSInt128::EmptyValue; + + +/** + @brief The function to produce scale multiplier/divisor for + wide decimals. +*/ +template +inline void getScaleDivisor(T& divisor, const int8_t scale) +{ + if (scale < 0) + { + std::string msg = "getScaleDivisor called with negative scale: " + std::to_string(scale); + throw std::invalid_argument(msg); + } + else if (scale < 19) + { + divisor = mcs_pow_10[scale]; + } + else + { + divisor = mcs_pow_10_128[scale-19]; + } +} + + +/** + @brief Contains subset of decimal related operations. + + Most of the methods are static to allow to call them from + the existing code. + The main purpose of the class is to collect methods for a + base Datatype class. +*/ +class Decimal +{ + public: + Decimal() { }; + ~Decimal() { }; + + static constexpr uint8_t MAXLENGTH16BYTES = TSInt128::maxLength(); + static constexpr uint8_t MAXLENGTH8BYTES = 23; + + static inline bool isWideDecimalNullValue(const int128_t& val) + { + const uint64_t* ptr = reinterpret_cast(&val); + return (ptr[0] == utils::BINARYNULLVALUELOW && ptr[1] == utils::BINARYNULLVALUEHIGH); + } + + static inline bool isWideDecimalEmptyValue(const int128_t& val) + { + const uint64_t* ptr = reinterpret_cast(&val); + return (ptr[0] == utils::BINARYEMPTYVALUELOW && ptr[1] == utils::BINARYEMPTYVALUEHIGH); + } + + static inline void setWideDecimalNullValue(int128_t& val) + { + uint64_t* ptr = reinterpret_cast(&val); + ptr[0] = utils::BINARYNULLVALUELOW; + ptr[1] = utils::BINARYNULLVALUEHIGH; + } + + static inline void setWideDecimalEmptyValue(int128_t& val) + { + uint64_t* ptr = reinterpret_cast(&val); + ptr[0] = utils::BINARYEMPTYVALUELOW; + ptr[1] = utils::BINARYEMPTYVALUEHIGH; + } + + static inline void setWideDecimalNullValue(int128_t* val) + { + uint64_t* ptr = reinterpret_cast(val); + ptr[0] = utils::BINARYNULLVALUELOW; + ptr[1] = utils::BINARYNULLVALUEHIGH; + } + + static inline void setWideDecimalEmptyValue(int128_t* val) + { + uint64_t* ptr = reinterpret_cast(val); + ptr[0] = utils::BINARYEMPTYVALUELOW; + ptr[1] = utils::BINARYEMPTYVALUEHIGH; + } + + + static constexpr int128_t minInt128 = int128_t(0x8000000000000000LL) << 64; + static constexpr int128_t maxInt128 = (int128_t(0x7FFFFFFFFFFFFFFFLL) << 64) + 0xFFFFFFFFFFFFFFFFLL; + + /** + @brief Compares two VDecimal taking scale into account. + */ + static int compare(const VDecimal& l, const VDecimal& r); + /** + @brief Addition template that supports overflow check and + two internal representations of decimal. + */ + template + static void addition(const VDecimal& l, + const VDecimal& r, + VDecimal& result); + + /** + @brief Subtraction template that supports overflow check and + two internal representations of decimal. + */ + template + static void subtraction(const VDecimal& l, + const VDecimal& r, + VDecimal& result); + + /** + @brief Division template that supports overflow check and + two internal representations of decimal. + */ + template + static void division(const VDecimal& l, + const VDecimal& r, + VDecimal& result); + + /** + @brief Multiplication template that supports overflow check and + two internal representations of decimal. + */ + template + static void multiplication(const VDecimal& l, + const VDecimal& r, + VDecimal& result); + + /** + @brief The method detects whether decimal type is wide + using precision. + */ + static inline bool isWideDecimalTypeByPrecision(const int32_t precision) + { + return precision > INT64MAXPRECISION + && precision <= INT128MAXPRECISION; + } + + /** + @brief The method converts a __float128 value to an int64_t. + */ + static inline int64_t getInt64FromFloat128(const __float128& value) + { + if (value > static_cast<__float128>(INT64_MAX)) + return INT64_MAX; + else if (value < static_cast<__float128>(INT64_MIN)) + return INT64_MIN; + + return static_cast(value); + } + + /** + @brief The method converts a __float128 value to an uint64_t. + */ + static inline uint64_t getUInt64FromFloat128(const __float128& value) + { + if (value > static_cast<__float128>(UINT64_MAX)) + return UINT64_MAX; + else if (value < 0) + return 0; + + return static_cast(value); + } + + /** + @brief The method converts a wide decimal value to an uint32_t. + */ + static inline uint32_t getUInt32FromWideDecimal(const int128_t& value) + { + if (value > static_cast(UINT32_MAX)) + return UINT32_MAX; + else if (value < 0) + return 0; + + return static_cast(value); + } + + /** + @brief The method converts a wide decimal value to an int32_t. + */ + static inline int32_t getInt32FromWideDecimal(const int128_t& value) + { + if (value > static_cast(INT32_MAX)) + return INT32_MAX; + else if (value < static_cast(INT32_MIN)) + return INT32_MIN; + + return static_cast(value); + } + + /** + @brief The method converts a wide decimal value to an uint64_t. + */ + static inline uint64_t getUInt64FromWideDecimal(const int128_t& value) + { + if (value > static_cast(UINT64_MAX)) + return UINT64_MAX; + else if (value < 0) + return 0; + + return static_cast(value); + } + + /** + @brief The method converts a wide decimal value to a double. + */ + static inline double getDoubleFromWideDecimal(const int128_t& value, int8_t scale) + { + int128_t scaleDivisor; + + getScaleDivisor(scaleDivisor, scale); + + __float128 tmpval = (__float128) value / scaleDivisor; + + return getDoubleFromFloat128(tmpval); + } + + /** + @brief The method converts a wide decimal value to a double. + */ + static inline double getDoubleFromWideDecimal(const int128_t& value) + { + return getDoubleFromFloat128(static_cast<__float128>(value)); + } + + /** + @brief The method converts a wide decimal value to a long double. + */ + static inline long double getLongDoubleFromWideDecimal(const int128_t& value) + { + return getLongDoubleFromFloat128(static_cast<__float128>(value)); + } + + /** + @brief The method converts a wide decimal value to an int64_t, + saturating the value if necessary. + */ + static inline int64_t getInt64FromWideDecimal(const int128_t& value) + { + if (value > static_cast(INT64_MAX)) + return INT64_MAX; + else if (value < static_cast(INT64_MIN)) + return INT64_MIN; + + return static_cast(value); + } + + /** + @brief MDB increases scale by up to 4 digits calculating avg() + */ + static inline void setScalePrecision4Avg( + unsigned int& precision, + unsigned int& scale) + { + uint32_t scaleAvailable = INT128MAXPRECISION - scale; + uint32_t precisionAvailable = INT128MAXPRECISION - precision; + scale += (scaleAvailable >= MAXSCALEINC4AVG) ? MAXSCALEINC4AVG : scaleAvailable; + precision += (precisionAvailable >= MAXSCALEINC4AVG) ? MAXSCALEINC4AVG : precisionAvailable; + } +}; + +/** + @brief The structure contains an overflow check for int128 + division. +*/ +struct DivisionOverflowCheck { + void operator()(const int128_t& x, const int128_t& y) + { + if (x == Decimal::minInt128 && y == -1) + { + throw logging::OperationOverflowExcept( + "Decimal::division produces an overflow."); + } + } + void operator()(const int64_t x, const int64_t y) + { + if (x == std::numeric_limits::min() && y == -1) + { + throw logging::OperationOverflowExcept( + "Decimal::division produces an overflow."); + } + } +}; + +/** + @brief The structure contains an overflow check for int128 + and int64_t multiplication. +*/ +struct MultiplicationOverflowCheck { + void operator()(const int128_t& x, const int128_t& y) + { + if (x * y / y != x) + { + throw logging::OperationOverflowExcept( + "Decimal::multiplication or scale multiplication \ +produces an overflow."); + } + } + bool operator()(const int128_t& x, const int128_t& y, int128_t& r) + { + if ((r = x * y) / y != x) + { + throw logging::OperationOverflowExcept( + "Decimal::multiplication or scale multiplication \ +produces an overflow."); + } + return true; + } + void operator()(const int64_t x, const int64_t y) + { + if (x * y / y != x) + { + throw logging::OperationOverflowExcept( + "Decimal::multiplication or scale multiplication \ +produces an overflow."); + } + } + bool operator()(const int64_t x, const int64_t y, int64_t& r) + { + if ((r = x * y) / y != x) + { + throw logging::OperationOverflowExcept( + "Decimal::multiplication or scale multiplication \ +produces an overflow."); + } + return true; + } +}; + +/** + @brief The strucuture runs an empty overflow check for int128 + multiplication operation. +*/ +struct MultiplicationNoOverflowCheck { + void operator()(const int128_t& x, const int128_t& y, int128_t& r) + { + r = x * y; + } +}; + +/** + @brief The structure contains an overflow check for int128 + and int64 addition. +*/ +struct AdditionOverflowCheck { + void operator()(const int128_t& x, const int128_t& y) + { + if ((y > 0 && x > Decimal::maxInt128 - y) + || (y < 0 && x < Decimal::minInt128 - y)) + { + throw logging::OperationOverflowExcept( + "Decimal::addition produces an overflow."); + } + } + void operator()(const int64_t x, const int64_t y) + { + if ((y > 0 && x > std::numeric_limits::max() - y) + || (y < 0 && x < std::numeric_limits::min() - y)) + { + throw logging::OperationOverflowExcept( + "Decimal::addition produces an overflow."); + } + } +}; + +/** + @brief The structure contains an overflow check for int128 + subtraction. +*/ +struct SubtractionOverflowCheck { + void operator()(const int128_t& x, const int128_t& y) + { + if ((y > 0 && x < Decimal::minInt128 + y) + || (y < 0 && x > Decimal::maxInt128 + y)) + { + throw logging::OperationOverflowExcept( + "Decimal::subtraction produces an overflow."); + } + } + void operator()(const int64_t x, const int64_t y) + { + if ((y > 0 && x < std::numeric_limits::min() + y) + || (y < 0 && x > std::numeric_limits::max() + y)) + { + throw logging::OperationOverflowExcept( + "Decimal::subtraction produces an overflow."); + } + } +}; + +/** + @brief The strucuture runs an empty overflow check for int128 + operation. +*/ +struct NoOverflowCheck { + void operator()(const int128_t& x, const int128_t& y) + { + return; + } +}; + + +/** + * @brief VDecimal type + * + */ +class VDecimal: public TSInt128 +{ + public: + VDecimal(): value(0), scale(0), precision(0) + { + } + + VDecimal(int64_t val, int8_t s, uint8_t p, const int128_t &val128 = 0) : + TSInt128(val128), + value(val), + scale(s), + precision(p) + { } + + VDecimal(int64_t unused, int8_t s, uint8_t p, const int128_t* val128Ptr) : + TSInt128(val128Ptr), + value(unused), + scale(s), + precision(p) + { } + + + int decimalComp(const VDecimal& d) const + { + lldiv_t d1 = lldiv(value, static_cast(mcs_pow_10[scale])); + lldiv_t d2 = lldiv(d.value, static_cast(mcs_pow_10[d.scale])); + + int ret = 0; + + if (d1.quot > d2.quot) + { + ret = 1; + } + else if (d1.quot < d2.quot) + { + ret = -1; + } + else + { + // rem carries the value's sign, but needs to be normalized. + int64_t s = scale - d.scale; + + if (s < 0) + { + if ((d1.rem * static_cast(mcs_pow_10[-s])) > d2.rem) + ret = 1; + else if ((d1.rem * static_cast(mcs_pow_10[-s])) < d2.rem) + ret = -1; + } + else + { + if (d1.rem > (d2.rem * static_cast(mcs_pow_10[s]))) + ret = 1; + else if (d1.rem < (d2.rem * static_cast(mcs_pow_10[s]))) + ret = -1; + } + } + + return ret; + } + + bool operator==(const VDecimal& rhs) const + { + if (precision > datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return s128Value == rhs.s128Value; + else + return (datatypes::Decimal::compare(*this, rhs) == 0); + } + else if (precision > datatypes::INT64MAXPRECISION && + rhs.precision <= datatypes::INT64MAXPRECISION) + { + const_cast(rhs).s128Value = rhs.value; + + if (scale == rhs.scale) + return s128Value == rhs.s128Value; + else + return (datatypes::Decimal::compare(*this, rhs) == 0); + } + else if (precision <= datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return (int128_t) value == rhs.s128Value; + else + return (datatypes::Decimal::compare(VDecimal(0, scale, precision, (int128_t) value), rhs) == 0); + } + else + { + if (scale == rhs.scale) + return value == rhs.value; + else + return (decimalComp(rhs) == 0); + } + } + + bool operator>(const VDecimal& rhs) const + { + if (precision > datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return s128Value > rhs.s128Value; + else + return (datatypes::Decimal::compare(*this, rhs) > 0); + } + else if (precision > datatypes::INT64MAXPRECISION && + rhs.precision <= datatypes::INT64MAXPRECISION) + { + VDecimal rhstmp(0, rhs.scale, rhs.precision, (int128_t) rhs.value); + + if (scale == rhstmp.scale) + return s128Value > rhstmp.s128Value; + else + return (datatypes::Decimal::compare(*this, rhstmp) > 0); + } + else if (precision <= datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return (int128_t) value > rhs.s128Value; + else + return (datatypes::Decimal::compare(VDecimal(0, scale, precision, (int128_t) value), rhs) > 0); + } + else + { + if (scale == rhs.scale) + return value > rhs.value; + else + return (decimalComp(rhs) > 0); + } + } + + bool operator<(const VDecimal& rhs) const + { + if (precision > datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return s128Value < rhs.s128Value; + else + return (datatypes::Decimal::compare(*this, rhs) < 0); + } + else if (precision > datatypes::INT64MAXPRECISION && + rhs.precision <= datatypes::INT64MAXPRECISION) + { + VDecimal rhstmp(0, rhs.scale, rhs.precision, (int128_t) rhs.value); + + if (scale == rhstmp.scale) + return s128Value < rhstmp.s128Value; + else + return (datatypes::Decimal::compare(*this, rhstmp) < 0); + } + else if (precision <= datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return (int128_t) value < rhs.s128Value; + else + return (datatypes::Decimal::compare(VDecimal(0, scale, precision, (int128_t) value), rhs) < 0); + } + else + { + if (scale == rhs.scale) + return value < rhs.value; + else + return (decimalComp(rhs) < 0); + } + } + + bool operator>=(const VDecimal& rhs) const + { + if (precision > datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return s128Value >= rhs.s128Value; + else + return (datatypes::Decimal::compare(*this, rhs) >= 0); + } + else if (precision > datatypes::INT64MAXPRECISION && + rhs.precision <= datatypes::INT64MAXPRECISION) + { + VDecimal rhstmp(0, rhs.scale, rhs.precision, (int128_t) rhs.value); + + if (scale == rhstmp.scale) + return s128Value >= rhstmp.s128Value; + else + return (datatypes::Decimal::compare(*this, rhstmp) >= 0); + } + else if (precision <= datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return (int128_t) value >= rhs.s128Value; + else + return (datatypes::Decimal::compare(VDecimal(0, scale, precision, (int128_t) value), rhs) >= 0); + } + else + { + if (scale == rhs.scale) + return value >= rhs.value; + else + return (decimalComp(rhs) >= 0); + } + } + + bool operator<=(const VDecimal& rhs) const + { + if (precision > datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return s128Value <= rhs.s128Value; + else + return (datatypes::Decimal::compare(*this, rhs) <= 0); + } + else if (precision > datatypes::INT64MAXPRECISION && + rhs.precision <= datatypes::INT64MAXPRECISION) + { + VDecimal rhstmp(0, rhs.scale, rhs.precision, (int128_t) rhs.value); + + if (scale == rhstmp.scale) + return s128Value <= rhstmp.s128Value; + else + return (datatypes::Decimal::compare(*this, rhstmp) <= 0); + } + else if (precision <= datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return (int128_t) value <= rhs.s128Value; + else + return (datatypes::Decimal::compare(VDecimal(0, scale, precision, (int128_t) value), rhs) <= 0); + } + else + { + if (scale == rhs.scale) + return value <= rhs.value; + else + return (decimalComp(rhs) <= 0); + } + } + + bool operator!=(const VDecimal& rhs) const + { + if (precision > datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return s128Value != rhs.s128Value; + else + return (datatypes::Decimal::compare(*this, rhs) != 0); + } + else if (precision > datatypes::INT64MAXPRECISION && + rhs.precision <= datatypes::INT64MAXPRECISION) + { + VDecimal rhstmp(0, rhs.scale, rhs.precision, (int128_t) rhs.value); + + if (scale == rhstmp.scale) + return s128Value != rhstmp.s128Value; + else + return (datatypes::Decimal::compare(*this, rhstmp) != 0); + } + else if (precision <= datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return (int128_t) value != rhs.s128Value; + else + return (datatypes::Decimal::compare(VDecimal(0, scale, precision, (int128_t) value), rhs) != 0); + } + else + { + if (scale == rhs.scale) + return value != rhs.value; + else + return (decimalComp(rhs) != 0); + } + } + + inline bool isTSInt128ByPrecision() const + { + return precision > INT64MAXPRECISION + && precision <= INT128MAXPRECISION; + } + // hasTSInt128 explicitly tells to print int128 out in cases + // where precision can't detect decimal type properly, e.g. + // DECIMAL(10)/DECIMAL(38) + std::string toString(bool hasTSInt128 = false) const; + friend std::ostream& operator<<(std::ostream& os, const VDecimal& dec); + + int64_t value; + int8_t scale; // 0~38 + uint8_t precision; // 1~38 + + // STRICTLY for unit tests!!! + void setTSInt64Value(const int64_t x) { value = x; } + void setTSInt128Value(const int128_t& x) { s128Value = x; } + +private: + uint8_t writeIntPart(const int128_t& x, + char* buf, + const uint8_t buflen) const; + uint8_t writeFractionalPart(const int128_t& x, + char* buf, + const uint8_t buflen) const; + std::string toStringTSInt128WithScale() const; + std::string toStringTSInt64() const; + +}; + +} //end of namespace +#endif diff --git a/datatypes/mcs_int128.cpp b/datatypes/mcs_int128.cpp new file mode 100644 index 000000000..d94f5f1bb --- /dev/null +++ b/datatypes/mcs_int128.cpp @@ -0,0 +1,178 @@ +/* + Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. +*/ + +#include + +#include "mcs_int128.h" +#include "exceptclasses.h" + +namespace datatypes +{ + // The method converts a wide decimal s128Value to a double. + inline double TSInt128::getDoubleFromWideDecimal() + { + return getDoubleFromFloat128(static_cast<__float128>(s128Value)); + } + + // The method converts a wide decimal s128Value to a long double. + inline long double TSInt128::getLongDoubleFromWideDecimal() + { + return getLongDoubleFromFloat128(static_cast<__float128>(s128Value)); + } + + uint8_t TSInt128::printPodParts(char* buf, + const int128_t& high, + const int128_t& mid, + const int128_t& low) const + { + char* p = buf; + // pod[0] is low 8 bytes, pod[1] is high 8 bytes + const uint64_t* high_pod = reinterpret_cast(&high); + const uint64_t* mid_pod = reinterpret_cast(&mid); + const uint64_t* low_pod = reinterpret_cast(&low); + + if (high_pod[0] != 0) + { + p += sprintf(p, "%lu", high_pod[0]); + p += sprintf(p, "%019lu", mid_pod[0]); + p += sprintf(p, "%019lu", low_pod[0]); + } + else if (mid_pod[0] != 0) + { + p += sprintf(p, "%lu", mid_pod[0]); + p += sprintf(p, "%019lu", low_pod[0]); + } + else + { + p += sprintf(p, "%lu", low_pod[0]); + } + return p - buf; + } + + // This method writes unsigned integer representation of TSInt128 + uint8_t TSInt128::writeIntPart(const int128_t& x, + char* buf, + const uint8_t buflen) const + { + char* p = buf; + int128_t high = 0, mid = 0, low = 0; + uint64_t maxUint64divisor = 10000000000000000000ULL; + + low = x % maxUint64divisor; + int128_t value = x / maxUint64divisor; + mid = value % maxUint64divisor; + high = value / maxUint64divisor; + + p += printPodParts(p, high, mid, low); + uint8_t written = p - buf; + if (buflen <= written) + { + throw logging::QueryDataExcept("TSInt128::writeIntPart() char buffer overflow.", + logging::formatErr); + } + + return written; + } + + // conversion to std::string + std::string TSInt128::toString() const + { + if (isNull()) + { + return std::string("NULL"); + } + + if (isEmpty()) + { + return std::string("EMPTY"); + } + + int128_t tempValue = s128Value; + char buf[TSInt128::MAXLENGTH16BYTES]; + uint8_t left = sizeof(buf); + char* p = buf; + // sign + if (tempValue < static_cast(0)) + { + *p++ = '-'; + tempValue *= -1; + left--; + } + // integer part + // reduce the size by one to account for \0 + left--; + p += writeIntPart(tempValue, p, left); + *p = '\0'; + + return std::string(buf); + } + + std::ostream& operator<<(std::ostream& os, const TSInt128& x) + { + os << x.toString(); + return os; + } + + // The method converts a wide decimal s128Value to an int64_t, + // saturating the s128Value if necessary. + inline int64_t TSInt128::getInt64FromWideDecimal() + { + if (s128Value > static_cast(INT64_MAX)) + return INT64_MAX; + else if (s128Value < static_cast(INT64_MIN)) + return INT64_MIN; + + return static_cast(s128Value); + } + + // The method converts a wide decimal s128Value to an uint32_t. + inline uint32_t TSInt128::getUInt32FromWideDecimal() + { + if (s128Value > static_cast(UINT32_MAX)) + return UINT32_MAX; + else if (s128Value < 0) + return 0; + + return static_cast(s128Value); + } + + // The method converts a wide decimal s128Value to an uint64_t. + inline uint64_t TSInt128::getUInt64FromWideDecimal() + { + if (s128Value > static_cast(UINT64_MAX)) + return UINT64_MAX; + else if (s128Value < 0) + return 0; + + return static_cast(s128Value); + } + + // The method converts a wide decimal s128Value to an int32_t. + inline int32_t TSInt128::getInt32FromWideDecimal() + { + if (s128Value > static_cast(INT32_MAX)) + return INT32_MAX; + else if (s128Value < static_cast(INT32_MIN)) + return INT32_MIN; + + return static_cast(s128Value); + } + +} // end of namespace datatypes +// vim:ts=2 sw=2: diff --git a/datatypes/mcs_int128.h b/datatypes/mcs_int128.h new file mode 100644 index 000000000..3d508420a --- /dev/null +++ b/datatypes/mcs_int128.h @@ -0,0 +1,248 @@ +/* + Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. +*/ +#ifndef MCS_INT128_H_INCLUDED +#define MCS_INT128_H_INCLUDED + +#include +#include +#include +#include +#include + +// Inline asm has three argument lists: output, input and clobber list +#if defined(__GNUC__) && (__GNUC___ > 7) + #define MACRO_VALUE_PTR_128(dst, \ + dst_restrictions, \ + src, \ + src_restrictions, \ + clobb) \ + __asm__ volatile("movups %1,%0" \ + :dst_restrictions ( *(dst) ) \ + :src_restrictions ( (src) ) \ + :clobb \ + ); + #define MACRO_PTR_PTR_128(dst, \ + dst_restrictions, \ + src, \ + src_restrictions, \ + clobb) \ + ::memcpy((dst), (src), sizeof(int128_t)); + +#else + #define MACRO_VALUE_PTR_128(dst, \ + dst_restrictions, \ + src, \ + src_restrictions, \ + clobb) \ + __asm__ volatile("movups %1,%0" \ + :dst_restrictions ( *(dst) ) \ + :src_restrictions ( (src) ) \ + :clobb \ + ); + #define MACRO_PTR_PTR_128(dst, \ + dst_restrictions, \ + src, \ + src_restrictions, \ + clobb) \ + __asm__ volatile("movdqu %1,%%xmm0;" \ + "movups %%xmm0,%0;" \ + :dst_restrictions ( *(dst) ) \ + :src_restrictions ( *(src) ) \ + :"memory", clobb \ + ); +#endif + +namespace datatypes +{ + +using int128_t = __int128; +using uint128_t = unsigned __int128; + + +// Type traits +template +struct is_allowed_numeric { + static const bool value = false; +}; + +template <> +struct is_allowed_numeric { + static const bool value = true; +}; + +template <> +struct is_allowed_numeric { + static const bool value = true; +}; + +template +struct is_int128_t { + static const bool value = false; +}; + +template<> +struct is_int128_t { + static const bool value = true; +}; + +template +struct is_uint128_t { + static const bool value = false; +}; + +template<> +struct is_uint128_t { + static const bool value = true; +}; + +// The method converts a __float128 s128Value to a double. +static inline double getDoubleFromFloat128(const __float128& value) +{ + if (value > static_cast<__float128>(DBL_MAX)) + return DBL_MAX; + else if (value < -static_cast<__float128>(DBL_MAX)) + return -DBL_MAX; + + return static_cast(value); +} + + +// The method converts a __float128 value to a long double. +static inline long double getLongDoubleFromFloat128(const __float128& value) +{ + if (value > static_cast<__float128>(LDBL_MAX)) + return LDBL_MAX; + else if (value < -static_cast<__float128>(LDBL_MAX)) + return -LDBL_MAX; + + return static_cast(value); +} + + +class TSInt128 +{ + public: + static constexpr uint8_t MAXLENGTH16BYTES = 42; + static constexpr int128_t NullValue = int128_t(0x8000000000000000LL) << 64; + static constexpr int128_t EmptyValue = (int128_t(0x8000000000000000LL) << 64) + 1; + + + // A variety of ctors for aligned and unaligned arguments + TSInt128(): s128Value(0) { } + + // aligned argument + TSInt128(const int128_t& x) { s128Value = x; } + + // unaligned argument + TSInt128(const int128_t* x) { assignPtrPtr(&s128Value, x); } + + // unaligned argument + TSInt128(const unsigned char* x) { assignPtrPtr(&s128Value, x); } + + // Method returns max length of a string representation + static constexpr uint8_t maxLength() + { + return TSInt128::MAXLENGTH16BYTES; + } + + // Checks if the value is NULL + inline bool isNull() const + { + return s128Value == NullValue; + } + + // Checks if the value is Empty + inline bool isEmpty() const + { + return s128Value == EmptyValue; + } + + // The method copies 16 bytes from one memory cell + // into another using memcpy or SIMD. + // memcpy in gcc >= 7 is replaced with SIMD instructions + template + static inline void assignPtrPtr(D* dst, const S* src) + { + MACRO_PTR_PTR_128(dst, "=m", src, "m", "xmm0") + } + + template + static inline void storeUnaligned(D* dst, const int128_t& src) + { + MACRO_VALUE_PTR_128(dst, "=m", src, "x", "memory") + } + + // operators + template::value> > + inline bool operator<(const T& x) const + { + return s128Value < static_cast(x); + } + + template::value> > + inline bool operator==(const T& x) const + { + return s128Value == static_cast(x); + } + + // print int128_t parts represented as PODs + uint8_t printPodParts(char* buf, + const int128_t& high, + const int128_t& mid, + const int128_t& low) const; + + // writes integer part of dec into a buffer + uint8_t writeIntPart(const int128_t& x, + char* buf, + const uint8_t buflen) const; + + // string representation of TSInt128 + std::string toString() const; + + friend std::ostream& operator<<(std::ostream& os, const TSInt128& x); + + // The method converts a wide decimal s128Value to a double. + inline double getDoubleFromWideDecimal(); + + // The method converts a wide decimal s128Value to a long double. + inline long double getLongDoubleFromWideDecimal(); + + // The method converts a wide decimal s128Value to an int64_t, + // saturating the s128Value if necessary. + inline int64_t getInt64FromWideDecimal(); + + // The method converts a wide decimal s128Value to an uint32_t. + inline uint32_t getUInt32FromWideDecimal(); + + // The method converts a wide decimal s128Value to an uint64_t. + inline uint64_t getUInt64FromWideDecimal(); + + // The method converts a wide decimal s128Value to an int32_t. + inline int32_t getInt32FromWideDecimal(); + + int128_t s128Value; + }; // end of class + + +} //end of namespace datatypes + +#endif // MCS_TSINT128_H_INCLUDED +// vim:ts=2 sw=2: diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index b4034c732..87739ba21 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -156,6 +156,7 @@ REFERENCES {return REFERENCES;} RENAME {return RENAME;} RESTRICT {return RESTRICT;} SESSION_USER {return SESSION_USER;} +SIGNED {return SIGNED;} SYSTEM_USER {return SYSTEM_USER;} SET {return SET;} SMALLINT {return SMALLINT;} @@ -188,6 +189,7 @@ LONGTEXT {return LONGTEXT;} BOOL {return BOOL;} BOOLEAN {return BOOLEAN;} MEDIUMINT {return MEDIUMINT;} +ZEROFILL {return ZEROFILL;} \n { lineno++;} diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index f95f3e3fb..bb1b1327a 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -104,15 +104,17 @@ char* copy_string(const char *str); %token ACTION ADD ALTER AUTO_INCREMENT BIGINT BIT BLOB IDB_BLOB CASCADE IDB_CHAR CHARACTER CHECK CLOB COLUMN +BOOL BOOLEAN COLUMNS COMMENT CONSTRAINT CONSTRAINTS CREATE CURRENT_USER DATETIME DEC DECIMAL DEFAULT DEFERRABLE DEFERRED IDB_DELETE DROP ENGINE FOREIGN FULL IMMEDIATE INDEX INITIALLY IDB_INT INTEGER KEY LONGBLOB LONGTEXT -MATCH MAX_ROWS MEDIUMBLOB MEDIUMTEXT +MATCH MAX_ROWS MEDIUMBLOB MEDIUMTEXT MEDIUMINT MIN_ROWS MODIFY NO NOT NULL_TOK NUMBER NUMERIC ON PARTIAL PRECISION PRIMARY REFERENCES RENAME RESTRICT SET SMALLINT TABLE TEXT TINYBLOB TINYTEXT -TINYINT TO UNIQUE UNSIGNED UPDATE USER SESSION_USER SYSTEM_USER VARCHAR VARBINARY +TINYINT TO UNIQUE UNSIGNED UPDATE USER SESSION_USER SIGNED SYSTEM_USER VARCHAR VARBINARY VARYING WITH ZONE DOUBLE IDB_FLOAT REAL CHARSET COLLATE IDB_IF EXISTS CHANGE TRUNCATE -BOOL BOOLEAN MEDIUMINT TIMESTAMP +TIMESTAMP +ZEROFILL %token DQ_IDENT IDENT FCONST SCONST CP_SEARCH_CONDITION_TEXT ICONST DATE TIME @@ -196,6 +198,8 @@ BOOL BOOLEAN MEDIUMINT TIMESTAMP %type opt_display_precision_scale_null %type opt_if_exists %type opt_if_not_exists +%type opt_signed +%type opt_zerofill %type trunc_table_statement %type rename_table_statement %type ident @@ -992,16 +996,14 @@ exact_numeric_type: $2->fLength = DDLDatatypeLength[DDL_UNSIGNED_NUMERIC]; $$ = $2; } - | DECIMAL opt_precision_scale + | DECIMAL opt_precision_scale opt_signed { $2->fType = DDL_DECIMAL; -/* $2->fLength = DDLDatatypeLength[DDL_DECIMAL]; */ $$ = $2; } - | DECIMAL opt_precision_scale UNSIGNED + | DECIMAL opt_precision_scale UNSIGNED opt_zerofill { $2->fType = DDL_UNSIGNED_DECIMAL; -/* $3->fLength = DDLDatatypeLength[DDL_DECIMAL]; */ $$ = $2; } | NUMBER opt_precision_scale @@ -1096,124 +1098,132 @@ opt_precision_scale: | {$$ = new ColumnType(10,0);} ; -opt_display_width: - '(' ICONST ')' {$$ = NULL;} - | {$$ = NULL;} - ; +opt_signed: + SIGNED {$$ = NULL;} + | {$$ = NULL;} -approximate_numeric_type: - DOUBLE opt_display_precision_scale_null - { - $$ = new ColumnType(DDL_DOUBLE); - $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; - } - | DOUBLE opt_display_precision_scale_null UNSIGNED - { - $$ = new ColumnType(DDL_UNSIGNED_DOUBLE); - $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; - } - | DOUBLE PRECISION opt_display_precision_scale_null - { - $$ = new ColumnType(DDL_DOUBLE); - $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; - } - | DOUBLE PRECISION opt_display_precision_scale_null UNSIGNED - { - $$ = new ColumnType(DDL_UNSIGNED_DOUBLE); - $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; - } - | REAL opt_display_precision_scale_null - { - $$ = new ColumnType(DDL_DOUBLE); - $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; - } - | REAL opt_display_precision_scale_null UNSIGNED - { - $$ = new ColumnType(DDL_UNSIGNED_DOUBLE); - $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; - } - | IDB_FLOAT opt_display_precision_scale_null - { - $$ = new ColumnType(DDL_FLOAT); - $$->fLength = DDLDatatypeLength[DDL_FLOAT]; - } - | IDB_FLOAT opt_display_precision_scale_null UNSIGNED - { - $$ = new ColumnType(DDL_UNSIGNED_FLOAT); - $$->fLength = DDLDatatypeLength[DDL_FLOAT]; - } - ; + opt_zerofill: + ZEROFILL {$$ = NULL;} + | {$$ = NULL;} -opt_display_precision_scale_null: - '(' ICONST ')' {$$ = NULL;} - | - '(' ICONST ',' ICONST ')' {$$ = NULL;} + opt_display_width: + '(' ICONST ')' {$$ = NULL;} | {$$ = NULL;} ; - -literal: - ICONST - | string_literal - | FCONST - ; -datetime_type: - DATETIME opt_time_precision - { - $$ = new ColumnType(DDL_DATETIME); - $$->fLength = DDLDatatypeLength[DDL_DATETIME]; - $$->fPrecision = $2; - } - | - DATE - { - $$ = new ColumnType(DDL_DATE); - $$->fLength = DDLDatatypeLength[DDL_DATE]; - } - | - TIME opt_time_precision - { - $$ = new ColumnType(DDL_TIME); - $$->fLength = DDLDatatypeLength[DDL_TIME]; - $$->fPrecision = $2; - } - | - TIMESTAMP opt_time_precision - { - $$ = new ColumnType(DDL_TIMESTAMP); - $$->fLength = DDLDatatypeLength[DDL_TIMESTAMP]; - $$->fPrecision = $2; - } + approximate_numeric_type: + DOUBLE opt_display_precision_scale_null + { + $$ = new ColumnType(DDL_DOUBLE); + $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; + } + | DOUBLE opt_display_precision_scale_null UNSIGNED + { + $$ = new ColumnType(DDL_UNSIGNED_DOUBLE); + $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; + } + | DOUBLE PRECISION opt_display_precision_scale_null + { + $$ = new ColumnType(DDL_DOUBLE); + $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; + } + | DOUBLE PRECISION opt_display_precision_scale_null UNSIGNED + { + $$ = new ColumnType(DDL_UNSIGNED_DOUBLE); + $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; + } + | REAL opt_display_precision_scale_null + { + $$ = new ColumnType(DDL_DOUBLE); + $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; + } + | REAL opt_display_precision_scale_null UNSIGNED + { + $$ = new ColumnType(DDL_UNSIGNED_DOUBLE); + $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; + } + | IDB_FLOAT opt_display_precision_scale_null + { + $$ = new ColumnType(DDL_FLOAT); + $$->fLength = DDLDatatypeLength[DDL_FLOAT]; + } + | IDB_FLOAT opt_display_precision_scale_null UNSIGNED + { + $$ = new ColumnType(DDL_UNSIGNED_FLOAT); + $$->fLength = DDLDatatypeLength[DDL_FLOAT]; + } + ; -opt_time_precision: - '(' ICONST ')' {$$ = atoi($2);} - | {$$ = -1;} - ; + opt_display_precision_scale_null: + '(' ICONST ')' {$$ = NULL;} + | + '(' ICONST ',' ICONST ')' {$$ = NULL;} + | {$$ = NULL;} + ; + + literal: + ICONST + | string_literal + | FCONST + ; -drop_column_def: - DROP column_name drop_behavior {$$ = new AtaDropColumn($2, $3);} - | DROP COLUMN column_name drop_behavior {$$ = new AtaDropColumn($3, $4);} - | DROP COLUMN '(' column_name_list ')' {$$ = new AtaDropColumns($4);} - | DROP '(' column_name_list ')' {$$ = new AtaDropColumns($3);} - | DROP COLUMNS '(' column_name_list ')' {$$ = new AtaDropColumns($4);} - ; + datetime_type: + DATETIME opt_time_precision + { + $$ = new ColumnType(DDL_DATETIME); + $$->fLength = DDLDatatypeLength[DDL_DATETIME]; + $$->fPrecision = $2; + } + | + DATE + { + $$ = new ColumnType(DDL_DATE); + $$->fLength = DDLDatatypeLength[DDL_DATE]; + } + | + TIME opt_time_precision + { + $$ = new ColumnType(DDL_TIME); + $$->fLength = DDLDatatypeLength[DDL_TIME]; + $$->fPrecision = $2; + } + | + TIMESTAMP opt_time_precision + { + $$ = new ColumnType(DDL_TIMESTAMP); + $$->fLength = DDLDatatypeLength[DDL_TIMESTAMP]; + $$->fPrecision = $2; + } -drop_behavior: - CASCADE {$$ = DDL_CASCADE;} - | RESTRICT {$$ = DDL_RESTRICT;} - | {$$ = DDL_NO_ACTION;} - ; + opt_time_precision: + '(' ICONST ')' {$$ = atoi($2);} + | {$$ = -1;} + ; -alter_column_def: - ALTER opt_column column_name SET default_clause {$$ = new AtaSetColumnDefault($3, $5);} - | ALTER opt_column column_name DROP DEFAULT {$$ = new AtaDropColumnDefault($3);} - ; + drop_column_def: + DROP column_name drop_behavior {$$ = new AtaDropColumn($2, $3);} + | DROP COLUMN column_name drop_behavior {$$ = new AtaDropColumn($3, $4);} + | DROP COLUMN '(' column_name_list ')' {$$ = new AtaDropColumns($4);} + | DROP '(' column_name_list ')' {$$ = new AtaDropColumns($3);} + | DROP COLUMNS '(' column_name_list ')' {$$ = new AtaDropColumns($4);} + ; -opt_column: - COLUMN - | - ; + drop_behavior: + CASCADE {$$ = DDL_CASCADE;} + | RESTRICT {$$ = DDL_RESTRICT;} + | {$$ = DDL_NO_ACTION;} + ; -%% + alter_column_def: + ALTER opt_column column_name SET default_clause {$$ = new AtaSetColumnDefault($3, $5);} + | ALTER opt_column column_name DROP DEFAULT {$$ = new AtaDropColumnDefault($3);} + ; + + opt_column: + COLUMN + | + ; + + %% diff --git a/dbcon/ddlpackage/ddlpkg.cpp b/dbcon/ddlpackage/ddlpkg.cpp index 24404c52b..90bfa45a9 100644 --- a/dbcon/ddlpackage/ddlpkg.cpp +++ b/dbcon/ddlpackage/ddlpkg.cpp @@ -26,6 +26,7 @@ #define DDLPKG_DLLEXPORT #include "ddlpkg.h" #undef DDLPKG_DLLEXPORT +#include "../../utils/common/columnwidth.h" namespace ddlpackage { @@ -61,32 +62,6 @@ ostream& operator<<(ostream& os, const QualifiedName& qname) return os; } - -/** @brief Map a DECIMAL precision to data width in bytes */ -unsigned int precision_width(unsigned p) -{ - switch (p) - { - case 1: - case 2: - return 1; - - case 3: - case 4: - return 2; - - case 5: - case 6: - case 7: - case 8: - case 9: - return 4; - - default: - return 8; - } -} - ColumnType::ColumnType(int prec, int scale) : fType(DDL_INVALID_DATATYPE), fLength(0), @@ -94,7 +69,7 @@ ColumnType::ColumnType(int prec, int scale) : fScale(scale), fWithTimezone(false) { - fLength = precision_width(fPrecision); + fLength = utils::widthByPrecision(fPrecision); } ColumnType::ColumnType(int type) : @@ -141,19 +116,6 @@ ColumnType::ColumnType(int type) : break; } } -#if 0 -ColumnType::ColumnType(int type, int length, int precision, int scale, int compressiontype, const char* autoIncrement, int64_t nextValue, bool withTimezone) : - fType(type), - fLength(length), - fPrecision(precision), - fScale(scale), - fWithTimezone(withTimezone), - fCompressiontype(compressiontype), - fAutoincrement(autoIncrement), - fNextvalue(nextValue) -{ -} -#endif ColumnConstraintDef::ColumnConstraintDef(DDL_CONSTRAINTS type) : SchemaObject(), fDeferrable(false), @@ -199,14 +161,12 @@ void ColumnDef::convertDecimal() } else if ((fType->fPrecision > 0) && (fType->fPrecision < 3)) { - //dataType = CalpontSystemCatalog::TINYINT; fType->fType = DDL_TINYINT; fType->fLength = 1; } else if (fType->fPrecision < 5 && (fType->fPrecision > 2)) { - //dataType = CalpontSystemCatalog::SMALLINT; fType->fType = DDL_SMALLINT; fType->fLength = 2; } @@ -217,15 +177,18 @@ void ColumnDef::convertDecimal() } else if (fType->fPrecision > 6 && fType->fPrecision < 10) { - //dataType = CalpontSystemCatalog::INT; fType->fType = DDL_INT; fType->fLength = 4; } else if (fType->fPrecision > 9 && fType->fPrecision < 19) { - //dataType = CalpontSystemCatalog::BIGINT; fType->fType = DDL_BIGINT; fType->fLength = 8; } + else if (fType->fPrecision > 18 && fType->fPrecision < 39) + { + fType->fType = DDL_BINARY; + fType->fLength = 16; + } } -} +} // end of namespace diff --git a/dbcon/ddlpackage/ddlpkg.h b/dbcon/ddlpackage/ddlpkg.h index 0c7edd389..821efa440 100644 --- a/dbcon/ddlpackage/ddlpkg.h +++ b/dbcon/ddlpackage/ddlpkg.h @@ -238,6 +238,7 @@ enum DDL_DATATYPES DDL_TEXT, DDL_TIME, DDL_TIMESTAMP, + DDL_BINARY, DDL_INVALID_DATATYPE }; @@ -276,7 +277,8 @@ const std::string DDLDatatypeString[] = "unsigned-numeric", "text", "time", - "timestamp" + "timestamp", + "binary", "" }; diff --git a/dbcon/ddlpackageproc/altertableprocessor.cpp b/dbcon/ddlpackageproc/altertableprocessor.cpp index 743779c68..1e0fa76e2 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.cpp +++ b/dbcon/ddlpackageproc/altertableprocessor.cpp @@ -247,6 +247,11 @@ bool typesAreSame(const CalpontSystemCatalog::ColType& colType, const ColumnType break; + case (CalpontSystemCatalog::BINARY): + if (newType.fType == DDL_BINARY && colType.colWidth == newType.fLength) return true; + + break; + default: break; } diff --git a/dbcon/ddlpackageproc/createindexprocessor.cpp b/dbcon/ddlpackageproc/createindexprocessor.cpp index 30b8eeaf7..92787f02f 100644 --- a/dbcon/ddlpackageproc/createindexprocessor.cpp +++ b/dbcon/ddlpackageproc/createindexprocessor.cpp @@ -220,12 +220,12 @@ CreateIndexProcessor::DDLResult CreateIndexProcessor::processPackage(ddlpackage: } else if (CONSTRAINTPRIM_COL == column.tableColName.column) { - colTuple.data = getNullValueForType(column.colType); + colTuple.data =column.colType.getNullValueForType(); isNull = true; } else if (CONSTRAINTTEXT_COL == column.tableColName.column) { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); isNull = true; } else if (INDEXNAME_COL == column.tableColName.column) @@ -235,7 +235,7 @@ CreateIndexProcessor::DDLResult CreateIndexProcessor::processPackage(ddlpackage: } else { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); isNull = true; } @@ -336,7 +336,7 @@ CreateIndexProcessor::DDLResult CreateIndexProcessor::processPackage(ddlpackage: } else { - colTupleCol.data = getNullValueForType(column.colType); + colTupleCol.data = column.colType.getNullValueForType(); isNull = true; } diff --git a/dbcon/ddlpackageproc/createtableprocessor.cpp b/dbcon/ddlpackageproc/createtableprocessor.cpp index a49dda249..e8f615c93 100644 --- a/dbcon/ddlpackageproc/createtableprocessor.cpp +++ b/dbcon/ddlpackageproc/createtableprocessor.cpp @@ -573,6 +573,10 @@ keepGoing: { colDefPtr->fType->fLength = 8; } + else if (colDefPtr->fType->fPrecision > 18 && colDefPtr->fType->fPrecision < 39) + { + colDefPtr->fType->fLength = 16; + } } bytestream << (fStartingColOID + (colNum++) + 1); diff --git a/dbcon/ddlpackageproc/ddlindexpopulator.h b/dbcon/ddlpackageproc/ddlindexpopulator.h index de623aa76..71910c32d 100644 --- a/dbcon/ddlpackageproc/ddlindexpopulator.h +++ b/dbcon/ddlpackageproc/ddlindexpopulator.h @@ -291,7 +291,7 @@ private: : DDLPackageProcessor(), fType(ctype) {} boost::any operator()(execplan::CalpontSystemCatalog::ColType& ctype) { - return getNullValueForType(fType); + return ctype.getNullValueForType(); } const execplan::CalpontSystemCatalog::ColType& fType; }; diff --git a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp index 0419fc70e..0f21db9cb 100644 --- a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp +++ b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp @@ -27,6 +27,7 @@ using namespace std; #include "ddlpackageprocessor.h" +#include "mcs_datatype.h" #include "dataconvert.h" using namespace dataconvert; #include "calpontselectexecutionplan.h" @@ -251,6 +252,10 @@ execplan::CalpontSystemCatalog::ColDataType DDLPackageProcessor::convertDataType case ddlpackage::DDL_TEXT: colDataType = CalpontSystemCatalog::TEXT; break; + + case ddlpackage::DDL_BINARY: + colDataType = CalpontSystemCatalog::BINARY; + break; default: throw runtime_error("Unsupported datatype!"); @@ -386,237 +391,6 @@ char DDLPackageProcessor::getConstraintCode(ddlpackage::DDL_CONSTRAINTS type) return constraint_type; } -boost::any -DDLPackageProcessor::getNullValueForType(const execplan::CalpontSystemCatalog::ColType& colType) -{ - boost::any value; - - switch (colType.colDataType) - { - case execplan::CalpontSystemCatalog::BIT: - break; - - case execplan::CalpontSystemCatalog::TINYINT: - { - char tinyintvalue = joblist::TINYINTNULL; - value = tinyintvalue; - - } - break; - - case execplan::CalpontSystemCatalog::SMALLINT: - { - short smallintvalue = joblist::SMALLINTNULL; - value = smallintvalue; - } - break; - - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - { - int intvalue = joblist::INTNULL; - value = intvalue; - } - break; - - case execplan::CalpontSystemCatalog::BIGINT: - { - long long bigint = joblist::BIGINTNULL; - value = bigint; - } - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - if (colType.colWidth <= 4) - { - short smallintvalue = joblist::SMALLINTNULL; - value = smallintvalue; - } - else if (colType.colWidth <= 9) - { - int intvalue = joblist::INTNULL; - value = intvalue; - } - else if (colType.colWidth <= 18) - { - long long eightbyte = joblist::BIGINTNULL; - value = eightbyte; - } - else - { - WriteEngine::Token nullToken; - value = nullToken; - } - } - break; - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - uint32_t jlfloatnull = joblist::FLOATNULL; - float* fp = reinterpret_cast(&jlfloatnull); - value = *fp; - } - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - uint64_t jldoublenull = joblist::DOUBLENULL; - double* dp = reinterpret_cast(&jldoublenull); - value = *dp; - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int d = joblist::DATENULL; - value = d; - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - long long d = joblist::DATETIMENULL; - value = d; - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - long long d = joblist::TIMENULL; - value = d; - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - long long d = joblist::TIMESTAMPNULL; - value = d; - } - break; - - case execplan::CalpontSystemCatalog::CHAR: - { - std::string charnull; - - if (colType.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - { - //charnull = joblist::CHAR1NULL; - charnull = "\376"; - value = charnull; - } - else if (colType.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) - { - //charnull = joblist::CHAR2NULL; - charnull = "\377\376"; - value = charnull; - } - else if (colType.colWidth <= execplan::CalpontSystemCatalog::FOUR_BYTE) - { - //charnull = joblist::CHAR4NULL; - charnull = "\377\377\377\376"; - value = charnull; - } - else - { - WriteEngine::Token nullToken; - value = nullToken; - } - - } - break; - - case execplan::CalpontSystemCatalog::VARCHAR: - { - std::string charnull; - - if (colType.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - { - //charnull = joblist::CHAR2NULL; - charnull = "\377\376"; - value = charnull; - } - else if (colType.colWidth < execplan::CalpontSystemCatalog::FOUR_BYTE) - { - //charnull = joblist::CHAR4NULL; - charnull = "\377\377\377\376"; - value = charnull; - } - else - { - WriteEngine::Token nullToken; - value = nullToken; - } - - } - break; - - case execplan::CalpontSystemCatalog::VARBINARY: - { - std::string charnull; - - if (colType.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - { - //charnull = joblist::CHAR2NULL; - charnull = "\377\376"; - value = charnull; - } - else if (colType.colWidth < execplan::CalpontSystemCatalog::FOUR_BYTE) - { - //charnull = joblist::CHAR4NULL; - charnull = "\377\377\377\376"; - value = charnull; - } - else - { - WriteEngine::Token nullToken; - value = nullToken; - } - - } - break; - - case execplan::CalpontSystemCatalog::UTINYINT: - { - uint8_t utinyintvalue = joblist::UTINYINTNULL; - value = utinyintvalue; - - } - break; - - case execplan::CalpontSystemCatalog::USMALLINT: - { - uint16_t usmallintvalue = joblist::USMALLINTNULL; - value = usmallintvalue; - } - break; - - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - { - uint32_t uintvalue = joblist::UINTNULL; - value = uintvalue; - } - break; - - case execplan::CalpontSystemCatalog::UBIGINT: - { - uint64_t ubigint = joblist::UBIGINTNULL; - value = ubigint; - } - break; - - default: - throw std::runtime_error("getNullValueForType: unkown column data type"); - break; - - } - - return value; -} bool DDLPackageProcessor::isIndexConstraint(ddlpackage::DDL_CONSTRAINTS type) { @@ -1546,17 +1320,20 @@ void DDLPackageProcessor::updateSyscolumns(execplan::CalpontSystemCatalog::SCN t if (result.result != NO_ERROR) return; - WriteEngine::ColStructList colStructs; + WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; //std::vector colStructs; WriteEngine::ColStruct colStruct; + execplan::CalpontSystemCatalog::ColType colType; WriteEngine::DctnryStructList dctnryStructList; WriteEngine::DctnryValueList dctnryValueList; //Build column structure for COLUMNPOS_COL - colStruct.dataOid = OID_SYSCOLUMN_COLUMNPOS; - colStruct.colWidth = 4; + colType.columnOID = colStruct.dataOid = OID_SYSCOLUMN_COLUMNPOS; + colType.colWidth = colStruct.colWidth = 4; colStruct.tokenFlag = false; - colStruct.colDataType = CalpontSystemCatalog::INT; + colType.colDataType = colStruct.colDataType = CalpontSystemCatalog::INT; colStructs.push_back(colStruct); + cscColTypeList.push_back(colType); int error; std::string err; std::vector colOldValuesList1; @@ -1564,7 +1341,7 @@ void DDLPackageProcessor::updateSyscolumns(execplan::CalpontSystemCatalog::SCN t try { //@Bug 3051 use updateColumnRecs instead of updateColumnRec to use different value for diffrent rows. - if (NO_ERROR != (error = fWriteEngine.updateColumnRecs( txnID, colStructs, colValuesList, ridList ))) + if (NO_ERROR != (error = fWriteEngine.updateColumnRecs( txnID, cscColTypeList, colStructs, colValuesList, ridList ))) { // build the logging message WErrorCodes ec; diff --git a/dbcon/ddlpackageproc/ddlpackageprocessor.h b/dbcon/ddlpackageproc/ddlpackageprocessor.h index 5714d5cd6..84d50d805 100644 --- a/dbcon/ddlpackageproc/ddlpackageprocessor.h +++ b/dbcon/ddlpackageproc/ddlpackageprocessor.h @@ -418,12 +418,6 @@ protected: */ execplan::CalpontSystemCatalog::ColDataType convertDataType(int dataType); - /** @brief get the null representation for the given column type - * - * @param colType the column type - */ - boost::any getNullValueForType(const execplan::CalpontSystemCatalog::ColType& colType); - /** @brief return a tokenized value for the supplied data value * * @param result the result of the operation diff --git a/dbcon/dmlpackageproc/dmlpackageprocessor.cpp b/dbcon/dmlpackageproc/dmlpackageprocessor.cpp index 32684dee5..c268f238a 100644 --- a/dbcon/dmlpackageproc/dmlpackageprocessor.cpp +++ b/dbcon/dmlpackageproc/dmlpackageprocessor.cpp @@ -510,6 +510,8 @@ int DMLPackageProcessor::commitBatchAutoOnTransaction(uint64_t uniqueId, BRM::Tx aInfo.firstLbid = *iter; aInfo.max = numeric_limits::min(); // Not used aInfo.min = numeric_limits::max(); // Not used + utils::int128Min(aInfo.bigMax); // Not used + utils::int128Max(aInfo.bigMin); // Not used aInfo.seqNum = -1; cpInfos.push_back(aInfo); ++iter; diff --git a/dbcon/dmlpackageproc/dmlpackageprocessor.h b/dbcon/dmlpackageproc/dmlpackageprocessor.h index cc213c347..05c428781 100644 --- a/dbcon/dmlpackageproc/dmlpackageprocessor.h +++ b/dbcon/dmlpackageproc/dmlpackageprocessor.h @@ -478,7 +478,7 @@ protected: { if (((colType.colDataType == execplan::CalpontSystemCatalog::CHAR) && (colType.colWidth > 8)) || ((colType.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7)) - || ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 18)) + || ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 38)) || (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY) || (colType.colDataType == execplan::CalpontSystemCatalog::BLOB) || (colType.colDataType == execplan::CalpontSystemCatalog::TEXT)) diff --git a/dbcon/execplan/CMakeLists.txt b/dbcon/execplan/CMakeLists.txt index 8b7911838..798290c67 100755 --- a/dbcon/execplan/CMakeLists.txt +++ b/dbcon/execplan/CMakeLists.txt @@ -1,5 +1,5 @@ -include_directories( ${ENGINE_COMMON_INCLUDES} ${ENGINE_UTILS_UDFSDK_INCLUDE} ) +include_directories( ${ENGINE_COMMON_INCLUDES} ${ENGINE_UTILS_UDFSDK_INCLUDE} ${ENGINE_DATATYPES_INCLUDE}) ########### next target ############### @@ -48,7 +48,6 @@ add_library(execplan SHARED ${execplan_LIB_SRCS}) add_dependencies(execplan loggingcpp) -target_link_libraries(execplan ${NETSNMP_LIBRARIES} ${MARIADB_STRING_LIBS}) +target_link_libraries(execplan ${NETSNMP_LIBRARIES} ${MARIADB_STRING_LIBS} ${ENGINE_DT_LIB}) install(TARGETS execplan DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) - diff --git a/dbcon/execplan/arithmeticcolumn.cpp b/dbcon/execplan/arithmeticcolumn.cpp index 2128a47ed..e88adf899 100644 --- a/dbcon/execplan/arithmeticcolumn.cpp +++ b/dbcon/execplan/arithmeticcolumn.cpp @@ -62,7 +62,7 @@ namespace execplan */ ArithmeticColumn::ArithmeticColumn(): ReturnedColumn(), - fExpression (0) + fExpression(0) {} ArithmeticColumn::ArithmeticColumn(const string& sql, const uint32_t sessionID): @@ -314,8 +314,7 @@ const string ArithmeticColumn::toString() const oss << "expressionId=" << fExpressionId << endl; oss << "joinInfo=" << fJoinInfo << " returnAll=" << fReturnAll << " sequence#=" << fSequence << endl; - oss << "resultType=" << colDataTypeToString(fResultType.colDataType) << "|" << fResultType.colWidth << - endl; + oss << "resultType=" << colDataTypeToString(fResultType.colDataType) << "|" << fResultType.colWidth << endl; return oss.str(); } diff --git a/dbcon/execplan/arithmeticoperator.cpp b/dbcon/execplan/arithmeticoperator.cpp index 230786800..3f4fa695c 100644 --- a/dbcon/execplan/arithmeticoperator.cpp +++ b/dbcon/execplan/arithmeticoperator.cpp @@ -34,17 +34,20 @@ namespace execplan /** * Constructors/Destructors */ -ArithmeticOperator::ArithmeticOperator() : Operator() +ArithmeticOperator::ArithmeticOperator() : Operator(), + fDecimalOverflowCheck(false) { } -ArithmeticOperator::ArithmeticOperator(const string& operatorName): Operator(operatorName) +ArithmeticOperator::ArithmeticOperator(const string& operatorName): Operator(operatorName), + fDecimalOverflowCheck(false) { } ArithmeticOperator::ArithmeticOperator(const ArithmeticOperator& rhs): Operator(rhs), - fTimeZone(rhs.timeZone()) + fTimeZone(rhs.timeZone()), + fDecimalOverflowCheck(false) { } @@ -63,6 +66,7 @@ ostream& operator<<(ostream& output, const ArithmeticOperator& rhs) { output << rhs.toString(); output << "opType=" << rhs.operationType().colDataType << endl; + output << "decimalOverflowCheck=" << rhs.getOverflowCheck() << endl; return output; } @@ -73,6 +77,8 @@ void ArithmeticOperator::serialize(messageqcpp::ByteStream& b) const { b << (ObjectReader::id_t) ObjectReader::ARITHMETICOPERATOR; b << fTimeZone; + const messageqcpp::ByteStream::byte tmp = fDecimalOverflowCheck; + b << tmp; Operator::serialize(b); } @@ -80,6 +86,9 @@ void ArithmeticOperator::unserialize(messageqcpp::ByteStream& b) { ObjectReader::checkType(b, ObjectReader::ARITHMETICOPERATOR); b >> fTimeZone; + messageqcpp::ByteStream::byte tmp; + b >> tmp; + fDecimalOverflowCheck = tmp; Operator::unserialize(b); } @@ -116,70 +125,6 @@ bool ArithmeticOperator::operator!=(const TreeNode* t) const return (!(*this == t)); } -#if 0 -void ArithmeticOperator::operationType(const Type& l, const Type& r) -{ - if (l.colDataType == execplan::CalpontSystemCatalog::DECIMAL) - { - switch (r.colDataType) - { - case execplan::CalpontSystemCatalog::DECIMAL: - { - // should follow the result type that MySQL gives - fOperationType = fResultType; - break; - } - - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::BIGINT: - fOperationType.colDataType = execplan::CalpontSystemCatalog::DECIMAL; - fOperationType.scale = l.scale; - break; - - default: - fOperationType.colDataType = execplan::CalpontSystemCatalog::DOUBLE; - } - } - else if (r.colDataType == execplan::CalpontSystemCatalog::DECIMAL) - { - switch (l.colDataType) - { - case execplan::CalpontSystemCatalog::DECIMAL: - { - // should following the result type that MySQL gives based on the following logic? - // @NOTE is this trustable? - fOperationType = fResultType; - break; - } - - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::BIGINT: - fOperationType.colDataType = execplan::CalpontSystemCatalog::DECIMAL; - fOperationType.scale = r.scale; - break; - - default: - fOperationType.colDataType = execplan::CalpontSystemCatalog::DOUBLE; - } - } - else if ((l.colDataType == execplan::CalpontSystemCatalog::INT || - l.colDataType == execplan::CalpontSystemCatalog::MEDINT || - l.colDataType == execplan::CalpontSystemCatalog::TINYINT || - l.colDataType == execplan::CalpontSystemCatalog::BIGINT) && - (r.colDataType == execplan::CalpontSystemCatalog::INT || - r.colDataType == execplan::CalpontSystemCatalog::MEDINT || - r.colDataType == execplan::CalpontSystemCatalog::TINYINT || - r.colDataType == execplan::CalpontSystemCatalog::BIGINT)) - fOperationType.colDataType = execplan::CalpontSystemCatalog::BIGINT; - else - fOperationType.colDataType = execplan::CalpontSystemCatalog::DOUBLE; -} -#endif - void ArithmeticOperator::adjustResultType(const CalpontSystemCatalog::ColType& m) { if (m.colDataType != CalpontSystemCatalog::DECIMAL) diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index ddf0b7ead..042d47cd1 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -43,6 +43,7 @@ namespace execplan class ArithmeticOperator : public Operator { +using cscType = execplan::CalpontSystemCatalog::ColType; public: ArithmeticOperator(); @@ -195,12 +196,21 @@ public: return TreeNode::getBoolVal(); } void adjustResultType(const CalpontSystemCatalog::ColType& m); + inline bool getOverflowCheck() const + { + return fDecimalOverflowCheck; + } + inline void setOverflowCheck(bool check) + { + fDecimalOverflowCheck = check; + } private: template inline result_t execute(result_t op1, result_t op2, bool& isNull); inline void execute(IDB_Decimal& result, IDB_Decimal op1, IDB_Decimal op2, bool& isNull); std::string fTimeZone; + bool fDecimalOverflowCheck; }; #include "parsetree.h" @@ -236,12 +246,11 @@ inline void ArithmeticOperator::evaluate(rowgroup::Row& row, bool& isNull, Parse case execplan::CalpontSystemCatalog::LONGDOUBLE: fResult.longDoubleVal = execute(lop->getLongDoubleVal(row, isNull), rop->getLongDoubleVal(row, isNull), isNull); break; - + // WIP MCOL-641 case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: - execute (fResult.decimalVal, lop->getDecimalVal(row, isNull), rop->getDecimalVal(row, isNull), isNull); + execute(fResult.decimalVal, lop->getDecimalVal(row, isNull), rop->getDecimalVal(row, isNull), isNull); break; - default: { std::ostringstream oss; @@ -287,79 +296,152 @@ inline void ArithmeticOperator::execute(IDB_Decimal& result, IDB_Decimal op1, ID switch (fOp) { case OP_ADD: - if (result.scale == op1.scale && result.scale == op2.scale) + if (fOperationType.colWidth == datatypes::MAXDECIMALWIDTH) { - result.value = op1.value + op2.value; - break; + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::addition( + op1, op2, result); + } + else + { + datatypes::Decimal::addition( + op1, op2, result); + } + } + else if (fOperationType.colWidth == utils::MAXLEGACYWIDTH) + { + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::addition( + op1, op2, result); + } + else + { + datatypes::Decimal::addition( + op1, op2, result); + } } - - if (result.scale >= op1.scale) - op1.value *= IDB_pow[result.scale - op1.scale]; else - op1.value = (int64_t)(op1.value > 0 ? - (double)op1.value / IDB_pow[op1.scale - result.scale] + 0.5 : - (double)op1.value / IDB_pow[op1.scale - result.scale] - 0.5); - - if (result.scale >= op2.scale) - op2.value *= IDB_pow[result.scale - op2.scale]; - else - op2.value = (int64_t)(op2.value > 0 ? - (double)op2.value / IDB_pow[op2.scale - result.scale] + 0.5 : - (double)op2.value / IDB_pow[op2.scale - result.scale] - 0.5); - - result.value = op1.value + op2.value; + { + throw logging::InvalidArgumentExcept( + "Unexpected result width"); + } break; case OP_SUB: - if (result.scale == op1.scale && result.scale == op2.scale) + if (fOperationType.colWidth == datatypes::MAXDECIMALWIDTH) { - result.value = op1.value - op2.value; - break; + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::subtraction( + op1, op2, result); + } + else + { + datatypes::Decimal::subtraction( + op1, op2, result); + } + } + else if (fOperationType.colWidth == utils::MAXLEGACYWIDTH) + { + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::subtraction( + op1, op2, result); + } + else + { + datatypes::Decimal::subtraction( + op1, op2, result); + } } - - if (result.scale >= op1.scale) - op1.value *= IDB_pow[result.scale - op1.scale]; else - op1.value = (int64_t)(op1.value > 0 ? - (double)op1.value / IDB_pow[op1.scale - result.scale] + 0.5 : - (double)op1.value / IDB_pow[op1.scale - result.scale] - 0.5); - - if (result.scale >= op2.scale) - op2.value *= IDB_pow[result.scale - op2.scale]; - else - op2.value = (int64_t)(op2.value > 0 ? - (double)op2.value / IDB_pow[op2.scale - result.scale] + 0.5 : - (double)op2.value / IDB_pow[op2.scale - result.scale] - 0.5); - - result.value = op1.value - op2.value; + { + throw logging::InvalidArgumentExcept( + "Unexpected result width"); + } break; case OP_MUL: - if (result.scale >= op1.scale + op2.scale) - result.value = op1.value * op2.value * IDB_pow[result.scale - (op1.scale + op2.scale)]; + if (fOperationType.colWidth == datatypes::MAXDECIMALWIDTH) + { + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::multiplication( + op1, op2, result); + } + else + { + datatypes::Decimal::multiplication( + op1, op2, result); + } + } + else if (fOperationType.colWidth == utils::MAXLEGACYWIDTH) + { + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::multiplication( + op1, op2, result); + } + else + { + datatypes::Decimal::multiplication( + op1, op2, result); + } + } else - result.value = (int64_t)(( (op1.value > 0 && op2.value > 0) || (op1.value < 0 && op2.value < 0) ? - (double)op1.value * op2.value / IDB_pow[op1.scale + op2.scale - result.scale] + 0.5 : - (double)op1.value * op2.value / IDB_pow[op1.scale + op2.scale - result.scale] - 0.5)); - + { + throw logging::InvalidArgumentExcept( + "Unexpected result width"); + } break; case OP_DIV: - if (op2.value == 0) + if (fOperationType.colWidth == datatypes::MAXDECIMALWIDTH) { - isNull = true; - break; + if ((datatypes::Decimal::isWideDecimalTypeByPrecision(op2.precision) && op2.s128Value == 0) + || (!datatypes::Decimal::isWideDecimalTypeByPrecision(op2.precision) && op2.value == 0)) + { + isNull = true; + break; + } + + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::division( + op1, op2, result); + } + else + { + datatypes::Decimal::division( + op1, op2, result); + } } + else if (fOperationType.colWidth == utils::MAXLEGACYWIDTH) + { + if (op2.value == 0) + { + isNull = true; + break; + } - if (result.scale >= op1.scale - op2.scale) - result.value = (int64_t)(( (op1.value > 0 && op2.value > 0) || (op1.value < 0 && op2.value < 0) ? - (long double)op1.value / op2.value * IDB_pow[result.scale - (op1.scale - op2.scale)] + 0.5 : - (long double)op1.value / op2.value * IDB_pow[result.scale - (op1.scale - op2.scale)] - 0.5)); + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::division( + op1, op2, result); + } + else + { + datatypes::Decimal::division( + op1, op2, result); + } + } else - result.value = (int64_t)(( (op1.value > 0 && op2.value > 0) || (op1.value < 0 && op2.value < 0) ? - (long double)op1.value / op2.value / IDB_pow[op1.scale - op2.scale - result.scale] + 0.5 : - (long double)op1.value / op2.value / IDB_pow[op1.scale - op2.scale - result.scale] - 0.5)); - + { + throw logging::InvalidArgumentExcept( + "Unexpected result width"); + } break; default: diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index 042abcc04..18b0fb7c5 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -31,6 +31,7 @@ using namespace std; #include "messagequeue.h" #include "calpontsystemcatalog.h" +#include "dataconvert.h" #include "ddlpkg.h" #include "expressionparser.h" #include "calpontselectexecutionplan.h" @@ -226,6 +227,10 @@ const string colDataTypeToString(CalpontSystemCatalog::ColDataType cdt) return "udouble"; break; + case CalpontSystemCatalog::BINARY: + return "binary"; + break; + default: break; } @@ -6087,13 +6092,9 @@ void CalpontSystemCatalog::checkSysCatVer() } CalpontSystemCatalog::ColType::ColType() : - colWidth(0), constraintType(NO_CONSTRAINT), - colDataType(MEDINT), defaultValue(""), colPosition(-1), - scale(0), - precision(-1), compressionType(NO_COMPRESSION), columnOID(0), autoincrement(0), @@ -6104,15 +6105,12 @@ CalpontSystemCatalog::ColType::ColType() : } CalpontSystemCatalog::ColType::ColType(const ColType& rhs) + :TypeHolderStd(rhs) { - colWidth = rhs.colWidth; constraintType = rhs.constraintType; - colDataType = rhs.colDataType; ddn = rhs.ddn; defaultValue = rhs.defaultValue; colPosition = rhs.colPosition; - scale = rhs.scale; - precision = rhs.precision; compressionType = rhs.compressionType; columnOID = rhs.columnOID; autoincrement = rhs.autoincrement; @@ -6123,14 +6121,11 @@ CalpontSystemCatalog::ColType::ColType(const ColType& rhs) CalpontSystemCatalog::ColType& CalpontSystemCatalog::ColType::operator=(const ColType& rhs) { - colWidth = rhs.colWidth; + TypeHolderStd::operator=(rhs); constraintType = rhs.constraintType; - colDataType = rhs.colDataType; ddn = rhs.ddn; defaultValue = rhs.defaultValue; colPosition = rhs.colPosition; - scale = rhs.scale; - precision = rhs.precision; compressionType = rhs.compressionType; columnOID = rhs.columnOID; autoincrement = rhs.autoincrement; @@ -6141,6 +6136,9 @@ CalpontSystemCatalog::ColType& CalpontSystemCatalog::ColType::operator=(const Co return *this; } + + + CHARSET_INFO* CalpontSystemCatalog::ColType::getCharset() { if (!cs) @@ -6166,6 +6164,38 @@ const string CalpontSystemCatalog::ColType::toString() const return output.str(); } + +boost::any +CalpontSystemCatalog::ColType::convertColumnData(const std::string& data, + bool& pushWarning, + const std::string& timeZone, + bool nulFlag, + bool noRoundup, + bool isUpdate) const +{ + pushWarning = false; + const datatypes::TypeHandler *h= typeHandler(); + if (!h) + throw QueryDataExcept("convertColumnData: unknown column data type.", dataTypeErr); + + if (nulFlag) + return h->getNullValueForType(*this); + + const datatypes::ConvertFromStringParam prm(timeZone, noRoundup, isUpdate); + return h->convertFromString(*this, prm, data, pushWarning); +} + + +CalpontSystemCatalog::ColType CalpontSystemCatalog::ColType::convertUnionColType(vector& types) +{ + idbassert(types.size()); + CalpontSystemCatalog::ColType unionedType = types[0]; + for (uint64_t i = 1; i < types.size(); i++) + dataconvert::DataConvert::joinColTypeForUnion(unionedType, types[i]); + return unionedType; +} + + //format a session id that includes the module id //we want the top bit clear to use as a syscat flag, then we want 7 bits of module id, then 24 bits of thread id /*static*/ diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index e59663375..10ca7ddbe 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -39,6 +39,7 @@ #include #include +#include "mcs_basic_types.h" #include "../../writeengine/shared/we_typeext.h" #include "columnresult.h" #include "bytestream.h" @@ -48,49 +49,9 @@ #undef min #undef max -// Because including my_sys.h in a Columnstore header causes too many conflicts -struct charset_info_st; -typedef const struct charset_info_st CHARSET_INFO; +#include "mcs_datatype.h" -#ifdef _MSC_VER -#define __attribute__(x) -#endif - -namespace -{ -const int64_t MIN_TINYINT __attribute__ ((unused)) = std::numeric_limits::min() + 2; //-126; -const int64_t MAX_TINYINT __attribute__ ((unused)) = std::numeric_limits::max(); //127; -const int64_t MIN_SMALLINT __attribute__ ((unused)) = std::numeric_limits::min() + 2; //-32766; -const int64_t MAX_SMALLINT __attribute__ ((unused)) = std::numeric_limits::max(); //32767; -const int64_t MIN_MEDINT __attribute__ ((unused)) = -(1ULL << 23); //-8388608; -const int64_t MAX_MEDINT __attribute__ ((unused)) = (1ULL << 23) - 1; //8388607; -const int64_t MIN_INT __attribute__ ((unused)) = std::numeric_limits::min() + 2; //-2147483646; -const int64_t MAX_INT __attribute__ ((unused)) = std::numeric_limits::max(); //2147483647; -const int64_t MIN_BIGINT __attribute__ ((unused)) = std::numeric_limits::min() + 2; //-9223372036854775806LL; -const int64_t MAX_BIGINT __attribute__ ((unused)) = std::numeric_limits::max(); //9223372036854775807 - -const uint64_t MIN_UINT __attribute__ ((unused)) = 0; -const uint64_t MIN_UTINYINT __attribute__ ((unused)) = 0; -const uint64_t MIN_USMALLINT __attribute__ ((unused)) = 0; -const uint64_t MIN_UMEDINT __attribute__ ((unused)) = 0; -const uint64_t MIN_UBIGINT __attribute__ ((unused)) = 0; -const uint64_t MAX_UINT __attribute__ ((unused)) = std::numeric_limits::max() - 2; //4294967293 -const uint64_t MAX_UTINYINT __attribute__ ((unused)) = std::numeric_limits::max() - 2; //253; -const uint64_t MAX_USMALLINT __attribute__ ((unused)) = std::numeric_limits::max() - 2; //65533; -const uint64_t MAX_UMEDINT __attribute__ ((unused)) = (1ULL << 24) - 1; //16777215 -const uint64_t MAX_UBIGINT __attribute__ ((unused)) = std::numeric_limits::max() - 2; //18446744073709551613 - -const float MAX_FLOAT __attribute__ ((unused)) = std::numeric_limits::max(); //3.402823466385289e+38 -const float MIN_FLOAT __attribute__ ((unused)) = -std::numeric_limits::max(); -const double MAX_DOUBLE __attribute__ ((unused)) = std::numeric_limits::max(); //1.7976931348623157e+308 -const double MIN_DOUBLE __attribute__ ((unused)) = -std::numeric_limits::max(); -const long double MAX_LONGDOUBLE __attribute__ ((unused)) = std::numeric_limits::max(); //1.7976931348623157e+308 -const long double MIN_LONGDOUBLE __attribute__ ((unused)) = -std::numeric_limits::max(); - - -const uint64_t AUTOINCR_SATURATED __attribute__ ((unused)) = std::numeric_limits::max(); -} class ExecPlanTest; namespace messageqcpp @@ -98,6 +59,7 @@ namespace messageqcpp class MessageQueueClient; } + // This is now set in the Columnstore.xml file // Use, e.g., 0x500 for uid 500 so it is easy to spot in ipcs list //const int32_t BRM_UID = 0x0; @@ -116,7 +78,7 @@ const int32_t IDB_VTABLE_ID = CNX_VTABLE_ID; * * This object encapsulates the system catalog stored in the engine */ -class CalpontSystemCatalog +class CalpontSystemCatalog: public datatypes::SystemCatalog { public: @@ -130,49 +92,6 @@ public: */ enum Identity { EC = 0, FE }; - /** the set of Calpont column widths - * - */ - enum ColWidth { ONE_BIT, ONE_BYTE, TWO_BYTE, THREE_BYTE, FOUR_BYTE, FIVE_BYTE, SIX_BYTE, SEVEN_BYTE, EIGHT_BYTE }; - - /** the set of Calpont column data types - * - */ - enum ColDataType - { - BIT, /*!< BIT type */ - TINYINT, /*!< TINYINT type */ - CHAR, /*!< CHAR type */ - SMALLINT, /*!< SMALLINT type */ - DECIMAL, /*!< DECIMAL type */ - MEDINT, /*!< MEDINT type */ - INT, /*!< INT type */ - FLOAT, /*!< FLOAT type */ - DATE, /*!< DATE type */ - BIGINT, /*!< BIGINT type */ - DOUBLE, /*!< DOUBLE type */ - DATETIME, /*!< DATETIME type */ - VARCHAR, /*!< VARCHAR type */ - VARBINARY, /*!< VARBINARY type */ - CLOB, /*!< CLOB type */ - BLOB, /*!< BLOB type */ - UTINYINT, /*!< Unsigned TINYINT type */ - USMALLINT, /*!< Unsigned SMALLINT type */ - UDECIMAL, /*!< Unsigned DECIMAL type */ - UMEDINT, /*!< Unsigned MEDINT type */ - UINT, /*!< Unsigned INT type */ - UFLOAT, /*!< Unsigned FLOAT type */ - UBIGINT, /*!< Unsigned BIGINT type */ - UDOUBLE, /*!< Unsigned DOUBLE type */ - TEXT, /*!< TEXT type */ - TIME, /*!< TIME type */ - TIMESTAMP, /*!< TIMESTAMP type */ - NUM_OF_COL_DATA_TYPE, /* NEW TYPES ABOVE HERE */ - LONGDOUBLE, /* @bug3241, dev and variance calculation only */ - STRINT, /* @bug3532, string as int for fast comparison */ - UNDEFINED /*!< Undefined - used in UDAF API */ - }; - /** the set of column constraint types * */ @@ -278,21 +197,18 @@ public: */ typedef std::vector DictOIDList; + /** the structure returned by colType * * defaultValue is only meaningful when constraintType == DEFAULT_CONSTRAINT */ - struct ColType + struct ColType: public datatypes::SystemCatalog::TypeHolderStd { ColType(); - int32_t colWidth; ConstraintType constraintType; - ColDataType colDataType; DictOID ddn; std::string defaultValue; int32_t colPosition; // temporally put here. may need to have ColInfo struct later - int32_t scale; //number after decimal points - int32_t precision; int32_t compressionType; OID columnOID; bool autoincrement; //set to true if SYSCOLUMN autoincrement is �y� @@ -327,6 +243,23 @@ public: b >> charsetNumber; } + /** + * @brief convert a columns data, represnted as a string, + * to its native format + * @param data - the string representation + * @param [OUT] bSaturate - the value was truncated/adjusted + * @param timeZone - the time zone name, for TIMESTAMP conversion + * @param nullFlag - SQL NULL flag + * @param nRoundtrip + * @param isUpdate + */ + boost::any convertColumnData(const std::string& data, + bool& bSaturate, + const std::string& timeZone, + bool nulFlag = false, + bool noRoundup = false, + bool isUpdate = false) const; + const std::string toString() const; //Put these here so udf doesn't need to link libexecplan @@ -362,6 +295,8 @@ public: return !(*this == t); } + static ColType convertUnionColType(std::vector&); + }; /** the structure of a table infomation @@ -962,80 +897,9 @@ const CalpontSystemCatalog::TableAliasName make_aliastable(const std::string& s, const CalpontSystemCatalog::TableAliasName make_aliasview(const std::string& s, const std::string& t, const std::string& a, const std::string& v, const bool fisColumnStore = true, int lower_case_table_names=0); -/** convenience function to determine if column type is a char - * type - */ -inline bool isCharType(const execplan::CalpontSystemCatalog::ColDataType type) +inline bool isNull(int128_t val, const execplan::CalpontSystemCatalog::ColType& ct) { - return (execplan::CalpontSystemCatalog::VARCHAR == type || - execplan::CalpontSystemCatalog::CHAR == type || - execplan::CalpontSystemCatalog::BLOB == type || - execplan::CalpontSystemCatalog::TEXT == type); -} - -/** convenience function to determine if column type is a - * numeric type - */ -inline bool isNumeric(const execplan::CalpontSystemCatalog::ColDataType type) -{ - switch (type) - { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::UDECIMAL: - return true; - - default: - return false; - } -} - -/** convenience function to determine if column type is an - * unsigned type - */ -inline bool isUnsigned(const execplan::CalpontSystemCatalog::ColDataType type) -{ - switch (type) - { - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - return true; - - default: - return false; - } -} - -inline bool isSignedInteger(const execplan::CalpontSystemCatalog::ColDataType type) -{ - switch (type) - { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - return true; - - default: - return false; - } + return datatypes::Decimal::isWideDecimalNullValue(val); } inline bool isNull(int64_t val, const execplan::CalpontSystemCatalog::ColType& ct) @@ -1212,6 +1076,13 @@ inline bool isNull(int64_t val, const execplan::CalpontSystemCatalog::ColType& c break; } + case execplan::CalpontSystemCatalog::BINARY: + { + ret = false; + + break; + } + default: break; } diff --git a/dbcon/execplan/constantcolumn.cpp b/dbcon/execplan/constantcolumn.cpp index 28a7611d4..f92a59dc6 100644 --- a/dbcon/execplan/constantcolumn.cpp +++ b/dbcon/execplan/constantcolumn.cpp @@ -230,12 +230,14 @@ ConstantColumn::ConstantColumn(const int64_t val, TYPE type) : fResult.doubleVal = (double)fResult.intVal; fResult.longDoubleVal = (long double)fResult.intVal; fResult.decimalVal.value = fResult.intVal; + fResult.decimalVal.s128Value = fResult.intVal; fResult.decimalVal.scale = 0; fResultType.colDataType = CalpontSystemCatalog::BIGINT; fResultType.colWidth = 8; } -ConstantColumn::ConstantColumn(const uint64_t val, TYPE type) : +ConstantColumn::ConstantColumn(const uint64_t val, TYPE type, + int8_t scale, uint8_t precision) : ReturnedColumn(), fType(type) { @@ -250,7 +252,9 @@ ConstantColumn::ConstantColumn(const uint64_t val, TYPE type) : fResult.doubleVal = (double)fResult.uintVal; fResult.longDoubleVal = (long double)fResult.uintVal; fResult.decimalVal.value = fResult.uintVal; - fResult.decimalVal.scale = 0; + fResult.decimalVal.s128Value = fResult.uintVal; + fResult.decimalVal.scale = scale; + fResult.decimalVal.precision = precision; fResultType.colDataType = CalpontSystemCatalog::UBIGINT; fResultType.colWidth = 8; } @@ -317,6 +321,7 @@ void ConstantColumn::serialize(messageqcpp::ByteStream& b) const b << (uint8_t)fResult.boolVal; b << fResult.strVal; b << (uint64_t)fResult.decimalVal.value; + b << (uint128_t)fResult.decimalVal.s128Value; b << (uint8_t)fResult.decimalVal.scale; b << (uint8_t)fResult.decimalVal.precision; } @@ -341,6 +346,7 @@ void ConstantColumn::unserialize(messageqcpp::ByteStream& b) b >> (uint8_t&)fResult.boolVal; b >> fResult.strVal; b >> (uint64_t&)fResult.decimalVal.value; + b >> (uint128_t&)fResult.decimalVal.s128Value; b >> (uint8_t&)fResult.decimalVal.scale; b >> (uint8_t&)fResult.decimalVal.precision; } diff --git a/dbcon/execplan/constantcolumn.h b/dbcon/execplan/constantcolumn.h index a5c0a074b..334247c47 100644 --- a/dbcon/execplan/constantcolumn.h +++ b/dbcon/execplan/constantcolumn.h @@ -73,7 +73,8 @@ public: /** * ctor */ - ConstantColumn(const uint64_t val, TYPE type = NUM); // deprecate + ConstantColumn(const uint64_t val, TYPE type = NUM, + int8_t scale = 0, uint8_t precision = 0); // deprecate //There are more ctors below... /** diff --git a/dbcon/execplan/functioncolumn.h b/dbcon/execplan/functioncolumn.h index 2be25897f..cda618b85 100644 --- a/dbcon/execplan/functioncolumn.h +++ b/dbcon/execplan/functioncolumn.h @@ -246,15 +246,45 @@ public: { IDB_Decimal decimal = fFunctor->getDecimalVal(row, fFunctionParms, isNull, fOperationType); - if (fResultType.scale == decimal.scale) + if (UNLIKELY(fResultType.colWidth == utils::MAXLEGACYWIDTH + && fResultType.scale == decimal.scale)) return decimal; - if (fResultType.scale > decimal.scale) - decimal.value *= IDB_pow[fResultType.scale - decimal.scale]; - else - decimal.value = (int64_t)(decimal.value > 0 ? - (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] + 0.5 : - (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] - 0.5); + if (LIKELY(fResultType.colWidth == datatypes::MAXDECIMALWIDTH)) + { + decimal.s128Value = + (datatypes::Decimal::isWideDecimalTypeByPrecision(decimal.precision)) ? + decimal.s128Value : decimal.value; + + int128_t scaleMultiplier; + int32_t scaleDiff = fResultType.scale - decimal.scale; + datatypes::getScaleDivisor(scaleMultiplier, abs(scaleDiff)); + + if (scaleMultiplier > 1) + { + if (scaleDiff > 0) + { + // WIP MCOL-641 Unconditional overflow check + datatypes::MultiplicationNoOverflowCheck mul; + mul(decimal.s128Value, scaleMultiplier, decimal.s128Value); + } + else + { + decimal.s128Value = (int128_t)(decimal.s128Value > 0 ? + (__float128)decimal.s128Value / scaleMultiplier + 0.5 : + (__float128)decimal.s128Value / scaleMultiplier - 0.5); + } + } + } + else if (fResultType.colWidth == utils::MAXLEGACYWIDTH) + { + if (fResultType.scale > decimal.scale) + decimal.value *= IDB_pow[fResultType.scale - decimal.scale]; + else + decimal.value = (int64_t)(decimal.value > 0 ? + (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] + 0.5 : + (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] - 0.5); + } decimal.scale = fResultType.scale; decimal.precision = fResultType.precision; diff --git a/dbcon/execplan/parsetree.h b/dbcon/execplan/parsetree.h index f31a2a013..34131615d 100644 --- a/dbcon/execplan/parsetree.h +++ b/dbcon/execplan/parsetree.h @@ -30,6 +30,7 @@ #include #include "treenode.h" #include "operator.h" +#include "mcs_decimal.h" namespace rowgroup { diff --git a/dbcon/execplan/predicateoperator.cpp b/dbcon/execplan/predicateoperator.cpp index 3f4aee1f2..02dc3a710 100644 --- a/dbcon/execplan/predicateoperator.cpp +++ b/dbcon/execplan/predicateoperator.cpp @@ -126,6 +126,7 @@ bool PredicateOperator::operator!=(const TreeNode* t) const } //FIXME: VARBINARY??? +//FIXME: BINARY??? void PredicateOperator::setOpType(Type& l, Type& r) { fOperationType = l; // Default to left side. Modify as needed. diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index 4cd5d37f0..f28b289f1 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -120,6 +120,7 @@ public: void setOpType(Type& l, Type& r); private: + inline bool numericCompare(IDB_Decimal& op1, IDB_Decimal& op2); template inline bool numericCompare(result_t op1, result_t op2); inline bool strTrimCompare(const std::string& op1, const std::string& op2); @@ -127,6 +128,36 @@ private: const CHARSET_INFO* cs; }; +inline bool PredicateOperator::numericCompare(IDB_Decimal& op1, IDB_Decimal& op2) +{ + switch (fOp) + { + case OP_EQ: + return op1 == op2; + + case OP_NE: + return op1 != op2; + + case OP_GT: + return op1 > op2; + + case OP_GE: + return op1 >= op2; + + case OP_LT: + return op1 < op2; + + case OP_LE: + return op1 <= op2; + + default: + { + std::ostringstream oss; + oss << "invalid predicate operation: " << fOp; + throw logging::InvalidOperationExcept(oss.str()); + } + } +} template inline bool PredicateOperator::numericCompare(result_t op1, result_t op2) diff --git a/dbcon/execplan/returnedcolumn.cpp b/dbcon/execplan/returnedcolumn.cpp index 44278424f..5900f077f 100644 --- a/dbcon/execplan/returnedcolumn.cpp +++ b/dbcon/execplan/returnedcolumn.cpp @@ -49,6 +49,7 @@ ReturnedColumn::ReturnedColumn(): fReturnAll (false), fColPosition(-1), fHasAggregate(false), fInputIndex(-1), + fInputOffset(-1), fOutputIndex(-1), fExpressionId ((uint32_t) - 1) { @@ -69,6 +70,7 @@ ReturnedColumn::ReturnedColumn(const string& sql): fHasAggregate(false), fData(sql), fInputIndex(-1), + fInputOffset(-1), fOutputIndex(-1), fExpressionId ((uint32_t) - 1) { @@ -88,6 +90,7 @@ ReturnedColumn::ReturnedColumn(const uint32_t sessionID, const bool returnAll): fColPosition(-1), fHasAggregate(false), fInputIndex(-1), + fInputOffset(-1), fOutputIndex(-1), fExpressionId ((uint32_t) - 1) { @@ -109,6 +112,7 @@ ReturnedColumn::ReturnedColumn(const ReturnedColumn& rhs, const uint32_t session fHasAggregate(rhs.fHasAggregate), fData(rhs.fData), fInputIndex(rhs.fInputIndex), + fInputOffset(rhs.fInputOffset), fOutputIndex(rhs.fOutputIndex), fExpressionId (rhs.fExpressionId) { @@ -141,6 +145,7 @@ void ReturnedColumn::serialize(messageqcpp::ByteStream& b) const b << (uint64_t)fColSource; b << (int64_t)fColPosition; b << (uint32_t)fInputIndex; + b << (uint32_t)fInputOffset; b << (uint32_t)fOutputIndex; b << (int32_t)fSequence; b << (uint8_t)fReturnAll; @@ -163,6 +168,7 @@ void ReturnedColumn::unserialize(messageqcpp::ByteStream& b) b >> (uint64_t&)fColSource; b >> (int64_t&)fColPosition; b >> (uint32_t&)fInputIndex; + b >> (uint32_t&)fInputOffset; b >> (uint32_t&)fOutputIndex; b >> (int32_t&)fSequence; b >> (uint8_t&)fReturnAll; diff --git a/dbcon/execplan/returnedcolumn.h b/dbcon/execplan/returnedcolumn.h index 45f23c047..6c07f5694 100644 --- a/dbcon/execplan/returnedcolumn.h +++ b/dbcon/execplan/returnedcolumn.h @@ -374,6 +374,7 @@ public: protected: std::string fErrMsg; /// error occured in evaluation uint32_t fInputIndex; /// index to the input rowgroup + uint32_t fInputOffset; /// index to the input rowgroup uint32_t fOutputIndex; /// index to the output rowgroup uint32_t fExpressionId; /// unique id for this expression }; diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index 8110c5296..64620cb15 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -51,6 +51,7 @@ using namespace joblist; #include "aggregatecolumn.h" #include "constantfilter.h" #include "../../utils/windowfunction/windowfunction.h" +#include "utils/common/branchpred.h" namespace execplan { @@ -482,6 +483,7 @@ void SimpleColumn::setDerivedTable() // @todo make aggregate filter to having clause. not optimize it for now if (fDerivedRefCol && + // WIP replace with typeid() (dynamic_cast(fDerivedRefCol) || dynamic_cast(fDerivedRefCol))) fDerivedTable = ""; @@ -500,6 +502,12 @@ bool SimpleColumn::singleTable(CalpontSystemCatalog::TableAliasName& tan) // @todo move to inline void SimpleColumn::evaluate(Row& row, bool& isNull) { + // WIP Move this block into an appropriate place + if (UNLIKELY((int)(fInputOffset == (uint32_t)-1))) + { + fInputOffset = row.getOffset(fInputIndex); + } + bool isNull2 = row.isNullValue(fInputIndex); if (isNull2) @@ -625,35 +633,40 @@ void SimpleColumn::evaluate(Row& row, bool& isNull) { switch (fResultType.colWidth) { + case 16: + { + datatypes::TSInt128::assignPtrPtr(&fResult.decimalVal.s128Value, + row.getBinaryField_offset(fInputOffset)); + break; + } case 1: { fResult.decimalVal.value = row.getIntField<1>(fInputIndex); - fResult.decimalVal.scale = (unsigned)fResultType.scale; break; } case 2: { fResult.decimalVal.value = row.getIntField<2>(fInputIndex); - fResult.decimalVal.scale = (unsigned)fResultType.scale; break; } case 4: { fResult.decimalVal.value = row.getIntField<4>(fInputIndex); - fResult.decimalVal.scale = (unsigned)fResultType.scale; break; } default: { fResult.decimalVal.value = (int64_t)row.getUintField<8>(fInputIndex); - fResult.decimalVal.scale = (unsigned)fResultType.scale; break; } } + fResult.decimalVal.scale = (unsigned)fResultType.scale; + fResult.decimalVal.precision = (unsigned)fResultType.precision; + break; } @@ -690,6 +703,15 @@ void SimpleColumn::evaluate(Row& row, bool& isNull) break; } + case CalpontSystemCatalog::BINARY: + + { + // WIP MCOL-641 Binary representation could contain \0. + std::string value(row.getBinaryField(fInputIndex)); + fResult.strVal.swap(value); + break; + } + default: // treat as int64 { fResult.intVal = row.getUintField<8>(fInputIndex); diff --git a/dbcon/execplan/simplecolumn.h b/dbcon/execplan/simplecolumn.h index d6e60929b..ea29444a7 100644 --- a/dbcon/execplan/simplecolumn.h +++ b/dbcon/execplan/simplecolumn.h @@ -335,20 +335,18 @@ public: // the original decimal scale is stored in scale field, which is no use for double. if (fResultType.precision == -1) { - IDB_Decimal rv; if (fResultType.colDataType == CalpontSystemCatalog::DOUBLE) { - rv.scale = fResultType.scale; - rv.precision = 15; - rv.value = (int64_t)(TreeNode::getDoubleVal() * IDB_pow[rv.scale]); + IDB_Decimal rv( + (int64_t)(TreeNode::getDoubleVal() * IDB_pow[fResultType.scale]), + fResultType.scale, 15); return rv; } else if (fResultType.colDataType == CalpontSystemCatalog::LONGDOUBLE) { - IDB_Decimal rv; - rv.scale = fResultType.scale; - rv.precision = 19; - rv.value = (int64_t)(TreeNode::getLongDoubleVal() * IDB_pow[rv.scale]); + IDB_Decimal rv ( + (int64_t)(TreeNode::getLongDoubleVal() * IDB_pow[fResultType.scale]), + fResultType.scale, fResultType.precision); return rv; } } diff --git a/dbcon/execplan/simplecolumn_decimal.h b/dbcon/execplan/simplecolumn_decimal.h index c49b597c5..4e78607cf 100644 --- a/dbcon/execplan/simplecolumn_decimal.h +++ b/dbcon/execplan/simplecolumn_decimal.h @@ -25,6 +25,7 @@ #ifndef SIMPLECOLUMNDECIMAL_H #define SIMPLECOLUMNDECIMAL_H +#include #include #include @@ -141,7 +142,9 @@ void SimpleColumn_Decimal::setNullVal() case 1: fNullVal = joblist::TINYINTNULL; break; - + case 16: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; + //fallthrough default: fNullVal = joblist::BIGINTNULL; } @@ -150,8 +153,10 @@ void SimpleColumn_Decimal::setNullVal() template inline const std::string& SimpleColumn_Decimal:: getStrVal(rowgroup::Row& row, bool& isNull) { - dataconvert::DataConvert::decimalToString((int64_t)row.getIntField(fInputIndex), fResultType.scale, tmp, 22, fResultType.colDataType); - fResult.strVal = std::string(tmp); + datatypes::VDecimal dec((int64_t)row.getIntField(fInputIndex), + fResultType.scale, + fResultType.precision); + fResult.strVal = dec.toString(); return fResult.strVal; } @@ -223,6 +228,8 @@ void SimpleColumn_Decimal::serialize(messageqcpp::ByteStream& b) const case 8: b << (ObjectReader::id_t) ObjectReader::SIMPLECOLUMN_DECIMAL8; break; + case 16: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; } SimpleColumn::serialize(b); @@ -248,6 +255,8 @@ void SimpleColumn_Decimal::unserialize(messageqcpp::ByteStream& b) case 8: ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_DECIMAL8); break; + case 16: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix 16 Bytes ?" << std::endl; } SimpleColumn::unserialize(b); diff --git a/dbcon/execplan/simplecolumn_int.h b/dbcon/execplan/simplecolumn_int.h index 9454bba86..7e00207ca 100644 --- a/dbcon/execplan/simplecolumn_int.h +++ b/dbcon/execplan/simplecolumn_int.h @@ -25,12 +25,14 @@ #ifndef SIMPLECOLUMNINT_H #define SIMPLECOLUMNINT_H +#include #include #include "simplecolumn.h" #include "objectreader.h" #include "joblisttypes.h" #include "rowgroup.h" +#include "mcs_decimal.h" /** * Namespace @@ -216,7 +218,7 @@ inline IDB_Decimal SimpleColumn_INT::getDecimalVal(rowgroup::Row& row, bool isNull = true; fResult.decimalVal.value = (int64_t)row.getIntField(fInputIndex); - fResult.decimalVal.precision = 65; + fResult.decimalVal.precision = datatypes::INT64MAXPRECISION; fResult.decimalVal.scale = 0; return fResult.decimalVal; } diff --git a/dbcon/execplan/simplecolumn_uint.h b/dbcon/execplan/simplecolumn_uint.h index 731242809..edd811cba 100644 --- a/dbcon/execplan/simplecolumn_uint.h +++ b/dbcon/execplan/simplecolumn_uint.h @@ -25,12 +25,14 @@ #ifndef SIMPLECOLUMNUINT_H #define SIMPLECOLUMNUINT_H +#include #include #include "simplecolumn.h" #include "objectreader.h" #include "joblisttypes.h" #include "rowgroup.h" +#include "mcs_decimal.h" /** * Namespace @@ -140,7 +142,9 @@ void SimpleColumn_UINT::setNullVal() case 1: fNullVal = joblist::UTINYINTNULL; break; - + case 16: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix 16 Bytes ?" << std::endl; + //fallthrough default: fNullVal = joblist::UBIGINTNULL; } @@ -216,7 +220,7 @@ inline IDB_Decimal SimpleColumn_UINT::getDecimalVal(rowgroup::Row& row, boo isNull = true; fResult.decimalVal.value = (uint64_t)row.getUintField(fInputIndex); - fResult.decimalVal.precision = 65; + fResult.decimalVal.precision = datatypes::INT64MAXPRECISION; fResult.decimalVal.scale = 0; return fResult.decimalVal; } @@ -241,6 +245,8 @@ void SimpleColumn_UINT::serialize(messageqcpp::ByteStream& b) const case 8: b << (ObjectReader::id_t) ObjectReader::SIMPLECOLUMN_UINT8; break; + case 16: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix 16 Bytes ?" << std::endl; } SimpleColumn::serialize(b); @@ -266,6 +272,8 @@ void SimpleColumn_UINT::unserialize(messageqcpp::ByteStream& b) case 8: ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_UINT8); break; + case 16: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix 16 Bytes ?" << std::endl; } SimpleColumn::unserialize(b); diff --git a/dbcon/execplan/simplefilter.cpp b/dbcon/execplan/simplefilter.cpp index 3fcc8fcd6..f394f5e51 100644 --- a/dbcon/execplan/simplefilter.cpp +++ b/dbcon/execplan/simplefilter.cpp @@ -219,6 +219,7 @@ const string SimpleFilter::data() const fRhs->resultType().colDataType == CalpontSystemCatalog::DATE || fRhs->resultType().colDataType == CalpontSystemCatalog::DATETIME || fRhs->resultType().colDataType == CalpontSystemCatalog::TIMESTAMP || + fRhs->resultType().colDataType == CalpontSystemCatalog::BINARY || fRhs->resultType().colDataType == CalpontSystemCatalog::TIME)) rhs = "'" + SimpleFilter::escapeString(fRhs->data()) + "'"; else @@ -233,6 +234,7 @@ const string SimpleFilter::data() const fLhs->resultType().colDataType == CalpontSystemCatalog::DATE || fLhs->resultType().colDataType == CalpontSystemCatalog::TIME || fLhs->resultType().colDataType == CalpontSystemCatalog::TIMESTAMP || + fLhs->resultType().colDataType == CalpontSystemCatalog::BINARY || fLhs->resultType().colDataType == CalpontSystemCatalog::DATETIME)) lhs = "'" + SimpleFilter::escapeString(fLhs->data()) + "'"; else diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 5fab31a02..3d94a6e09 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -35,6 +35,8 @@ #include "calpontsystemcatalog.h" #include "exceptclasses.h" #include "dataconvert.h" +#include "columnwidth.h" +#include "mcs_decimal.h" namespace messageqcpp { @@ -55,104 +57,21 @@ namespace execplan typedef execplan::CalpontSystemCatalog::ColType Type; -/** - * @brief IDB_Decimal type - * - */ -struct IDB_Decimal +class IDB_Decimal: public datatypes::VDecimal { - IDB_Decimal(): value(0), scale(0), precision(0) {} - IDB_Decimal(int64_t val, int8_t s, uint8_t p) : - value (val), - scale(s), - precision(p) {} +public: + IDB_Decimal() = default; + IDB_Decimal(int64_t val, int8_t s, uint8_t p, const int128_t &val128 = 0) : + VDecimal(val, s, p, val128) {} - int decimalComp(const IDB_Decimal& d) const - { - lldiv_t d1 = lldiv(value, IDB_pow[scale]); - lldiv_t d2 = lldiv(d.value, IDB_pow[d.scale]); - - int ret = 0; - - if (d1.quot > d2.quot) - { - ret = 1; - } - else if (d1.quot < d2.quot) - { - ret = -1; - } - else - { - // rem carries the value's sign, but needs to be normalized. - int64_t s = scale - d.scale; - - if (s < 0) - { - if ((d1.rem * IDB_pow[-s]) > d2.rem) - ret = 1; - else if ((d1.rem * IDB_pow[-s]) < d2.rem) - ret = -1; - } - else - { - if (d1.rem > (d2.rem * IDB_pow[s])) - ret = 1; - else if (d1.rem < (d2.rem * IDB_pow[s])) - ret = -1; - } - } - - return ret; - } - - bool operator==(const IDB_Decimal& rhs) const - { - if (scale == rhs.scale) - return value == rhs.value; - else - return (decimalComp(rhs) == 0); - } - bool operator>(const IDB_Decimal& rhs) const - { - if (scale == rhs.scale) - return value > rhs.value; - else - return (decimalComp(rhs) > 0); - } - bool operator<(const IDB_Decimal& rhs) const - { - if (scale == rhs.scale) - return value < rhs.value; - else - return (decimalComp(rhs) < 0); - } - bool operator>=(const IDB_Decimal& rhs) const - { - if (scale == rhs.scale) - return value >= rhs.value; - else - return (decimalComp(rhs) >= 0); - } - bool operator<=(const IDB_Decimal& rhs) const - { - if (scale == rhs.scale) - return value <= rhs.value; - else - return (decimalComp(rhs) <= 0); - } - bool operator!=(const IDB_Decimal& rhs) const - { - if (scale == rhs.scale) - return value != rhs.value; - else - return (decimalComp(rhs) != 0); - } - - int64_t value; - int8_t scale; // 0~18 - uint8_t precision; // 1~18 + inline void operator=(const datatypes::TSInt128& rhs) + { + value = 0; scale = 0; precision = 0; + datatypes::TSInt128::operator=(rhs); + } }; + + typedef IDB_Decimal CNX_Decimal; /** @@ -233,7 +152,7 @@ struct Result { Result(): intVal(0), uintVal(0), origIntVal(0), dummy(0), doubleVal(0), longDoubleVal(0), floatVal(0), boolVal(false), - strVal(""), decimalVal(IDB_Decimal(0, 0, 0)), + strVal(""), decimalVal(IDB_Decimal()), valueConverted(false) {} int64_t intVal; uint64_t uintVal; @@ -537,7 +456,10 @@ inline bool TreeNode::getBoolVal() case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - return (fResult.decimalVal.value != 0); + if (fResultType.colWidth == datatypes::MAXDECIMALWIDTH) + return (fResult.decimalVal.s128Value != 0); + else + return (fResult.decimalVal.value != 0); default: throw logging::InvalidConversionExcept("TreeNode::getBoolVal: Invalid conversion."); @@ -708,8 +630,11 @@ inline const std::string& TreeNode::getStrVal(const std::string& timeZone) case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - dataconvert::DataConvert::decimalToString(fResult.decimalVal.value, fResult.decimalVal.scale, tmp, 22, fResultType.colDataType); - fResult.strVal = std::string(tmp); + if (fResultType.colWidth == datatypes::MAXDECIMALWIDTH) + // Explicit path for TSInt128 decimals with low precision + fResult.strVal = fResult.decimalVal.toString(true); + else + fResult.strVal = fResult.decimalVal.toString(); break; } @@ -741,6 +666,10 @@ inline const std::string& TreeNode::getStrVal(const std::string& timeZone) break; } + case CalpontSystemCatalog::BINARY: + { + break; + } default: throw logging::InvalidConversionExcept("TreeNode::getStrVal: Invalid conversion."); } @@ -801,7 +730,20 @@ inline int64_t TreeNode::getIntVal() case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - return (int64_t)(fResult.decimalVal.value / pow((double)10, fResult.decimalVal.scale)); + if (fResultType.colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t scaleDivisor; + + datatypes::getScaleDivisor(scaleDivisor, fResult.decimalVal.scale); + + int128_t tmpval = fResult.decimalVal.s128Value / scaleDivisor; + + return datatypes::Decimal::getInt64FromWideDecimal(tmpval); + } + else + { + return (int64_t)(fResult.decimalVal.value / pow((double)10, fResult.decimalVal.scale)); + } } case CalpontSystemCatalog::DATE: @@ -983,8 +925,15 @@ inline double TreeNode::getDoubleVal() case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - // this may not be accurate. if this is problematic, change to pre-calculated power array. - return (double)(fResult.decimalVal.value / pow((double)10, fResult.decimalVal.scale)); + if (fResultType.colWidth == datatypes::MAXDECIMALWIDTH) + { + return datatypes::Decimal::getDoubleFromWideDecimal(fResult.decimalVal.s128Value, fResult.decimalVal.scale); + } + else + { + // this may not be accurate. if this is problematic, change to pre-calculated power array. + return (double)(fResult.decimalVal.value / pow((double)10, fResult.decimalVal.scale)); + } } case CalpontSystemCatalog::DATE: @@ -1078,6 +1027,7 @@ inline IDB_Decimal TreeNode::getDecimalVal() case CalpontSystemCatalog::VARBINARY: case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::BINARY: throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: non-support conversion from binary string"); case CalpontSystemCatalog::BIGINT: diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index b4ebd8b42..f574dbf8b 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -359,7 +359,9 @@ bool WindowFunctionColumn::hasWindowFunc() void WindowFunctionColumn::adjustResultType() { - if (fResultType.colDataType == CalpontSystemCatalog::DECIMAL && + if ((fResultType.colDataType == CalpontSystemCatalog::DECIMAL || + fResultType.colDataType == CalpontSystemCatalog::UDECIMAL) + && !boost::iequals(fFunctionName, "COUNT") && !boost::iequals(fFunctionName, "COUNT(*)") && !boost::iequals(fFunctionName, "ROW_NUMBER") && @@ -385,11 +387,20 @@ void WindowFunctionColumn::adjustResultType() if (boost::iequals(fFunctionName, "SUM") || boost::iequals(fFunctionName, "AVG") || - boost::iequals(fFunctionName, "AVG_DISTINCT")) + boost::iequals(fFunctionName, "AVG_DISTINCT") || + boost::iequals(fFunctionName, "PERCENTILE")) { - fResultType.colDataType = CalpontSystemCatalog::LONGDOUBLE; - fResultType.colWidth = sizeof(long double); - fResultType.precision = -1; + if (fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::DECIMAL || + fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::UDECIMAL) + { + fResultType.colWidth = sizeof(int128_t); + } + else + { + fResultType.colDataType = CalpontSystemCatalog::LONGDOUBLE; + fResultType.colWidth = sizeof(long double); + fResultType.precision = -1; + } } } @@ -478,7 +489,9 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull) fResult.origIntVal = row.getUintField<8>(fInputIndex); break; - + case 16: + cout << __FILE__<< ":" <<__LINE__ << " Fix 16 Bytes ?" << endl; + //fallthrough default: if (row.equals(CPNULLSTRMARK, fInputIndex)) isNull = true; @@ -627,8 +640,10 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull) isNull = true; else { - fResult.decimalVal.value = row.getIntField<1>(fInputIndex); - fResult.decimalVal.scale = (unsigned)fResultType.scale; + fResult.decimalVal = IDB_Decimal( + row.getIntField<1>(fInputIndex), + fResultType.scale, + fResultType.precision); } break; @@ -640,8 +655,10 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull) isNull = true; else { - fResult.decimalVal.value = row.getIntField<2>(fInputIndex); - fResult.decimalVal.scale = (unsigned)fResultType.scale; + fResult.decimalVal = IDB_Decimal( + row.getIntField<2>(fInputIndex), + fResultType.scale, + fResultType.precision); } break; @@ -653,25 +670,47 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull) isNull = true; else { - fResult.decimalVal.value = row.getIntField<4>(fInputIndex); - fResult.decimalVal.scale = (unsigned)fResultType.scale; + fResult.decimalVal = IDB_Decimal( + row.getIntField<4>(fInputIndex), + fResultType.scale, + fResultType.precision); } break; } - default: + case 8: { if (row.equals<8>(BIGINTNULL, fInputIndex)) isNull = true; else { - fResult.decimalVal.value = (int64_t)row.getUintField<8>(fInputIndex); - fResult.decimalVal.scale = (unsigned)fResultType.scale; + fResult.decimalVal = IDB_Decimal( + row.getIntField<8>(fInputIndex), + fResultType.scale, + fResultType.precision); } break; } + + case 16: + { + int128_t val; + row.getInt128Field(fInputIndex, val); + + if (val == datatypes::Decimal128Null) + isNull = true; + else + { + fResult.decimalVal = IDB_Decimal(0, fResultType.scale, fResultType.precision, val); + } + + break; + } + default: + // Should log error + break; } break; diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.cpp b/dbcon/joblist/batchprimitiveprocessor-jl.cpp index 0f04c8a8e..c989a52ed 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.cpp +++ b/dbcon/joblist/batchprimitiveprocessor-jl.cpp @@ -60,6 +60,7 @@ BatchPrimitiveProcessorJL::BatchPrimitiveProcessorJL(const ResourceManager* rm) baseRid(0), ridCount(0), needStrValues(false), + wideColumnsWidths(0), filterCount(0), projectCount(0), needRidsAtDelivery(false), @@ -100,6 +101,8 @@ void BatchPrimitiveProcessorJL::addFilterStep(const pColScanStep& scan, vectorgetWidth())) + wideColumnsWidths |= cc->getWidth(); idbassert(sessionID == scan.sessionId()); } @@ -128,6 +131,8 @@ void BatchPrimitiveProcessorJL::addFilterStep(const pColStep& step) cc->setStepUuid(uuid); filterSteps.push_back(cc); filterCount++; + if (utils::isWide(cc->getWidth())) + wideColumnsWidths |= cc->getWidth(); idbassert(sessionID == step.sessionId()); } @@ -198,6 +203,8 @@ void BatchPrimitiveProcessorJL::addProjectStep(const pColStep& step) colWidths.push_back(cc->getWidth()); tupleLength += cc->getWidth(); projectCount++; + if (utils::isWide(cc->getWidth())) + wideColumnsWidths |= cc->getWidth(); idbassert(sessionID == step.sessionId()); } @@ -215,6 +222,9 @@ void BatchPrimitiveProcessorJL::addProjectStep(const PassThruStep& step) tupleLength += cc->getWidth(); projectCount++; + if (utils::isWide(cc->getWidth())) + wideColumnsWidths |= cc->getWidth(); + if (filterCount == 0 && !sendRowGroups) sendValues = true; @@ -411,6 +421,7 @@ void BatchPrimitiveProcessorJL::addElementType(const StringElementType& et, uint * block touches */ +// TODO MCOL-641 Add support here. Refer to BatchPrimitiveProcessor::makeResponse() void BatchPrimitiveProcessorJL::getElementTypes(ByteStream& in, vector* out, bool* validCPData, uint64_t* lbid, int64_t* min, int64_t* max, uint32_t* cachedIO, uint32_t* physIO, @@ -491,6 +502,7 @@ void BatchPrimitiveProcessorJL::getElementTypes(ByteStream& in, * blocks touched */ +// TODO MCOL-641 Add support here. Refer to BatchPrimitiveProcessor::makeResponse() void BatchPrimitiveProcessorJL::getStringElementTypes(ByteStream& in, vector* out, bool* validCPData, uint64_t* lbid, int64_t* min, int64_t* max, uint32_t* cachedIO, uint32_t* physIO, uint32_t* touchedBlocks) const @@ -547,6 +559,7 @@ void BatchPrimitiveProcessorJL::getStringElementTypes(ByteStream& in, * as when the output type is TableBands */ +// TODO MCOL-641 Add support here. Refer to BatchPrimitiveProcessor::makeResponse() void BatchPrimitiveProcessorJL::getTuples(messageqcpp::ByteStream& in, std::vector* out, bool* validCPData, uint64_t* lbid, int64_t* min, int64_t* max, uint32_t* cachedIO, uint32_t* physIO, uint32_t* touchedBlocks) const @@ -667,7 +680,9 @@ void BatchPrimitiveProcessorJL::getTuples(messageqcpp::ByteStream& in, columnData[j]++; pos++; break; - + case 16: + cout << __FILE__<< ":" <<__LINE__ << " Fix 16 Bytes ?" << endl; + //fallthrough default: cout << "BPP::getTuples(): bad column width of " << colWidths[j] << endl; @@ -691,9 +706,9 @@ bool BatchPrimitiveProcessorJL::countThisMsg(messageqcpp::ByteStream& in) const if (_hasScan) { if (data[offset] != 0) - offset += 25; // skip the CP data + offset += (data[offset + CP_FLAG_AND_LBID] * 2) + CP_FLAG_AND_LBID + 1; // skip the CP data with wide min/max values (16/32 bytes each) else - offset += 9; // skip only the "valid CP data" & LBID bytes + offset += CP_FLAG_AND_LBID; // skip only the "valid CP data" & LBID bytes } idbassert(in.length() > offset); @@ -717,11 +732,12 @@ void BatchPrimitiveProcessorJL::deserializeAggregateResult(ByteStream* in, } void BatchPrimitiveProcessorJL::getRowGroupData(ByteStream& in, vector* out, - bool* validCPData, uint64_t* lbid, int64_t* min, int64_t* max, + bool* validCPData, uint64_t* lbid, int128_t* min, int128_t* max, uint32_t* cachedIO, uint32_t* physIO, uint32_t* touchedBlocks, bool* countThis, - uint32_t threadID) const + uint32_t threadID, bool* hasWideColumn, const execplan::CalpontSystemCatalog::ColType& colType) const { uint64_t tmp64; + int128_t tmp128; uint8_t tmp8; RGData rgData; uint32_t rowCount; @@ -753,10 +769,34 @@ void BatchPrimitiveProcessorJL::getRowGroupData(ByteStream& in, vector* if (*validCPData) { in >> *lbid; - in >> tmp64; - *min = (int64_t) tmp64; - in >> tmp64; - *max = (int64_t) tmp64; + in >> tmp8; + *hasWideColumn = (tmp8 > utils::MAXLEGACYWIDTH); + if (UNLIKELY(*hasWideColumn)) + { + idbassert(colType.colWidth > utils::MAXLEGACYWIDTH); + if (LIKELY(colType.isWideDecimalType())) + { + in >> tmp128; + *min = tmp128; + in >> tmp128; + *max = tmp128; + } + else + { + std::ostringstream oss; + oss << __func__ << " WARNING!!! Not implemented for the data type "; + oss << colType.colDataType << std::endl; + std::cout << oss.str(); + idbassert(false); + } + } + else + { + in >> tmp64; + *min = static_cast(tmp64); + in >> tmp64; + *max = static_cast(tmp64); + } } else in >> *lbid; @@ -940,7 +980,7 @@ void BatchPrimitiveProcessorJL::createBPP(ByteStream& bs) const { ISMPacketHeader ism; uint32_t i; - uint8_t flags = 0; + uint16_t flags = 0; ism.Command = BATCH_PRIMITIVE_CREATE; @@ -976,8 +1016,14 @@ void BatchPrimitiveProcessorJL::createBPP(ByteStream& bs) const if (sendTupleJoinRowGroupData) flags |= JOIN_ROWGROUP_DATA; + if (wideColumnsWidths) + flags |= HAS_WIDE_COLUMNS; + bs << flags; + if (wideColumnsWidths) + bs << wideColumnsWidths; + bs << bop; bs << (uint8_t) (forHJ ? 1 : 0); diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.h b/dbcon/joblist/batchprimitiveprocessor-jl.h index a5f29a02b..fc535664d 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.h +++ b/dbcon/joblist/batchprimitiveprocessor-jl.h @@ -37,6 +37,7 @@ #include #include +#include "mcs_basic_types.h" #include "primitivemsg.h" #include "serializeable.h" #include "primitivestep.h" @@ -49,6 +50,8 @@ namespace joblist { const uint32_t LOGICAL_BLOCKS_CONVERTER = 23; // 10 + 13. 13 to convert to logical blocks, // 10 to convert to groups of 1024 logical blocks +const uint8_t CP_FLAG_AND_LBID = 9; // # bytes used for CP boolean and the lbid + // used by BatchPrimitiveProcessorJL::countThisMsg() // forward reference struct JobInfo; @@ -166,9 +169,9 @@ public: void deserializeAggregateResults(messageqcpp::ByteStream* in, std::vector* out) const; void getRowGroupData(messageqcpp::ByteStream& in, std::vector* out, - bool* validCPData, uint64_t* lbid, int64_t* min, int64_t* max, + bool* validCPData, uint64_t* lbid, int128_t* min, int128_t* max, uint32_t* cachedIO, uint32_t* physIO, uint32_t* touchedBlocks, bool* countThis, - uint32_t threadID) const; + uint32_t threadID, bool* hasBinaryColumn, const execplan::CalpontSystemCatalog::ColType& colType) const; void deserializeAggregateResult(messageqcpp::ByteStream* in, std::vector* out) const; bool countThisMsg(messageqcpp::ByteStream& in) const; @@ -279,10 +282,15 @@ private: uint16_t relRids[LOGICAL_BLOCK_RIDS]; boost::scoped_array absRids; + // TODO MCOL-641 Do we need uint128_t buffers here? + // When would sendValues=true, in which case values[] + // is sent to primproc? uint64_t values[LOGICAL_BLOCK_RIDS]; uint16_t ridCount; bool needStrValues; + uint16_t wideColumnsWidths; + std::vector filterSteps; std::vector projectSteps; //@bug 1136 diff --git a/dbcon/joblist/columncommand-jl.cpp b/dbcon/joblist/columncommand-jl.cpp index 1ea451b19..63a57743c 100644 --- a/dbcon/joblist/columncommand-jl.cpp +++ b/dbcon/joblist/columncommand-jl.cpp @@ -269,7 +269,9 @@ uint8_t ColumnCommandJL::getTableColumnType() case 1: return TableColumn::UINT8; - + case 16: + cout << __FILE__<< ":" <<__LINE__ << " Fix 16 Bytes ?" << endl; + //fallthrough default: throw logic_error("ColumnCommandJL: bad column width"); } @@ -287,7 +289,7 @@ string ColumnCommandJL::toString() if (isDict()) ret << " (tokens)"; - else if (execplan::isCharType(colType.colDataType)) + else if (datatypes::isCharType(colType.colDataType)) ret << " (is char)"; return ret.str(); diff --git a/dbcon/joblist/crossenginestep.cpp b/dbcon/joblist/crossenginestep.cpp index 455906c0f..bbdefce0e 100644 --- a/dbcon/joblist/crossenginestep.cpp +++ b/dbcon/joblist/crossenginestep.cpp @@ -46,9 +46,6 @@ using namespace execplan; #include "rowgroup.h" using namespace rowgroup; -#include "dataconvert.h" -using namespace dataconvert; - #include "querytele.h" using namespace querytele; @@ -247,7 +244,7 @@ int64_t CrossEngineStep::convertValueNum( // bool nulFlag, // bool noRoundup ) bool pushWarning = false; - boost::any anyVal = DataConvert::convertColumnData(ct, str, pushWarning, fTimeZone, false, true, false); + boost::any anyVal = ct.convertColumnData(str, pushWarning, fTimeZone, false, true, false); // Out of range values are treated as NULL as discussed during design review. if (pushWarning) diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index 0c8c355a9..a1d0a28ae 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -56,6 +56,7 @@ using namespace ordering; #include "jobstep.h" #include "jlf_common.h" #include "limitedorderby.h" +#include "mcs_decimal.h" namespace joblist { @@ -368,7 +369,7 @@ void GroupConcatAgUM::applyMapping(const boost::shared_array& mapping, cons // For some reason the rowgroup mapping fcns don't work right in this class. for (uint64_t i = 0; i < fRow.getColumnCount(); i++) { - if (fRow.getColumnWidth(i) > 8) + if (fRow.getColumnWidth(i) > datatypes::MAXLEGACYWIDTH) { if (fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::CHAR || fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::VARCHAR || @@ -380,6 +381,10 @@ void GroupConcatAgUM::applyMapping(const boost::shared_array& mapping, cons { fRow.setLongDoubleField(row.getLongDoubleField(mapping[i]), i); } + else if (datatypes::isWideDecimalType(fRow.getColType(i), fRow.getColumnWidth(i))) + { + row.copyBinaryField(fRow, i, mapping[i]); + } } else { @@ -440,25 +445,46 @@ void GroupConcator::outputRow(std::ostringstream& oss, const rowgroup::Row& row) case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: case CalpontSystemCatalog::BIGINT: + { + int64_t intVal = row.getIntField(*i); + + oss << intVal; + + break; + } + case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - int64_t intVal = row.getIntField(*i); int scale = (int) row.getScale(*i); - if (scale == 0) + if (LIKELY(row.getColumnWidth(*i) == datatypes::MAXDECIMALWIDTH)) { - oss << intVal; + datatypes::VDecimal dec(0, + scale, + row.getPrecision(*i), + row.getBinaryField(*i)); + oss << fixed << dec; } else { - long double dblVal = intVal / pow(10.0, (double)scale); - oss << fixed << setprecision(scale) << dblVal; + int64_t intVal = row.getIntField(*i); + + if (scale == 0) + { + oss << intVal; + } + else + { + long double dblVal = intVal / pow(10.0, (double)scale); + oss << fixed << setprecision(scale) << dblVal; + } } break; } + case CalpontSystemCatalog::UTINYINT: case CalpontSystemCatalog::USMALLINT: case CalpontSystemCatalog::UMEDINT: @@ -609,27 +635,7 @@ int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row) case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - int64_t v = row.getIntField(*i); - double scale = row.getScale(*i); - - if (scale > 0) - { - v /= (int64_t) pow(10.0, scale); - - if (v < 0) fieldLen++; - - while ((v /= 10) != 0) fieldLen++; - - fieldLen += (int64_t) scale + 2;; - } - else - { - if (v < 0) fieldLen++; - - while ((v /= 10) != 0) fieldLen++; - - fieldLen += 1;; - } + fieldLen += 1; break; } @@ -929,9 +935,11 @@ void GroupConcatOrderBy::getResult(uint8_t* buff, const string& sep) rowStack.pop(); } - size_t resultSize = oss.str().size(); + int64_t resultSize = oss.str().size(); + resultSize = (resultSize > fGroupConcatLen) ? fGroupConcatLen : resultSize; fOutputString.reset(new uint8_t[resultSize + 2]); - memset(fOutputString.get(), 0, resultSize + 2); + fOutputString[resultSize] = '\0'; + fOutputString[resultSize + 1] = '\0'; strncpy((char*)fOutputString.get(), oss.str().c_str(), resultSize); @@ -1091,9 +1099,11 @@ void GroupConcatNoOrder::getResult(uint8_t* buff, const string& sep) fDataQueue.pop(); } - size_t resultSize = oss.str().size(); + int64_t resultSize = oss.str().size(); + resultSize = (resultSize > fGroupConcatLen) ? fGroupConcatLen : resultSize; fOutputString.reset(new uint8_t[resultSize + 2]); - memset(fOutputString.get(), 0, resultSize + 2); + fOutputString[resultSize] = '\0'; + fOutputString[resultSize + 1] = '\0'; strncpy((char*)fOutputString.get(), oss.str().c_str(), resultSize); diff --git a/dbcon/joblist/jlf_common.cpp b/dbcon/joblist/jlf_common.cpp index c40425b16..794ed4c07 100644 --- a/dbcon/joblist/jlf_common.cpp +++ b/dbcon/joblist/jlf_common.cpp @@ -38,6 +38,7 @@ using namespace BRM; #include "jlf_common.h" using namespace joblist; +#include "mcs_decimal.h" namespace { @@ -332,6 +333,9 @@ string extractTableAlias(const SSC& sc) //------------------------------------------------------------------------------ CalpontSystemCatalog::OID isDictCol(const CalpontSystemCatalog::ColType& colType) { + if (colType.isWideDecimalType() || + colType.colDataType == CalpontSystemCatalog::BINARY) return 0; + if (colType.colWidth > 8) return colType.ddn.dictOID; if (colType.colDataType == CalpontSystemCatalog::VARCHAR && diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 8c86908af..42bbf8904 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -65,9 +65,6 @@ namespace ba = boost::algorithm; #include "simplescalarfilter.h" using namespace execplan; -#include "dataconvert.h" -using namespace dataconvert; - #include "configcpp.h" using namespace config; @@ -88,7 +85,7 @@ using namespace logging; #include "jlf_common.h" #include "jlf_subquery.h" #include "jlf_tuplejoblist.h" - +#include "mcs_decimal.h" namespace { @@ -130,11 +127,12 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo); /* This looks like an inefficient way to get NULL values. Much easier ways to do it. */ -int64_t valueNullNum(const CalpontSystemCatalog::ColType& ct, const string& timeZone) +template +void valueNullNum(const CalpontSystemCatalog::ColType& ct, const string& timeZone, T& val) { - int64_t n = 0; + T& n = val; bool pushWarning = false; - boost::any anyVal = DataConvert::convertColumnData(ct, "", pushWarning, timeZone, true, false, false); + boost::any anyVal = ct.convertColumnData("", pushWarning, timeZone, true, false, false); switch (ct.colDataType) { @@ -292,34 +290,38 @@ int64_t valueNullNum(const CalpontSystemCatalog::ColType& ct, const string& time case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - n = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) - n = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::FOUR_BYTE) - n = boost::any_cast(anyVal); + if (LIKELY(ct.colWidth == datatypes::MAXDECIMALWIDTH)) + n = boost::any_cast(anyVal); else if (ct.colWidth == execplan::CalpontSystemCatalog::EIGHT_BYTE) n = boost::any_cast(anyVal); - else - n = 0xfffffffffffffffeLL; + else if (ct.colWidth == execplan::CalpontSystemCatalog::FOUR_BYTE) + n = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) + n = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) + n = boost::any_cast(anyVal); break; default: break; } - - return n; } -int64_t convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct, bool isNull, uint8_t& rf, const string& timeZone) +template +void convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct, bool isNull, uint8_t& rf, const string& timeZone, T& v) { - if (str.size() == 0 || isNull ) return valueNullNum(ct, timeZone); + if (str.size() == 0 || isNull ) + { + valueNullNum(ct, timeZone, v); + return; + } - int64_t v = 0; + + v = 0; rf = 0; bool pushWarning = false; - boost::any anyVal = DataConvert::convertColumnData(ct, str, pushWarning, timeZone, false, true, false); + boost::any anyVal = ct.convertColumnData(str, pushWarning, timeZone, false, true, false); switch (ct.colDataType) { @@ -439,10 +441,10 @@ int64_t convertValueNum(const string& str, const CalpontSystemCatalog::ColType& case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - v = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) - v = boost::any_cast(anyVal); + if (LIKELY(ct.colWidth == datatypes::MAXDECIMALWIDTH)) + v = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::EIGHT_BYTE) + v = boost::any_cast(anyVal); else if (ct.colWidth == execplan::CalpontSystemCatalog::FOUR_BYTE) #ifdef _MSC_VER v = boost::any_cast(anyVal); @@ -450,8 +452,10 @@ int64_t convertValueNum(const string& str, const CalpontSystemCatalog::ColType& #else v = boost::any_cast(anyVal); #endif - else - v = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) + v = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) + v = boost::any_cast(anyVal); break; @@ -485,8 +489,6 @@ int64_t convertValueNum(const string& str, const CalpontSystemCatalog::ColType& rf = (data[0] == '-') ? ROUND_NEG : ROUND_POS; } - - return v; } //TODO: make this totaly case-insensitive @@ -1601,7 +1603,6 @@ bool optimizeIdbPatitionSimpleFilter(SimpleFilter* sf, JobStepVector& jsv, JobIn return true; } - const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) { JobStepVector jsv; @@ -1841,6 +1842,7 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) { // @bug 1151 string longer than colwidth of char/varchar. int64_t value = 0; + int128_t value128 = 0; uint8_t rf = 0; #ifdef FAILED_ATOI_IS_ZERO @@ -1849,7 +1851,7 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) try { bool isNull = ConstantColumn::NULLDATA == cc->type(); - value = convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone); + convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value); if (ct.colDataType == CalpontSystemCatalog::FLOAT && !isNull) { @@ -1888,7 +1890,13 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) #else bool isNull = ConstantColumn::NULLDATA == cc->type(); - value = convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone); + // WIP MCOL-641 width check must be a f() not a literal + // make a template from convertValueNum to avoid extra if + // this condition doesn't support UDECIMAL + if (ct.isWideDecimalType()) + convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value128); + else + convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value); if (ct.colDataType == CalpontSystemCatalog::FLOAT && !isNull) { @@ -1921,7 +1929,12 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) pcs = new PseudoColStep(sc->oid(), tbl_oid, pc->pseudoType(), ct, jobInfo); if (sc->isColumnStore()) - pcs->addFilter(cop, value, rf); + { + if (ct.isWideDecimalType()) + pcs->addFilter(cop, value128, rf); + else + pcs->addFilter(cop, value, rf); + } pcs->alias(alias); pcs->view(view); @@ -2988,12 +3001,17 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo) //add each filter to pColStep int8_t cop = op2num(sop); int64_t value = 0; + int128_t value128 = 0; string constval = cc->constval(); // @bug 1151 string longer than colwidth of char/varchar. uint8_t rf = 0; bool isNull = ConstantColumn::NULLDATA == cc->type(); - value = convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone); + + if (ct.isWideDecimalType()) + convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value128); + else + convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value); if (ct.colDataType == CalpontSystemCatalog::FLOAT && !isNull) { @@ -3010,7 +3028,10 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo) if (ConstantColumn::NULLDATA == cc->type() && (opeq == *sop || opne == *sop)) cop = COMPARE_NIL; - pcs->addFilter(cop, value, rf); + if (ct.isWideDecimalType()) + pcs->addFilter(cop, value128, rf); + else + pcs->addFilter(cop, value, rf); } } @@ -3433,7 +3454,6 @@ JLF_ExecPlanToJobList::walkTree(execplan::ParseTree* n, JobInfo& jobInfo) break; case CONSTANTFILTER: - //cout << "ConstantFilter" << endl; jsv = doConstantFilter(dynamic_cast(tn), jobInfo); JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); break; diff --git a/dbcon/joblist/jlf_tuplejoblist.cpp b/dbcon/joblist/jlf_tuplejoblist.cpp index 5f7ea2dff..ddab457cc 100644 --- a/dbcon/joblist/jlf_tuplejoblist.cpp +++ b/dbcon/joblist/jlf_tuplejoblist.cpp @@ -4072,7 +4072,7 @@ SJSTEP unionQueries(JobStepVector& queries, uint64_t distinctUnionNum, JobInfo& // get unioned column types for (uint64_t j = 0; j < colCount; ++j) { - CalpontSystemCatalog::ColType colType = DataConvert::convertUnionColType(queryColTypes[j]); + CalpontSystemCatalog::ColType colType = CalpontSystemCatalog::ColType::convertUnionColType(queryColTypes[j]); types.push_back(colType.colDataType); csNums.push_back(colType.charsetNumber); scale.push_back(colType.scale); diff --git a/dbcon/joblist/joblisttypes.h b/dbcon/joblist/joblisttypes.h index 6d5664c57..ccdf13228 100644 --- a/dbcon/joblist/joblisttypes.h +++ b/dbcon/joblist/joblisttypes.h @@ -83,6 +83,12 @@ const uint16_t NULL_UINT16 = USMALLINTNULL; const uint32_t NULL_UINT32 = UINTNULL; const uint64_t NULL_UINT64 = UBIGINTNULL; +const uint64_t BINARYNULLVALUELOW = 0ULL; +const uint64_t BINARYNULLVALUEHIGH = 0x8000000000000000ULL; +const uint64_t BINARYEMPTYVALUELOW = 1ULL; +const uint64_t BINARYEMPTYVALUEHIGH = 0x8000000000000000ULL; +const uint64_t BINARYEMPTYROW = 0ULL; + const std::string CPNULLSTRMARK("_CpNuLl_"); const std::string CPSTRNOTFOUND("_CpNoTf_"); diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index 631e9a597..60180c8f8 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -28,6 +28,7 @@ #include "brm.h" #include "brmtypes.h" #include "dataconvert.h" +#include "mcs_decimal.h" #define IS_VERBOSE (fDebug >= 4) #define IS_DETAIL (fDebug >= 3) @@ -180,8 +181,8 @@ void LBIDList::Dump(long Index, int Count) const // Store max/min structure for update later. lbidPartitionVector serves a list of lbid,max,min to update // when the primitive returns with valid max/min values and the brm returns an invalid flag for the // requested lbid -// -bool LBIDList::GetMinMax(int64_t& min, int64_t& max, int64_t& seq, int64_t lbid, +template +bool LBIDList::GetMinMax(T& min, T& max, int64_t& seq, int64_t lbid, const std::vector* pEMEntries, execplan::CalpontSystemCatalog::ColDataType colDataType) { @@ -240,8 +241,16 @@ bool LBIDList::GetMinMax(int64_t& min, int64_t& max, int64_t& seq, int64_t lbid, } else { - mmp->max = numeric_limits::min(); - mmp->min = numeric_limits::max(); + if (typeid(T) == typeid(int128_t)) + { + mmp->bigMax = datatypes::Decimal::minInt128; + mmp->bigMin = datatypes::Decimal::maxInt128; + } + else + { + mmp->max = numeric_limits::min(); + mmp->min = numeric_limits::max(); + } } mmp->isValid = retVal; @@ -257,7 +266,8 @@ bool LBIDList::GetMinMax(int64_t& min, int64_t& max, int64_t& seq, int64_t lbid, return false; } -bool LBIDList::GetMinMax(int64_t* min, int64_t* max, int64_t* seq, +template +bool LBIDList::GetMinMax(T* min, T* max, int64_t* seq, int64_t lbid, const tr1::unordered_map& entries, execplan::CalpontSystemCatalog::ColDataType colDataType) { @@ -283,8 +293,16 @@ bool LBIDList::GetMinMax(int64_t* min, int64_t* max, int64_t* seq, } else { - mmp->max = numeric_limits::min(); - mmp->min = numeric_limits::max(); + if (typeid(T) == typeid(int128_t)) + { + mmp->bigMax = datatypes::Decimal::minInt128; + mmp->bigMin = datatypes::Decimal::maxInt128; + } + else + { + mmp->max = numeric_limits::min(); + mmp->min = numeric_limits::max(); + } } mmp->isValid = entry.partition.cprange.isValid; @@ -293,16 +311,28 @@ bool LBIDList::GetMinMax(int64_t* min, int64_t* max, int64_t* seq, return false; } - *min = entry.partition.cprange.lo_val; - *max = entry.partition.cprange.hi_val; + // *DRRTUY min/max should be 16 aligned here + if (typeid(T) == typeid(int128_t)) + { + *min = entry.partition.cprange.bigLoVal; + *max = entry.partition.cprange.bigHiVal; + } + else + { + *min = entry.partition.cprange.loVal; + *max = entry.partition.cprange.hiVal; + } + *seq = entry.partition.cprange.sequenceNum; + return true; } // Get the min, max, and sequence number for the specified LBID by searching // the given vector of ExtentMap entries. -int LBIDList::getMinMaxFromEntries(int64_t& min, int64_t& max, int32_t& seq, +template +int LBIDList::getMinMaxFromEntries(T& min, T& max, int32_t& seq, int64_t lbid, const std::vector& EMEntries) { for (unsigned i = 0; i < EMEntries.size(); i++) @@ -311,9 +341,18 @@ int LBIDList::getMinMaxFromEntries(int64_t& min, int64_t& max, int32_t& seq, if (lbid >= EMEntries[i].range.start && lbid <= lastLBID) { - min = EMEntries[i].partition.cprange.lo_val; - max = EMEntries[i].partition.cprange.hi_val; - seq = EMEntries[i].partition.cprange.sequenceNum; + // *DRRTUY min/max should be 16 aligned here + if (typeid(T) == typeid(int128_t)) + { + min = EMEntries[i].partition.cprange.bigLoVal; + max = EMEntries[i].partition.cprange.bigHiVal; + } + else + { + min = EMEntries[i].partition.cprange.loVal; + max = EMEntries[i].partition.cprange.hiVal; + } + seq = EMEntries[i].partition.cprange.sequenceNum; return EMEntries[i].partition.cprange.isValid; } } @@ -321,8 +360,8 @@ int LBIDList::getMinMaxFromEntries(int64_t& min, int64_t& max, int32_t& seq, return BRM::CP_INVALID; } -// -void LBIDList::UpdateMinMax(int64_t min, int64_t max, int64_t lbid, CalpontSystemCatalog::ColDataType type, +template +void LBIDList::UpdateMinMax(T min, T max, int64_t lbid, CalpontSystemCatalog::ColDataType type, bool validData) { MinMaxPartition* mmp = NULL; @@ -361,7 +400,7 @@ void LBIDList::UpdateMinMax(int64_t min, int64_t max, int64_t lbid, CalpontSyste if (mmp->isValid == BRM::CP_INVALID) { - if (execplan::isCharType(type)) + if (datatypes::isCharType(type)) { if (order_swap(min) < order_swap(mmp->min) || mmp->min == numeric_limits::max()) @@ -371,7 +410,7 @@ void LBIDList::UpdateMinMax(int64_t min, int64_t max, int64_t lbid, CalpontSyste mmp->max == numeric_limits::min()) mmp->max = max; } - else if (execplan::isUnsigned(type)) + else if (datatypes::isUnsigned(type)) { if (static_cast(min) < static_cast(mmp->min)) mmp->min = min; @@ -381,11 +420,26 @@ void LBIDList::UpdateMinMax(int64_t min, int64_t max, int64_t lbid, CalpontSyste } else { - if (min < mmp->min) - mmp->min = min; + if (typeid(T) == typeid(int128_t)) + { + if (min < mmp->bigMin) + { + mmp->bigMin = min; + } - if (max > mmp->max) - mmp->max = max; + if (max > mmp->bigMax) + { + mmp->bigMax = max; + } + } + else + { + if (min < mmp->min) + mmp->min = min; + + if (max > mmp->max) + mmp->max = max; + } } } @@ -405,7 +459,7 @@ void LBIDList::UpdateMinMax(int64_t min, int64_t max, int64_t lbid, CalpontSyste } } -void LBIDList::UpdateAllPartitionInfo() +void LBIDList::UpdateAllPartitionInfo(const execplan::CalpontSystemCatalog::ColType& colType) { MinMaxPartition* mmp = NULL; #ifdef DEBUG @@ -431,8 +485,17 @@ void LBIDList::UpdateAllPartitionInfo() if ((mmp->isValid == BRM::CP_INVALID) && (mmp->blksScanned > 0)) { cpInfo.firstLbid = mmp->lbid; - cpInfo.max = mmp->max; - cpInfo.min = mmp->min; + cpInfo.isBinaryColumn = (colType.colWidth == 16); + if (cpInfo.isBinaryColumn) + { + cpInfo.bigMax = mmp->bigMax; + cpInfo.bigMin = mmp->bigMin; + } + else + { + cpInfo.max = mmp->max; + cpInfo.min = mmp->min; + } cpInfo.seqNum = (int32_t)mmp->seq; vCpInfo.push_back(cpInfo); @@ -595,11 +658,14 @@ inline bool LBIDList::compareVal(const T& Min, const T& Max, const T& value, cha return true; } -bool LBIDList::checkSingleValue(int64_t min, int64_t max, int64_t value, +template +bool LBIDList::checkSingleValue(T min, T max, T value, execplan::CalpontSystemCatalog::ColDataType type) { if (isCharType(type)) { + // MCOL-641 LBIDList::CasualPartitionDataType() returns false if + // width > 8 for a character type, so T cannot be int128_t here uint64_t mmin = order_swap(min); uint64_t mmax = order_swap(max); uint64_t vvalue = order_swap(value); @@ -616,11 +682,14 @@ bool LBIDList::checkSingleValue(int64_t min, int64_t max, int64_t value, } } -bool LBIDList::checkRangeOverlap(int64_t min, int64_t max, int64_t tmin, int64_t tmax, +template +bool LBIDList::checkRangeOverlap(T min, T max, T tmin, T tmax, execplan::CalpontSystemCatalog::ColDataType type) { if (isCharType(type)) { + // MCOL-641 LBIDList::CasualPartitionDataType() returns false if + // width > 8 for a character type, so T cannot be int128_t here uint64_t min2 = order_swap(min); uint64_t max2 = order_swap(max); uint64_t tmin2 = order_swap(tmin); @@ -638,8 +707,7 @@ bool LBIDList::checkRangeOverlap(int64_t min, int64_t max, int64_t tmin, int64_t } } -bool LBIDList::CasualPartitionPredicate(const int64_t Min, - const int64_t Max, +bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, const messageqcpp::ByteStream* bs, const uint16_t NOPS, const execplan::CalpontSystemCatalog::ColType& ct, @@ -650,8 +718,9 @@ bool LBIDList::CasualPartitionPredicate(const int64_t Min, const char* MsgDataPtr = (const char*) bs->buf(); bool scan = true; int64_t value = 0; - bool bIsUnsigned = execplan::isUnsigned(ct.colDataType); - bool bIsChar = execplan::isCharType(ct.colDataType); + int128_t bigValue = 0; + bool bIsUnsigned = datatypes::isUnsigned(ct.colDataType); + bool bIsChar = datatypes::isCharType(ct.colDataType); for (int i = 0; i < NOPS; i++) { @@ -698,6 +767,7 @@ bool LBIDList::CasualPartitionPredicate(const int64_t Min, { uint64_t val = *(int64_t*)MsgDataPtr; value = static_cast(val); + break; } } } @@ -730,12 +800,22 @@ bool LBIDList::CasualPartitionPredicate(const int64_t Min, { int64_t val = *(int64_t*)MsgDataPtr; value = val; + break; + } + + case 16: + { + datatypes::TSInt128::assignPtrPtr(&bigValue, MsgDataPtr); + break; } } } - // Should we also check for empty here? - if (isNull(value, ct)) // This will work even if the data column is unsigned. + if (ct.isWideDecimalType() && execplan::isNull(bigValue, ct)) + { + continue; + } + else if (execplan::isNull(value, ct)) // This will work even if the data column is unsigned. { continue; } @@ -746,8 +826,8 @@ bool LBIDList::CasualPartitionPredicate(const int64_t Min, { // MCOL-1246 Trim trailing whitespace for matching so that we have // the same as InnoDB behaviour - int64_t tMin = Min; - int64_t tMax = Max; + int64_t tMin = cpRange.loVal; + int64_t tMax = cpRange.hiVal; dataconvert::DataConvert::trimWhitespace(tMin); dataconvert::DataConvert::trimWhitespace(tMax); @@ -757,11 +837,18 @@ bool LBIDList::CasualPartitionPredicate(const int64_t Min, } else if (bIsUnsigned) { - scan = compareVal(static_cast(Min), static_cast(Max), static_cast(value), op, lcf); + scan = compareVal(static_cast(cpRange.loVal), static_cast(cpRange.hiVal), static_cast(value), op, lcf); } else { - scan = compareVal(Min, Max, value, op, lcf); + if (ct.colWidth != datatypes::MAXDECIMALWIDTH) + { + scan = compareVal(cpRange.loVal, cpRange.hiVal, value, op, lcf); + } + else + { + scan = compareVal(cpRange.bigLoVal, cpRange.bigHiVal, bigValue, op, lcf); + } } if (BOP == BOP_AND && !scan) @@ -816,6 +903,49 @@ void LBIDList::copyLbidList(const LBIDList& rhs) fDebug = rhs.fDebug; } +template +bool LBIDList::GetMinMax(int128_t& min, int128_t& max, int64_t& seq, int64_t lbid, + const std::vector* pEMEntries, + execplan::CalpontSystemCatalog::ColDataType colDataType); +template +bool LBIDList::GetMinMax(int64_t& min, int64_t& max, int64_t& seq, int64_t lbid, + const std::vector* pEMEntries, + execplan::CalpontSystemCatalog::ColDataType colDataType); + +template +bool LBIDList::GetMinMax(int128_t* min, int128_t* max, int64_t* seq, + int64_t lbid, const tr1::unordered_map& entries, + execplan::CalpontSystemCatalog::ColDataType colDataType); + +template +bool LBIDList::GetMinMax(int64_t* min, int64_t* max, int64_t* seq, + int64_t lbid, const tr1::unordered_map& entries, + execplan::CalpontSystemCatalog::ColDataType colDataType); + +template +void LBIDList::UpdateMinMax(int128_t min, int128_t max, int64_t lbid, + execplan::CalpontSystemCatalog::ColDataType type, bool validData = true); + +template +void LBIDList::UpdateMinMax(int64_t min, int64_t max, int64_t lbid, + execplan::CalpontSystemCatalog::ColDataType type, bool validData = true); + +template +bool LBIDList::checkSingleValue(int128_t min, int128_t max, int128_t value, + execplan::CalpontSystemCatalog::ColDataType type); + +template +bool LBIDList::checkSingleValue(int64_t min, int64_t max, int64_t value, + execplan::CalpontSystemCatalog::ColDataType type); + +template +bool LBIDList::checkRangeOverlap(int128_t min, int128_t max, int128_t tmin, int128_t tmax, + execplan::CalpontSystemCatalog::ColDataType type); + +template +bool LBIDList::checkRangeOverlap(int64_t min, int64_t max, int64_t tmin, int64_t tmax, + execplan::CalpontSystemCatalog::ColDataType type); + } //namespace joblist // vim:ts=4 sw=4: diff --git a/dbcon/joblist/lbidlist.h b/dbcon/joblist/lbidlist.h index 56b26315a..1d666cccf 100644 --- a/dbcon/joblist/lbidlist.h +++ b/dbcon/joblist/lbidlist.h @@ -50,11 +50,19 @@ struct MinMaxPartition { int64_t lbid; int64_t lbidmax; - int64_t min; - int64_t max; int64_t seq; int isValid; uint32_t blksScanned; + union + { + int128_t bigMin; + int64_t min; + }; + union + { + int128_t bigMax; + int64_t max; + }; }; /** @brief class LBIDList @@ -83,33 +91,37 @@ public: // Functions to handle min/max values per lbid for casual partitioning; // If pEMEntries is provided, then min/max will be extracted from that // vector, else extents in BRM will be searched. If type is unsigned, caller - // should static cast returned min and max to uint64_t - bool GetMinMax(int64_t& min, int64_t& max, int64_t& seq, int64_t lbid, + // should static cast returned min and max to uint64_t/int128_t + template + bool GetMinMax(T& min, T& max, int64_t& seq, int64_t lbid, const std::vector* pEMEntries, execplan::CalpontSystemCatalog::ColDataType type); - bool GetMinMax(int64_t* min, int64_t* max, int64_t* seq, int64_t lbid, + template + bool GetMinMax(T* min, T* max, int64_t* seq, int64_t lbid, const std::tr1::unordered_map& entries, execplan::CalpontSystemCatalog::ColDataType type); - void UpdateMinMax(int64_t min, int64_t max, int64_t lbid, + template + void UpdateMinMax(T min, T max, int64_t lbid, execplan::CalpontSystemCatalog::ColDataType type, bool validData = true); - void UpdateAllPartitionInfo(); + void UpdateAllPartitionInfo(const execplan::CalpontSystemCatalog::ColType& colType); bool IsRangeBoundary(uint64_t lbid); - bool CasualPartitionPredicate(const int64_t Min, - const int64_t Max, + bool CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, const messageqcpp::ByteStream* MsgDataPtr, const uint16_t NOPS, const execplan::CalpontSystemCatalog::ColType& ct, const uint8_t BOP); - bool checkSingleValue(int64_t min, int64_t max, int64_t value, + template + bool checkSingleValue(T min, T max, T value, execplan::CalpontSystemCatalog::ColDataType type); - bool checkRangeOverlap(int64_t min, int64_t max, int64_t tmin, int64_t tmax, + template + bool checkRangeOverlap(T min, T max, T tmin, T tmax, execplan::CalpontSystemCatalog::ColDataType type); // check the column data type and the column size to determine if it @@ -135,7 +147,8 @@ private: template inline bool compareVal(const T& Min, const T& Max, const T& value, char op, uint8_t lcf); - int getMinMaxFromEntries(int64_t& min, int64_t& max, int32_t& seq, + template + int getMinMaxFromEntries(T& min, T& max, int32_t& seq, int64_t lbid, const std::vector& EMEntries); boost::shared_ptr em; diff --git a/dbcon/joblist/orderby-tests.cpp b/dbcon/joblist/orderby-tests.cpp index fa98f0772..792f65596 100644 --- a/dbcon/joblist/orderby-tests.cpp +++ b/dbcon/joblist/orderby-tests.cpp @@ -174,6 +174,7 @@ private: // create two columns RG. 1st is the sorting key, second is the data column std::vector offsets, roids, tkeys, cscale, cprecision; + std::vector charSetNumVec; std::vector types; offsets.push_back(2); offsets.push_back(10); offsets.push_back(18); roids.push_back(oid); roids.push_back(oid); @@ -182,11 +183,13 @@ private: types.push_back(execplan::CalpontSystemCatalog::UBIGINT); cscale.push_back(0); cscale.push_back(0); cprecision.push_back(20); cprecision.push_back(20); + charSetNumVec.push_back(8); charSetNumVec.push_back(8); rowgroup::RowGroup inRG(2, //column count offsets, //oldOffset roids, // column oids tkeys, //keys types, // types + charSetNumVec, // charset numbers cscale, //scale cprecision, // precision 20, // sTableThreshold diff --git a/dbcon/joblist/passthrucommand-jl.cpp b/dbcon/joblist/passthrucommand-jl.cpp index a11571c87..9fcf79131 100644 --- a/dbcon/joblist/passthrucommand-jl.cpp +++ b/dbcon/joblist/passthrucommand-jl.cpp @@ -73,7 +73,12 @@ PassThruCommandJL::PassThruCommandJL(const PassThruStep& p) case 8: tableColumnType = TableColumn::UINT64; break; - + + case 16: + case 32: + tableColumnType = TableColumn::STRING; + break; + default: throw logic_error("PassThruCommandJL(): bad column width?"); } diff --git a/dbcon/joblist/pcolscan.cpp b/dbcon/joblist/pcolscan.cpp index da35ad8ce..c870eca56 100644 --- a/dbcon/joblist/pcolscan.cpp +++ b/dbcon/joblist/pcolscan.cpp @@ -148,11 +148,6 @@ pColScanStep::pColScanStep( int err, i, mask; BRM::LBIDRange_v::iterator it; - //pthread_mutex_init(&mutex, NULL); - //pthread_mutex_init(&dlMutex, NULL); - //pthread_mutex_init(&cpMutex, NULL); - //pthread_cond_init(&condvar, NULL); - //pthread_cond_init(&condvarWakeupProducer, NULL); finishedSending = false; recvWaiting = 0; recvExited = 0; @@ -178,7 +173,11 @@ pColScanStep::pColScanStep( fColType.colWidth = 8; fIsDict = true; } - else if (fColType.colWidth > 8 ) + // MCOL-641 WIP + else if (fColType.colWidth > 8 + && fColType.colDataType != CalpontSystemCatalog::BINARY + && fColType.colDataType != CalpontSystemCatalog::DECIMAL + && fColType.colDataType != CalpontSystemCatalog::UDECIMAL) { fColType.colWidth = 8; fIsDict = true; diff --git a/dbcon/joblist/pcolstep.cpp b/dbcon/joblist/pcolstep.cpp index b9809e8e0..722fb57ce 100644 --- a/dbcon/joblist/pcolstep.cpp +++ b/dbcon/joblist/pcolstep.cpp @@ -177,7 +177,11 @@ pColStep::pColStep( fColType.colWidth = 8; fIsDict = true; } - else if (fColType.colWidth > 8 ) + // WIP MCOL-641 + else if (fColType.colWidth > 8 + && fColType.colDataType != CalpontSystemCatalog::BINARY + && fColType.colDataType != CalpontSystemCatalog::DECIMAL + && fColType.colDataType != CalpontSystemCatalog::UDECIMAL) { fColType.colWidth = 8; fIsDict = true; @@ -630,6 +634,18 @@ void pColStep::addFilter(int8_t COP, int64_t value, uint8_t roundFlag) fFilterCount++; } +// WIP MCOL-641 +void pColStep::addFilter(int8_t COP, const int128_t& value, uint8_t roundFlag) +{ + fFilterString << (uint8_t) COP; + fFilterString << roundFlag; + + // bitwise copies into the filter ByteStream + fFilterString << *reinterpret_cast(&value); + + fFilterCount++; +} + void pColStep::setRidList(DataList* dl) { ridList = dl; @@ -723,8 +739,8 @@ void pColStep::sendPrimitiveMessages() // { // // bool flag = lbidList->CasualPartitionPredicate( -// extents[idx].partition.cprange.lo_val, -// extents[idx].partition.cprange.hi_val, +// extents[idx].partition.cprange.loVal, +// extents[idx].partition.cprange.hiVal, // &fFilterString, // fFilterCount, // fColType, @@ -733,8 +749,8 @@ void pColStep::sendPrimitiveMessages() //#ifdef DEBUG // if (fOid >= 3000 && flushInterval == 0) // cout << (flag ? " will scan " : " will not scan ") -// << "extent with range " << extents[idx].partition.cprange.lo_val -// << "-" << extents[idx].partition.cprange.hi_val << endl; +// << "extent with range " << extents[idx].partition.cprange.loVal +// << "-" << extents[idx].partition.cprange.hiVal << endl; //#endif // // } diff --git a/dbcon/joblist/primitivemsg.h b/dbcon/joblist/primitivemsg.h index f2cad3e76..7660f38ef 100644 --- a/dbcon/joblist/primitivemsg.h +++ b/dbcon/joblist/primitivemsg.h @@ -190,14 +190,15 @@ enum ISMPACKETCOMMAND #undef PRIM_DELIVERBASE /* Flags for BPP messages */ -const uint8_t NEED_STR_VALUES = 0x01; //1; -const uint8_t GOT_ABS_RIDS = 0x02; //2; -const uint8_t GOT_VALUES = 0x04; //4; -const uint8_t LBID_TRACE = 0x08; //8; -const uint8_t HAS_JOINER = 0x10; //16; -const uint8_t SEND_RIDS_AT_DELIVERY = 0x20; //32; -const uint8_t HAS_ROWGROUP = 0x40; //64; -const uint8_t JOIN_ROWGROUP_DATA = 0x80; //128 +const uint16_t NEED_STR_VALUES = 0x01; //1; +const uint16_t GOT_ABS_RIDS = 0x02; //2; +const uint16_t GOT_VALUES = 0x04; //4; +const uint16_t LBID_TRACE = 0x08; //8; +const uint16_t HAS_JOINER = 0x10; //16; +const uint16_t SEND_RIDS_AT_DELIVERY = 0x20; //32; +const uint16_t HAS_ROWGROUP = 0x40; //64; +const uint16_t JOIN_ROWGROUP_DATA = 0x80; //128 +const uint16_t HAS_WIDE_COLUMNS = 0x100; //256; //TODO: put this in a namespace to stop global ns pollution enum PrimFlags @@ -704,8 +705,8 @@ struct NewColResultHeader uint16_t NVALS; uint16_t ValidMinMax; // 1 if Min/Max are valid, otherwise 0 uint32_t OutputType; - int64_t Min; // Minimum value in this block for signed data types - int64_t Max; // Maximum value in this block for signed data types + int128_t Min; // Minimum value in this block for signed data types + int128_t Max; // Maximum value in this block for signed data types uint32_t CacheIO; // I/O count from buffer cache uint32_t PhysicalIO; // Physical I/O count from disk // if OutputType was OT_DATAVALUE, what follows is DataType[NVALS] diff --git a/dbcon/joblist/primitivestep.h b/dbcon/joblist/primitivestep.h index 35ecf86de..d594685b1 100644 --- a/dbcon/joblist/primitivestep.h +++ b/dbcon/joblist/primitivestep.h @@ -46,6 +46,7 @@ #include #include +#include "mcs_basic_types.h" #include "calpontsystemcatalog.h" #include "calpontselectexecutionplan.h" #include "brm.h" @@ -195,6 +196,8 @@ public: */ void addFilter(int8_t COP, int64_t value, uint8_t roundFlag = 0); void addFilter(int8_t COP, float value); + // WIP MCOL-641 + void addFilter(int8_t COP, const int128_t& value, uint8_t roundFlag = 0); /** @brief Sets the DataList to get RID values from. * @@ -1325,7 +1328,8 @@ public: * Note that it is an adder not a setter. For an extent to be scanned, all calls * must have a non-empty intersection. */ - void addCPPredicates(uint32_t OID, const std::vector& vals, bool isRange); + void addCPPredicates(uint32_t OID, const std::vector& vals, bool isRange, + bool isSmallSideWideDecimal); /* semijoin adds */ void setJoinFERG(const rowgroup::RowGroup& rg); @@ -1507,13 +1511,16 @@ private: /* Pseudo column filter processing. Think about refactoring into a separate class. */ bool processPseudoColFilters(uint32_t extentIndex, boost::shared_ptr > dbRootPMMap) const; - bool processOneFilterType(int8_t colWidth, int64_t value, uint32_t type) const; - bool processSingleFilterString(int8_t BOP, int8_t colWidth, int64_t val, const uint8_t* filterString, + template + bool processOneFilterType(int8_t colWidth, T value, uint32_t type) const; + template + bool processSingleFilterString(int8_t BOP, int8_t colWidth, T val, const uint8_t* filterString, uint32_t filterCount) const; bool processSingleFilterString_ranged(int8_t BOP, int8_t colWidth, int64_t min, int64_t max, const uint8_t* filterString, uint32_t filterCount) const; bool processLBIDFilter(const BRM::EMEntry& emEntry) const; - bool compareSingleValue(uint8_t COP, int64_t val1, int64_t val2) const; + template + bool compareSingleValue(uint8_t COP, T val1, T val2) const; bool compareRange(uint8_t COP, int64_t min, int64_t max, int64_t val) const; bool hasPCFilter, hasPMFilter, hasRIDFilter, hasSegmentFilter, hasDBRootFilter, hasSegmentDirFilter, hasPartitionFilter, hasMaxFilter, hasMinFilter, hasLBIDFilter, hasExtentIDFilter; diff --git a/dbcon/joblist/pseudocc-jl.cpp b/dbcon/joblist/pseudocc-jl.cpp index 8ad0ccb2e..7a48fb096 100644 --- a/dbcon/joblist/pseudocc-jl.cpp +++ b/dbcon/joblist/pseudocc-jl.cpp @@ -46,23 +46,57 @@ void PseudoCCJL::runCommand(ByteStream& bs) const { if (function == PSEUDO_EXTENTMAX) { - int64_t max = extents[currentExtentIndex].partition.cprange.hi_val; - int64_t min = extents[currentExtentIndex].partition.cprange.lo_val; + if (!colType.isWideDecimalType()) + { + int64_t max = extents[currentExtentIndex].partition.cprange.hiVal; + int64_t min = extents[currentExtentIndex].partition.cprange.loVal; - if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) - bs << max; + if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) + bs << max; + else + bs << utils::getNullValue(colType.colDataType, colType.colWidth); + } else - bs << utils::getNullValue(colType.colDataType, colType.colWidth); + { + int128_t max = extents[currentExtentIndex].partition.cprange.bigHiVal; + int128_t min = extents[currentExtentIndex].partition.cprange.bigLoVal; + + if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) + bs << (uint128_t) max; + else + { + int128_t int128Null; + datatypes::Decimal::setWideDecimalNullValue(int128Null); + bs << (uint128_t) int128Null; + } + } } else if (function == PSEUDO_EXTENTMIN) { - int64_t max = extents[currentExtentIndex].partition.cprange.hi_val; - int64_t min = extents[currentExtentIndex].partition.cprange.lo_val; + if (!colType.isWideDecimalType()) + { + int64_t max = extents[currentExtentIndex].partition.cprange.hiVal; + int64_t min = extents[currentExtentIndex].partition.cprange.loVal; - if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) - bs << min; + if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) + bs << min; + else + bs << utils::getNullValue(colType.colDataType, colType.colWidth); + } else - bs << utils::getNullValue(colType.colDataType, colType.colWidth); + { + int128_t max = extents[currentExtentIndex].partition.cprange.bigHiVal; + int128_t min = extents[currentExtentIndex].partition.cprange.bigLoVal; + + if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) + bs << (uint128_t) min; + else + { + int128_t int128Null; + datatypes::Decimal::setWideDecimalNullValue(int128Null); + bs << (uint128_t) int128Null; + } + } } else if (function == PSEUDO_EXTENTID) bs << extents[currentExtentIndex].range.start; diff --git a/dbcon/joblist/rowestimator.cpp b/dbcon/joblist/rowestimator.cpp index 21369332f..d40d15248 100644 --- a/dbcon/joblist/rowestimator.cpp +++ b/dbcon/joblist/rowestimator.cpp @@ -140,12 +140,13 @@ uint64_t RowEstimator::adjustValue(const execplan::CalpontSystemCatalog::ColType // Estimates the number of distinct values given a min/max range. When the range has not been set, // rules from the requirements are used based on the column type. +template uint32_t RowEstimator::estimateDistinctValues(const execplan::CalpontSystemCatalog::ColType& ct, - const uint64_t& min, - const uint64_t& max, + const T& min, + const T& max, const char cpStatus) { - uint64_t ret = 10; + T ret = 10; // If no casual partitioning info available for extent. These rules were defined in the requirements. if (cpStatus != BRM::CP_VALID) @@ -210,7 +211,10 @@ uint32_t RowEstimator::estimateDistinctValues(const execplan::CalpontSystemCatal // Returns a floating point number between 0 and 1 representing the percentage of matching rows for the given predicate against // the given range. This function is used for estimating an individual operation such as col1 = 2. template -float RowEstimator::estimateOpFactor(const T& min, const T& max, const T& value, char op, uint8_t lcf, uint32_t distinctValues, char cpStatus) +float RowEstimator::estimateOpFactor(const T& min, const T& max, const T& value, + char op, uint8_t lcf, + uint32_t distinctValues, char cpStatus, + const execplan::CalpontSystemCatalog::ColType& ct) { float factor = 1.0; @@ -220,7 +224,10 @@ float RowEstimator::estimateOpFactor(const T& min, const T& max, const T& value, case COMPARE_NGE: if (cpStatus == BRM::CP_VALID) { - factor = (1.0 * value - min) / (max - min + 1); + if (!ct.isWideDecimalType()) + factor = (1.0 * value - min) / (max - min + 1); + else + factor = ((__float128) value - min) / (max - min + 1); } break; @@ -229,7 +236,10 @@ float RowEstimator::estimateOpFactor(const T& min, const T& max, const T& value, case COMPARE_NGT: if (cpStatus == BRM::CP_VALID) { - factor = (1.0 * value - min + 1) / (max - min + 1); + if (!ct.isWideDecimalType()) + factor = (1.0 * value - min + 1) / (max - min + 1); + else + factor = ((__float128) value - min + 1) / (max - min + 1); } break; @@ -238,7 +248,10 @@ float RowEstimator::estimateOpFactor(const T& min, const T& max, const T& value, case COMPARE_NLE: if (cpStatus == BRM::CP_VALID) { - factor = (1.0 * max - value) / (1.0 * max - min + 1); + if (!ct.isWideDecimalType()) + factor = (1.0 * max - value) / (1.0 * max - min + 1); + else + factor = ((__float128) max - value) / (max - min + 1); } break; @@ -248,7 +261,10 @@ float RowEstimator::estimateOpFactor(const T& min, const T& max, const T& value, if (cpStatus == BRM::CP_VALID) { // TODO: Best way to convert to floating point arithmetic? - factor = (1.0 * max - value + 1) / (max - min + 1); + if (!ct.isWideDecimalType()) + factor = (1.0 * max - value + 1) / (max - min + 1); + else + factor = ((__float128) max - value + 1) / (max - min + 1); } break; @@ -283,15 +299,30 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, const uint8_t BOP, const uint32_t& rowsInExtent) { - bool bIsUnsigned = execplan::isUnsigned(ct.colDataType); + bool bIsUnsigned = datatypes::isUnsigned(ct.colDataType); float factor = 1.0; float tempFactor = 1.0; + uint64_t adjustedMin = 0, adjustedMax = 0; + uint128_t adjustedBigMin, adjustedBigMax; + uint32_t distinctValuesEstimate; + // Adjust values based on column type and estimate the - uint64_t adjustedMin = adjustValue(ct, emEntry.partition.cprange.lo_val); - uint64_t adjustedMax = adjustValue(ct, emEntry.partition.cprange.hi_val); - uint32_t distinctValuesEstimate = estimateDistinctValues( - ct, adjustedMin, adjustedMax, emEntry.partition.cprange.isValid); + if (!ct.isWideDecimalType()) + { + adjustedMin = adjustValue(ct, emEntry.partition.cprange.loVal); + adjustedMax = adjustValue(ct, emEntry.partition.cprange.hiVal); + distinctValuesEstimate = estimateDistinctValues( + ct, adjustedMin, adjustedMax, emEntry.partition.cprange.isValid); + } + else + { + adjustedBigMin = emEntry.partition.cprange.bigLoVal; + adjustedBigMax = emEntry.partition.cprange.bigHiVal; + distinctValuesEstimate = estimateDistinctValues( + ct, adjustedBigMin, adjustedBigMax, emEntry.partition.cprange.isValid); + } + // Loop through the operations and estimate the percentage of rows that will qualify. // For example, there are two operations for "col1 > 5 and col1 < 10": @@ -300,6 +331,7 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, int length = bs->length(), pos = 0; const char* msgDataPtr = (const char*) bs->buf(); int64_t value = 0; + int128_t bigValue = 0; bool firstQualifyingOrCondition = true; uint16_t comparisonLimit = (NOPS <= fMaxComparisons) ? NOPS : fMaxComparisons; @@ -343,6 +375,18 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, break; } + case 16: + { + if (ct.colDataType == execplan::CalpontSystemCatalog::DECIMAL || + ct.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) + { + uint128_t val = *(uint128_t*)msgDataPtr; + bigValue = static_cast(val); + break; + } + } + // fallthrough + case 8: default: { @@ -377,6 +421,18 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, break; } + case 16: + { + if (ct.colDataType == execplan::CalpontSystemCatalog::DECIMAL || + ct.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) + { + int128_t val = *(int128_t*)msgDataPtr; + bigValue = static_cast(val); + break; + } + } + // fallthrough + case 8: default: { @@ -396,23 +452,41 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, } #if ROW_EST_DEBUG - cout << " Min-" << emEntry.partition.cprange.lo_val << - ", Max-" << emEntry.partition.cprange.hi_val << + cout << " Min-" << emEntry.partition.cprange.loVal << + ", Max-" << emEntry.partition.cprange.hiVal << ", Val-" << value; #endif // Get the factor for the individual operation. if (bIsUnsigned) { - tempFactor = estimateOpFactor( - adjustedMin, adjustedMax, adjustValue(ct, value), op, lcf, - distinctValuesEstimate, emEntry.partition.cprange.isValid); + if (!ct.isWideDecimalType()) + { + tempFactor = estimateOpFactor( + adjustedMin, adjustedMax, adjustValue(ct, value), op, lcf, + distinctValuesEstimate, emEntry.partition.cprange.isValid, ct); + } + else + { + tempFactor = estimateOpFactor( + adjustedBigMin, adjustedBigMax, bigValue, op, lcf, + distinctValuesEstimate, emEntry.partition.cprange.isValid, ct); + } } else { - tempFactor = estimateOpFactor( - adjustedMin, adjustedMax, adjustValue(ct, value), op, lcf, - distinctValuesEstimate, emEntry.partition.cprange.isValid); + if (!ct.isWideDecimalType()) + { + tempFactor = estimateOpFactor( + adjustedMin, adjustedMax, adjustValue(ct, value), op, lcf, + distinctValuesEstimate, emEntry.partition.cprange.isValid, ct); + } + else + { + tempFactor = estimateOpFactor( + adjustedBigMin, adjustedBigMax, bigValue, op, lcf, + distinctValuesEstimate, emEntry.partition.cprange.isValid, ct); + } } #if ROW_EST_DEBUG @@ -611,4 +685,5 @@ uint64_t RowEstimator::estimateRowsForNonCPColumn(ColumnCommandJL& colCmd) return estimatedRows; } + } //namespace joblist diff --git a/dbcon/joblist/rowestimator.h b/dbcon/joblist/rowestimator.h index 477bc2b30..a4c702a44 100644 --- a/dbcon/joblist/rowestimator.h +++ b/dbcon/joblist/rowestimator.h @@ -89,9 +89,10 @@ private: uint32_t daysThroughMonth(uint32_t mth); + template uint32_t estimateDistinctValues(const execplan::CalpontSystemCatalog::ColType& ct, - const uint64_t& min, - const uint64_t& max, + const T& min, + const T& max, const char cpStatus); /** @brief returns a factor between 0 and 1 for the estimate of rows that will qualify the given individual operation. @@ -106,7 +107,8 @@ private: */ template float estimateOpFactor(const T& min, const T& max, const T& value, char op, uint8_t lcf, - uint32_t distinctValues, char cpStatus); + uint32_t distinctValues, char cpStatus, + const execplan::CalpontSystemCatalog::ColType& ct); /** @brief returns a factor between 0 and 1 for the estimate of rows that will qualify * the given operation(s). diff --git a/dbcon/joblist/tuple-bps.cpp b/dbcon/joblist/tuple-bps.cpp index 02b479886..2d91f87de 100644 --- a/dbcon/joblist/tuple-bps.cpp +++ b/dbcon/joblist/tuple-bps.cpp @@ -838,8 +838,7 @@ void TupleBPS::storeCasualPartitionInfo(const bool estimateRowCounts) scanFlags[idx] = scanFlags[idx] && (ignoreCP || extent.partition.cprange.isValid != BRM::CP_VALID || lbidListVec[i]->CasualPartitionPredicate( - extent.partition.cprange.lo_val, - extent.partition.cprange.hi_val, + extent.partition.cprange, &(colCmd->getFilterString()), colCmd->getFilterCount(), colCmd->getColType(), @@ -928,7 +927,8 @@ void TupleBPS::prepCasualPartitioning() { uint32_t i; int64_t min, max, seq; - boost::mutex::scoped_lock lk(cpMutex); + int128_t bigMin, bigMax; + boost::mutex::scoped_lock lk(cpMutex); for (i = 0; i < scannedExtents.size(); i++) { @@ -938,8 +938,18 @@ void TupleBPS::prepCasualPartitioning() if (scanFlags[i] && lbidList->CasualPartitionDataType(fColType.colDataType, fColType.colWidth)) - lbidList->GetMinMax(min, max, seq, (int64_t) scannedExtents[i].range.start, - &scannedExtents, fColType.colDataType); + { + if (fColType.colWidth <= 8) + { + lbidList->GetMinMax(min, max, seq, (int64_t) scannedExtents[i].range.start, + &scannedExtents, fColType.colDataType); + } + else if (fColType.colWidth == 16) + { + lbidList->GetMinMax(bigMin, bigMax, seq, (int64_t) scannedExtents[i].range.start, + &scannedExtents, fColType.colDataType); + } + } } else scanFlags[i] = true; @@ -1411,7 +1421,8 @@ void TupleBPS::sendJobs(const vector& jobs) } } -bool TupleBPS::compareSingleValue(uint8_t COP, int64_t val1, int64_t val2) const +template +bool TupleBPS::compareSingleValue(uint8_t COP, T val1, T val2) const { switch (COP) { @@ -1526,7 +1537,8 @@ bool TupleBPS::processSingleFilterString_ranged(int8_t BOP, int8_t colWidth, int return ret; } -bool TupleBPS::processSingleFilterString(int8_t BOP, int8_t colWidth, int64_t val, const uint8_t* filterString, +template +bool TupleBPS::processSingleFilterString(int8_t BOP, int8_t colWidth, T val, const uint8_t* filterString, uint32_t filterCount) const { uint j; @@ -1536,6 +1548,7 @@ bool TupleBPS::processSingleFilterString(int8_t BOP, int8_t colWidth, int64_t va { int8_t COP; int64_t val2; + int128_t bigVal2; bool thisPredicate; COP = *filterString++; filterString++; // skip the round var, don't think that applies here @@ -1562,11 +1575,20 @@ bool TupleBPS::processSingleFilterString(int8_t BOP, int8_t colWidth, int64_t va filterString += 8; break; + case 16: + bigVal2 = *((int128_t*) filterString); + filterString += 16; + break; + default: throw logic_error("invalid column width"); } - thisPredicate = compareSingleValue(COP, val, val2); + // Assumption is that colWidth > 0 + if (static_cast(colWidth) < datatypes::MAXDECIMALWIDTH) + thisPredicate = compareSingleValue(COP, (int64_t) val, val2); + else + thisPredicate = compareSingleValue(COP, (int128_t) val, bigVal2); if (j == 0) ret = thisPredicate; @@ -1580,7 +1602,8 @@ bool TupleBPS::processSingleFilterString(int8_t BOP, int8_t colWidth, int64_t va return ret; } -bool TupleBPS::processOneFilterType(int8_t colWidth, int64_t value, uint32_t type) const +template +bool TupleBPS::processOneFilterType(int8_t colWidth, T value, uint32_t type) const { const vector& filters = fBPP->getFilterSteps(); uint i; @@ -1671,9 +1694,13 @@ bool TupleBPS::processPseudoColFilters(uint32_t extentIndex, boost::shared_ptr fromPrimProc; bool validCPData; - int64_t min; - int64_t max; + bool hasBinaryColumn; + int128_t min; + int128_t max; uint64_t lbid; vector<_CPInfo> cpv; uint32_t cachedIO; @@ -2154,7 +2195,7 @@ void TupleBPS::receiveMultiPrimitiveMessages(uint32_t threadID) fromPrimProc.clear(); fBPP->getRowGroupData(*bs, &fromPrimProc, &validCPData, &lbid, &min, &max, - &cachedIO, &physIO, &touchedBlocks, &unused, threadID); + &cachedIO, &physIO, &touchedBlocks, &unused, threadID, &hasBinaryColumn, fColType); /* Another layer of messiness. Need to refactor this fcn. */ while (!fromPrimProc.empty() && !cancelled()) @@ -2318,7 +2359,16 @@ void TupleBPS::receiveMultiPrimitiveMessages(uint32_t threadID) touchedBlocks_Thread += touchedBlocks; if (fOid >= 3000 && ffirstStepType == SCAN && bop == BOP_AND) - cpv.push_back(_CPInfo(min, max, lbid, validCPData)); + { + if (fColType.colWidth <= 8) + { + cpv.push_back(_CPInfo((int64_t) min, (int64_t) max, lbid, validCPData)); + } + else if (fColType.colWidth == 16) + { + cpv.push_back(_CPInfo(min, max, lbid, validCPData)); + } + } } // end of the per-rowgroup processing loop // insert the resulting rowgroup data from a single bytestream into dlp @@ -2344,8 +2394,16 @@ void TupleBPS::receiveMultiPrimitiveMessages(uint32_t threadID) for (i = 0; i < size; i++) { - lbidList->UpdateMinMax(cpv[i].min, cpv[i].max, cpv[i].LBID, fColType.colDataType, - cpv[i].valid); + if (fColType.colWidth > 8) + { + lbidList->UpdateMinMax(cpv[i].bigMin, cpv[i].bigMax, cpv[i].LBID, fColType.colDataType, + cpv[i].valid); + } + else + { + lbidList->UpdateMinMax(cpv[i].min, cpv[i].max, cpv[i].LBID, fColType.colDataType, + cpv[i].valid); + } } cpMutex.unlock(); @@ -2619,7 +2677,7 @@ out: if (ffirstStepType == SCAN && bop == BOP_AND && !cancelled()) { cpMutex.lock(); - lbidList->UpdateAllPartitionInfo(); + lbidList->UpdateAllPartitionInfo(fColType); cpMutex.unlock(); } } @@ -3155,7 +3213,8 @@ void TupleBPS::setJoinFERG(const RowGroup& rg) fBPP->setJoinFERG(rg); } -void TupleBPS::addCPPredicates(uint32_t OID, const vector& vals, bool isRange) +void TupleBPS::addCPPredicates(uint32_t OID, const vector& vals, bool isRange, + bool isSmallSideWideDecimal) { if (fTraceFlags & CalpontSelectExecutionPlan::IGNORE_CP || fOid < 3000) @@ -3163,6 +3222,7 @@ void TupleBPS::addCPPredicates(uint32_t OID, const vector& vals, bool i uint32_t i, j, k; int64_t min, max, seq; + int128_t bigMin, bigMax; bool isValid, intersection; vector colCmdVec = fBPP->getFilterSteps(); ColumnCommandJL* cmd; @@ -3186,7 +3246,9 @@ void TupleBPS::addCPPredicates(uint32_t OID, const vector& vals, bool i if (cmd != NULL && cmd->getOID() == OID) { - if (!ll.CasualPartitionDataType(cmd->getColType().colDataType, cmd->getColType().colWidth) + const execplan::CalpontSystemCatalog::ColType& colType = cmd->getColType(); + + if (!ll.CasualPartitionDataType(colType.colDataType, colType.colWidth) || cmd->isDict()) return; @@ -3213,25 +3275,73 @@ void TupleBPS::addCPPredicates(uint32_t OID, const vector& vals, bool i extentsPtr = &mref; } - for (j = 0; j < extents.size(); j++) + if (colType.colWidth <= 8) { - isValid = ll.GetMinMax(&min, &max, &seq, extents[j].range.start, *extentsPtr, - cmd->getColType().colDataType); - - if (isValid) + for (j = 0; j < extents.size(); j++) { - if (isRange) - runtimeCPFlags[j] = ll.checkRangeOverlap(min, max, vals[0], vals[1], - cmd->getColType().colDataType) && runtimeCPFlags[j]; - else + isValid = ll.GetMinMax(&min, &max, &seq, extents[j].range.start, *extentsPtr, + colType.colDataType); + + if (isValid) { - intersection = false; + if (isRange) + { + if (!isSmallSideWideDecimal) + { + runtimeCPFlags[j] = ll.checkRangeOverlap(min, max, (int64_t) vals[0], (int64_t) vals[1], + colType.colDataType) && runtimeCPFlags[j]; + } + else + { + runtimeCPFlags[j] = ll.checkRangeOverlap((int128_t) min, (int128_t) max, vals[0], vals[1], + colType.colDataType) && runtimeCPFlags[j]; + } + } + else + { + intersection = false; - for (k = 0; k < vals.size(); k++) - intersection = intersection || - ll.checkSingleValue(min, max, vals[k], cmd->getColType().colDataType); + for (k = 0; k < vals.size(); k++) + { + if (!isSmallSideWideDecimal) + { + intersection = intersection || + ll.checkSingleValue(min, max, (int64_t) vals[k], colType.colDataType); + } + else + { + intersection = intersection || + ll.checkSingleValue((int128_t) min, (int128_t) max, vals[k], colType.colDataType); + } + } - runtimeCPFlags[j] = intersection && runtimeCPFlags[j]; + runtimeCPFlags[j] = intersection && runtimeCPFlags[j]; + } + } + } + } + else + { + for (j = 0; j < extents.size(); j++) + { + isValid = ll.GetMinMax(&bigMin, &bigMax, &seq, extents[j].range.start, *extentsPtr, + colType.colDataType); + + if (isValid) + { + if (isRange) + runtimeCPFlags[j] = ll.checkRangeOverlap(bigMin, bigMax, vals[0], vals[1], + colType.colDataType) && runtimeCPFlags[j]; + else + { + intersection = false; + + for (k = 0; k < vals.size(); k++) + intersection = intersection || + ll.checkSingleValue(bigMin, bigMax, vals[k], colType.colDataType); + + runtimeCPFlags[j] = intersection && runtimeCPFlags[j]; + } } } } @@ -3289,5 +3399,22 @@ void TupleBPS::abort() abort_nolock(); } +template +bool TupleBPS::processOneFilterType(int8_t colWidth, int64_t value, uint32_t type) const; +template +bool TupleBPS::processOneFilterType(int8_t colWidth, int128_t value, uint32_t type) const; + +template +bool TupleBPS::processSingleFilterString(int8_t BOP, int8_t colWidth, int64_t val, const uint8_t* filterString, + uint32_t filterCount) const; +template +bool TupleBPS::processSingleFilterString(int8_t BOP, int8_t colWidth, int128_t val, const uint8_t* filterString, + uint32_t filterCount) const; + +template +bool TupleBPS::compareSingleValue(uint8_t COP, int64_t val1, int64_t val2) const; +template +bool TupleBPS::compareSingleValue(uint8_t COP, int128_t val1, int128_t val2) const; + } //namespace // vim:ts=4 sw=4: diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index fd256f36b..4a6e39324 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -106,17 +106,14 @@ struct cmpTuple return true; if (pUDAFa == pUDAFb) { - if (pUDAFa == NULL) - return false; std::vector* paramKeysa = boost::get<3>(a); std::vector* paramKeysb = boost::get<3>(b); - + if (paramKeysa == NULL || paramKeysb == NULL) + return false; if (paramKeysa->size() < paramKeysb->size()) return true; if (paramKeysa->size() == paramKeysb->size()) { - if (paramKeysa == NULL) - return false; for (uint64_t i = 0; i < paramKeysa->size(); ++i) { if ((*paramKeysa)[i] < (*paramKeysb)[i]) @@ -343,6 +340,33 @@ string keyName(uint64_t i, uint32_t key, const joblist::JobInfo& jobInfo) namespace joblist { +void wideDecimalOrLongDouble(const uint64_t colProj, + const CalpontSystemCatalog::ColDataType type, + const vector& precisionProj, + const vector& scaleProj, + const vector& width, + vector& typeAgg, + vector& scaleAgg, + vector& precisionAgg, + vector& widthAgg) +{ + if ((type == CalpontSystemCatalog::DECIMAL + || type == CalpontSystemCatalog::UDECIMAL) + && datatypes::Decimal::isWideDecimalTypeByPrecision(precisionProj[colProj])) + { + typeAgg.push_back(type); + scaleAgg.push_back(scaleProj[colProj]); + precisionAgg.push_back(precisionProj[colProj]); + widthAgg.push_back(width[colProj]); + } + else + { + typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); + scaleAgg.push_back(0); + precisionAgg.push_back(-1); + widthAgg.push_back(sizeof(long double)); + } +} TupleAggregateStep::TupleAggregateStep( const SP_ROWAGG_UM_t& agg, @@ -717,25 +741,56 @@ void TupleAggregateStep::configDeliveredRowGroup(const JobInfo& jobInfo) // correct the scale vector scale = fRowGroupOut.getScale(); + vector precision = fRowGroupOut.getPrecision(); -// for (uint64_t i = 0; i < scale.size(); i++) -// { - // to support CNX_DECIMAL_SCALE the avg column's scale is coded with two scales: - // fe's avg column scale << 8 + original column scale - //if ((scale[i] & 0x0000FF00) > 0) -// scale[i] = scale[i] & 0x000000FF; -// } - - size_t retColCount = jobInfo.nonConstDelCols.size(); + size_t retColCount = 0; + auto scaleIter = scale.begin(); + auto precisionIter = precision.begin(); if (jobInfo.havingStep) + { retColCount = jobInfo.returnedColVec.size(); + idbassert(jobInfo.returnedColVec.size() == jobInfo.nonConstCols.size()); + + for (size_t i = 0; i < jobInfo.nonConstCols.size() && + scaleIter != scale.end(); i++) + { + const auto& colType = jobInfo.nonConstCols[i]->resultType(); + + if (colType.isWideDecimalType()) + { + *scaleIter = colType.scale; + *precisionIter = colType.precision; + } + + scaleIter++; precisionIter++; + } + } + else + { + retColCount = jobInfo.nonConstDelCols.size(); + + for (size_t i = 0; i < jobInfo.nonConstDelCols.size() && + scaleIter != scale.end(); i++) + { + const auto& colType = jobInfo.nonConstDelCols[i]->resultType(); + + if (colType.isWideDecimalType()) + { + *scaleIter = colType.scale; + *precisionIter = colType.precision; + } + + scaleIter++; precisionIter++; + } + } + vector::const_iterator offsets0 = fRowGroupOut.getOffsets().begin(); vector::const_iterator types0 = fRowGroupOut.getColTypes().begin(); vector csNums = fRowGroupOut.getCharsetNumbers(); - vector::const_iterator precision0 = fRowGroupOut.getPrecision().begin(); + vector::const_iterator precision0 = precision.begin(); fRowGroupDelivered = RowGroup(retColCount, vector(offsets0, offsets0 + retColCount + 1), vector(oids.begin(), oids.begin() + retColCount), @@ -896,7 +951,6 @@ SJSTEP TupleAggregateStep::prepAggregate(SJSTEP& step, JobInfo& jobInfo) // preprocess the columns used by group_concat jobInfo.groupConcatInfo.prepGroupConcat(jobInfo); bool doUMOnly = jobInfo.groupConcatInfo.columns().size() > 0 -// || jobInfo.windowSet.size() > 0 || sas || ces; @@ -1304,13 +1358,13 @@ void TupleAggregateStep::prep1PhaseAggregate( throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); } + wideDecimalOrLongDouble(colProj, typeProj[colProj], + precisionProj, scaleProj, width, + typeAgg, scaleAgg, precisionAgg, widthAgg); + oidsAgg.push_back(oidsProj[colProj]); keysAgg.push_back(key); - typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); csNumAgg.push_back(csNumProj[colProj]); - precisionAgg.push_back(-1); - widthAgg.push_back(sizeof(long double)); - scaleAgg.push_back(0); } break; @@ -1418,7 +1472,7 @@ void TupleAggregateStep::prep1PhaseAggregate( // find if this func is a duplicate AGG_MAP::iterator iter = aggFuncMap.find(boost::make_tuple(key, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); - if (iter != aggFuncMap.end()) + if (aggOp != ROWAGG_UDAF && aggOp != ROWAGG_MULTI_PARM && iter != aggFuncMap.end()) { if (funct->fAggFunction == ROWAGG_AVG) funct->fAggFunction = ROWAGG_DUP_AVG; @@ -1460,7 +1514,7 @@ void TupleAggregateStep::prep1PhaseAggregate( } // there is avg(k), but no count(k) in the select list - uint64_t lastCol = returnedColVec.size(); + uint64_t lastCol = outIdx; for (map::iterator k = avgFuncMap.begin(); k != avgFuncMap.end(); k++) { @@ -1755,11 +1809,6 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( throw logic_error(emsg.str()); } - // skip sum / count(column) if avg is also selected -// if ((aggOp == ROWAGG_SUM || aggOp == ROWAGG_COUNT_COL_NAME) && -// (avgSet.find(aggKey) != avgSet.end())) -// continue; - if (aggOp == ROWAGG_DISTINCT_SUM || aggOp == ROWAGG_DISTINCT_AVG || aggOp == ROWAGG_COUNT_DISTINCT_COL_NAME) @@ -1807,8 +1856,12 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( } // skip if this is a duplicate - if (aggFuncMap.find(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)) != aggFuncMap.end()) + if (aggOp != ROWAGG_UDAF && aggOp != ROWAGG_MULTI_PARM + && aggFuncMap.find(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)) != aggFuncMap.end()) + { + // skip if this is a duplicate continue; + } functionVec1.push_back(funct); aggFuncMap.insert(make_pair(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), colAgg)); @@ -3086,9 +3139,13 @@ void TupleAggregateStep::prep2PhasesAggregate( } // skip if this is a duplicate - if (aggFuncMap.find(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)) != aggFuncMap.end()) + if (aggOp != ROWAGG_UDAF && aggOp != ROWAGG_MULTI_PARM + && aggFuncMap.find(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)) != aggFuncMap.end()) + { + // skip if this is a duplicate continue; - + } + functionVecPm.push_back(funct); aggFuncMap.insert(make_pair(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), colAggPm)); @@ -3129,13 +3186,13 @@ void TupleAggregateStep::prep2PhasesAggregate( throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); } + wideDecimalOrLongDouble(colProj, typeProj[colProj], + precisionProj, scaleProj, width, + typeAggPm, scaleAggPm, precisionAggPm, widthAggPm); + oidsAggPm.push_back(oidsProj[colProj]); keysAggPm.push_back(aggKey); - typeAggPm.push_back(CalpontSystemCatalog::LONGDOUBLE); csNumAggPm.push_back(8); - scaleAggPm.push_back(0); - precisionAggPm.push_back(-1); - widthAggPm.push_back(sizeof(long double)); colAggPm++; } @@ -3418,13 +3475,13 @@ void TupleAggregateStep::prep2PhasesAggregate( if (aggOp == ROWAGG_SUM) { - oidsAggUm.push_back(oidsAggPm[colPm]); + wideDecimalOrLongDouble(colPm, typeProj[colPm], + precisionProj, scaleProj, widthAggPm, + typeAggUm, scaleAggUm, precisionAggUm, widthAggUm); + + oidsAggUm.push_back(oidsProj[colPm]); keysAggUm.push_back(retKey); - scaleAggUm.push_back(0); - typeAggUm.push_back(CalpontSystemCatalog::LONGDOUBLE); csNumAggUm.push_back(8); - precisionAggUm.push_back(-1); - widthAggUm.push_back(sizeof(long double)); } else { @@ -3967,8 +4024,12 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( } // skip if this is a duplicate - if (aggFuncMap.find(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)) != aggFuncMap.end()) + if (aggOp != ROWAGG_UDAF && aggOp != ROWAGG_MULTI_PARM + && aggFuncMap.find(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)) != aggFuncMap.end()) + { + // skip if this is a duplicate continue; + } functionVecPm.push_back(funct); aggFuncMap.insert(make_pair(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), colAggPm-multiParm)); diff --git a/dbcon/joblist/tupleconstantstep.cpp b/dbcon/joblist/tupleconstantstep.cpp index 15e5927a1..ccbff89c7 100644 --- a/dbcon/joblist/tupleconstantstep.cpp +++ b/dbcon/joblist/tupleconstantstep.cpp @@ -512,25 +512,18 @@ void TupleConstantStep::fillInConstants(const rowgroup::Row& rowIn, rowgroup::Ro if (fIndexConst.size() > 1 || fIndexConst[0] != 0) { copyRow(fRowConst, &rowOut); - //memcpy(rowOut.getData(), fRowConst.getData(), fRowConst.getSize()); rowOut.setRid(rowIn.getRelRid()); for (uint64_t j = 0; j < fIndexMapping.size(); ++j) rowIn.copyField(rowOut, fIndexMapping[j], j); - - //rowIn.copyField(rowOut.getData() + rowOut.getOffset(fIndexMapping[j]), j); } else // only first column is constant { - //size_t n = rowOut.getOffset(rowOut.getColumnCount()) - rowOut.getOffset(1); rowOut.setRid(rowIn.getRelRid()); fRowConst.copyField(rowOut, 0, 0); - //fRowConst.copyField(rowOut.getData()+2, 0); // hardcoded 2 for rid length for (uint32_t i = 1; i < rowOut.getColumnCount(); i++) rowIn.copyField(rowOut, i, i - 1); - - //memcpy(rowOut.getData()+rowOut.getOffset(1), rowIn.getData()+2, n); } } diff --git a/dbcon/joblist/tuplehashjoin.cpp b/dbcon/joblist/tuplehashjoin.cpp index e7ddbb235..6b616c083 100644 --- a/dbcon/joblist/tuplehashjoin.cpp +++ b/dbcon/joblist/tuplehashjoin.cpp @@ -476,7 +476,9 @@ void TupleHashJoinStep::forwardCPData() for (col = 0; col < joiners[i]->getSmallKeyColumns().size(); col++) { - if (smallRGs[i].isLongString(joiners[i]->getSmallKeyColumns()[col])) + uint32_t idx = joiners[i]->getSmallKeyColumns()[col]; + + if (smallRGs[i].isLongString(idx)) continue; // @bug3683, not to add CP predicates if large side is not simple column @@ -484,8 +486,12 @@ void TupleHashJoinStep::forwardCPData() fFunctionJoinKeys.end()) continue; + bool isSmallSideWideDecimal = + datatypes::isWideDecimalType(smallRGs[i].getColType(idx), smallRGs[i].getColumnWidth(idx)); + largeBPS->addCPPredicates(largeRG.getOIDs()[joiners[i]->getLargeKeyColumns()[col]], - joiners[i]->getCPData()[col], !joiners[i]->discreteCPValues()[col]); + joiners[i]->getCPData()[col], !joiners[i]->discreteCPValues()[col], + isSmallSideWideDecimal); } } } diff --git a/dbcon/joblist/tupleunion.cpp b/dbcon/joblist/tupleunion.cpp index 914bd850d..438dc8ef2 100644 --- a/dbcon/joblist/tupleunion.cpp +++ b/dbcon/joblist/tupleunion.cpp @@ -534,7 +534,11 @@ dec1: else val *= (uint64_t) pow((double) 10, (double) diff); - out->setIntField(val, i); + if (out->getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) + out->setInt128Field(val, i); + else + out->setIntField(val, i); + break; } @@ -661,7 +665,11 @@ dec2: else val *= (uint64_t) pow((double) 10, (double) diff); - out->setIntField(val, i); + if (out->getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) + out->setInt128Field(val, i); + else + out->setIntField(val, i); + break; } @@ -979,7 +987,7 @@ dec2: case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { -dec3: /* have to pick a scale to use for the double. using 5... */ +dec3: /* have to pick a scale to use for the double. using 5... */ uint32_t scale = 5; uint64_t ival = (uint64_t) (double) (val * pow((double) 10, (double) scale)); int diff = out->getScale(i) - scale; @@ -989,7 +997,11 @@ dec3: /* have to pick a scale to use for the double. using 5... */ else ival *= (uint64_t) pow((double) 10, (double) diff); - out->setIntField((int64_t) val, i); + if (out->getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) + out->setInt128Field(ival, i); + else + out->setIntField(ival, i); + break; } @@ -1066,7 +1078,11 @@ dec4: /* have to pick a scale to use for the double. using 5... */ else ival *= (uint64_t) pow((double) 10, (double) diff); - out->setIntField((int64_t) val, i); + if (out->getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) + out->setInt128Field(ival, i); + else + out->setIntField(ival, i); + break; } @@ -1083,8 +1099,19 @@ dec4: /* have to pick a scale to use for the double. using 5... */ case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - int64_t val = in.getIntField(i); - uint32_t scale = in.getScale(i); + int64_t val = 0; + int128_t val128 = 0; + bool isInputWide = false; + + if (in.getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) + { + in.getInt128Field(i, val128); + isInputWide = true; + } + else + val = in.getIntField(i); + + uint32_t scale = in.getScale(i); switch (out->getColTypes()[i]) { @@ -1101,12 +1128,31 @@ dec4: /* have to pick a scale to use for the double. using 5... */ case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - if (out->getScale(i) == scale) - out->setIntField(val, i); - else if (out->getScale(i) > scale) - out->setIntField(IDB_pow[out->getScale(i) - scale]*val, i); - else // should not happen, the output's scale is the largest - throw logic_error("TupleUnion::normalize(): incorrect scale setting"); + if (out->getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) + { + if (out->getScale(i) == scale) + out->setInt128Field(isInputWide ? val128 : val, i); + else if (out->getScale(i) > scale) + { + int128_t divisor = 1; + datatypes::getScaleDivisor(divisor, out->getScale(i) - scale); + int128_t temp = isInputWide ? divisor*val128 : divisor*val; + out->setInt128Field(temp, i); + } + else // should not happen, the output's scale is the largest + throw logic_error("TupleUnion::normalize(): incorrect scale setting"); + } + // If output type is narrow decimal, input type + // has to be narrow decimal as well. + else + { + if (out->getScale(i) == scale) + out->setIntField(val, i); + else if (out->getScale(i) > scale) + out->setIntField(IDB_pow[out->getScale(i) - scale]*val, i); + else // should not happen, the output's scale is the largest + throw logic_error("TupleUnion::normalize(): incorrect scale setting"); + } break; } @@ -1139,15 +1185,21 @@ dec4: /* have to pick a scale to use for the double. using 5... */ case CalpontSystemCatalog::VARCHAR: default: { - char buf[50]; - dataconvert::DataConvert::decimalToString(val, scale, buf, 50, out->getColTypes()[i]); - /* ostringstream oss; - if (scale == 0) - oss << val; - else - oss << (val / IDB_pow[scale]) << "." - << setw(scale) << setfill('0') << (val % IDB_pow[scale]); */ - out->setStringField(string(buf), i); + if (LIKELY(isInputWide)) + { + datatypes::VDecimal dec(0, + in.getScale(i), + in.getPrecision(i), + val128); + out->setStringField(dec.toString(), i); + } + else + { + datatypes::VDecimal dec(val, + in.getScale(i), + in.getPrecision(i)); + out->setStringField(dec.toString(), i); + } break; } } diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index 491a59323..7274ed525 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -7,6 +7,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} SET ( libcalmysql_SRCS + ../../datatypes/mcs_datatype.cpp ha_mcs_sysvars.cpp ha_mcs_client_udfs.cpp ha_mcs_opt_rewrites.cpp diff --git a/dbcon/mysql/ha_mcs_datatype.h b/dbcon/mysql/ha_mcs_datatype.h new file mode 100644 index 000000000..d3bae935f --- /dev/null +++ b/dbcon/mysql/ha_mcs_datatype.h @@ -0,0 +1,1207 @@ +/* + Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ +#ifndef HA_MCS_DATATYPE_H_INCLUDED +#define HA_MCS_DATATYPE_H_INCLUDED + +/* + Interface classes for MariaDB data types (e.g. Field) for TypeHandler. + These classes are needed to avoid TypeHandler dependency on + MariaDB header files. +*/ + + +namespace datatypes { + +class StoreFieldMariaDB: public StoreField +{ + Field *m_field; + const CalpontSystemCatalog::ColType &m_type; +public: + StoreFieldMariaDB(Field *f, CalpontSystemCatalog::ColType &type) + :m_field(f), m_type(type) + { } + const CalpontSystemCatalog::ColType &type() const { return m_type; } + int32_t colWidth() const override { return m_type.colWidth; } + int32_t precision() const override { return m_type.precision; } + int32_t scale() const override { return m_type.scale; } + + int store_date(int64_t val) override + { + char tmp[256]; + DataConvert::dateToString(val, tmp, sizeof(tmp)-1); + return store_string(tmp, strlen(tmp)); + } + + int store_datetime(int64_t val) override + { + char tmp[256]; + DataConvert::datetimeToString(val, tmp, sizeof(tmp)-1, m_type.precision); + return store_string(tmp, strlen(tmp)); + } + + int store_time(int64_t val) override + { + char tmp[256]; + DataConvert::timeToString(val, tmp, sizeof(tmp)-1, m_type.precision); + return store_string(tmp, strlen(tmp)); + } + + int store_timestamp(int64_t val) override + { + char tmp[256]; + DataConvert::timestampToString(val, tmp, sizeof(tmp), + current_thd->variables.time_zone->get_name()->ptr(), + m_type.precision); + return store_string(tmp, strlen(tmp)); + } + + int store_string(const char *str, size_t length) override + { + return m_field->store(str, length, m_field->charset()); + } + int store_varbinary(const char *str, size_t length) override + { + if (get_varbin_always_hex(current_thd)) + { + size_t ll = length * 2; + boost::scoped_array sca(new char[ll]); + ConstString(str, length).bin2hex(sca.get()); + return m_field->store_binary(sca.get(), ll); + } + return m_field->store_binary(str, length); + } + + int store_xlonglong(int64_t val) override + { + idbassert(dynamic_cast(m_field)); + return m_field->store(val, static_cast(m_field)->unsigned_flag); + } + + int store_float(float dl) override + { + if (dl == std::numeric_limits::infinity()) + { + m_field->set_null(); + return 1; + } + + // bug 3485, reserve enough space for the longest float value + // -3.402823466E+38 to -1.175494351E-38, 0, and + // 1.175494351E-38 to 3.402823466E+38. + m_field->field_length = 40; + return m_field->store(dl); + } + + int store_double(double dl) override + { + if (dl == std::numeric_limits::infinity()) + { + m_field->set_null(); + return 1; + } + + if (m_field->type() == MYSQL_TYPE_NEWDECIMAL) + { + char buf[310]; + // reserve enough space for the longest double value + // -1.7976931348623157E+308 to -2.2250738585072014E-308, 0, and + // 2.2250738585072014E-308 to 1.7976931348623157E+308. + snprintf(buf, 310, "%.18g", dl); + return m_field->store(buf, strlen(buf), m_field->charset()); + } + + // The server converts dl=-0 to dl=0 in (*f)->store(). + // This happens in the call to truncate_double(). + // This is an unexpected behaviour, so we directly store the + // double value using the lower level float8store() function. + // TODO Remove this when (*f)->store() handles this properly. + m_field->field_length = 310; + if (dl == 0) + { + float8store(m_field->ptr,dl); + return 0; + } + return m_field->store(dl); + } + + int store_long_double(long double dl) override + { + if (dl == std::numeric_limits::infinity()) + { + m_field->set_null(); + return 1; + } + + if (m_field->type() == MYSQL_TYPE_NEWDECIMAL) + { + char buf[310]; + snprintf(buf, 310, "%.20Lg", dl); + return m_field->store(buf, strlen(buf), m_field->charset()); + } + + // reserve enough space for the longest double value + // -1.7976931348623157E+308 to -2.2250738585072014E-308, 0, and + // 2.2250738585072014E-308 to 1.7976931348623157E+308. + m_field->field_length = 310; + return m_field->store(static_cast(dl)); + } + + int store_decimal64(const datatypes::VDecimal& dec) override + { + std::string decAsAStr = dec.toString(); + return m_field->store(decAsAStr.c_str(), decAsAStr.length(), m_field->charset()); + } + + int store_decimal128(const datatypes::VDecimal& dec) override + { + std::string decAsAStr = dec.toString(true); + return m_field->store(decAsAStr.c_str(), decAsAStr.length(), m_field->charset()); + } + + int store_lob(const char *str, size_t length) override + { + idbassert(dynamic_cast(m_field)); + Field_blob* f2 = static_cast(m_field); + f2->set_ptr(length, (uchar*) str); + return 0; + } + +}; + + +/*******************************************************************************/ + +class WriteBatchFieldMariaDB: public WriteBatchField +{ +public: + Field *m_field; + const CalpontSystemCatalog::ColType &m_type; + uint32_t m_mbmaxlen; + WriteBatchFieldMariaDB(Field *field, + const CalpontSystemCatalog::ColType type, + uint32_t mbmaxlen) + :m_field(field), m_type(type), m_mbmaxlen(mbmaxlen) + { } + size_t ColWriteBatchDate(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + // QQ: OLD DATE is not handled + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + fprintf(ci.filePtr(), "%c", ci.delimiter()); + } + else + { + const uchar* tmp1 = buf; + uint32_t tmp = (tmp1[2] << 16) + (tmp1[1] << 8) + tmp1[0]; + int day = tmp & 0x0000001fl; + int month = (tmp >> 5) & 0x0000000fl; + int year = tmp >> 9; + fprintf(ci.filePtr(), "%04d-%02d-%02d%c", year, month, day, ci.delimiter()); + } + return 3; + } + size_t ColWriteBatchDatetime(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + fprintf(ci.filePtr(), "%c", ci.delimiter()); + + if (m_field->real_type() == MYSQL_TYPE_DATETIME2) + return m_field->pack_length(); + else + return 8; + } + + if (m_field->real_type() == MYSQL_TYPE_DATETIME2) + { + // mariadb 10.1 compatibility -- MYSQL_TYPE_DATETIME2 introduced in mysql 5.6 + MYSQL_TIME ltime; + longlong tmp = my_datetime_packed_from_binary(buf, m_field->decimals()); + TIME_from_longlong_datetime_packed(<ime, tmp); + + if (!ltime.second_part) + { + fprintf(ci.filePtr(), "%04d-%02d-%02d %02d:%02d:%02d%c", + ltime.year, ltime.month, ltime.day, + ltime.hour, ltime.minute, ltime.second, ci.delimiter()); + } + else + { + fprintf(ci.filePtr(), "%04d-%02d-%02d %02d:%02d:%02d.%ld%c", + ltime.year, ltime.month, ltime.day, + ltime.hour, ltime.minute, ltime.second, + ltime.second_part, ci.delimiter()); + } + + return m_field->pack_length(); + } + + // Old DATETIME + long long value = *((long long*) buf); + long datePart = (long) (value / 1000000ll); + int day = datePart % 100; + int month = (datePart / 100) % 100; + int year = datePart / 10000; + fprintf(ci.filePtr(), "%04d-%02d-%02d ", year, month, day); + + long timePart = (long) (value - (long long) datePart * 1000000ll); + int second = timePart % 100; + int min = (timePart / 100) % 100; + int hour = timePart / 10000; + fprintf(ci.filePtr(), "%02d:%02d:%02d%c", hour, min, second, ci.delimiter()); + return 8; + } + size_t ColWriteBatchTime(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + // QQ: why old TIME is not handled? + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + fprintf(ci.filePtr(), "%c", ci.delimiter()); + return m_field->pack_length(); + } + + MYSQL_TIME ltime; + longlong tmp = my_time_packed_from_binary(buf, m_field->decimals()); + TIME_from_longlong_time_packed(<ime, tmp); + + if (ltime.neg) + fprintf(ci.filePtr(), "-"); + + if (!ltime.second_part) + { + fprintf(ci.filePtr(), "%02d:%02d:%02d%c", + ltime.hour, ltime.minute, ltime.second, ci.delimiter()); + } + else + { + fprintf(ci.filePtr(), "%02d:%02d:%02d.%ld%c", + ltime.hour, ltime.minute, ltime.second, + ltime.second_part, ci.delimiter()); + } + + return m_field->pack_length(); + } + + size_t ColWriteBatchTimestamp(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + // QQ: old TIMESTAMP is not handled + + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + fprintf(ci.filePtr(), "%c", ci.delimiter()); + return m_field->pack_length(); + } + + struct timeval tm; + my_timestamp_from_binary(&tm, buf, m_field->decimals()); + + MySQLTime time; + gmtSecToMySQLTime(tm.tv_sec, time, current_thd->variables.time_zone->get_name()->ptr()); + + if (!tm.tv_usec) + { + fprintf(ci.filePtr(), "%04d-%02d-%02d %02d:%02d:%02d%c", + time.year, time.month, time.day, + time.hour, time.minute, time.second, ci.delimiter()); + } + else + { + fprintf(ci.filePtr(), "%04d-%02d-%02d %02d:%02d:%02d.%ld%c", + time.year, time.month, time.day, + time.hour, time.minute, time.second, + tm.tv_usec, ci.delimiter()); + } + return m_field->pack_length(); + } + + size_t ColWriteBatchChar(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + uint32_t colWidthInBytes = m_type.colWidth * m_mbmaxlen; + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + fprintf(ci.filePtr(), "%c", ci.delimiter()); + } + else + { + if (current_thd->variables.sql_mode & MODE_PAD_CHAR_TO_FULL_LENGTH) + { + std::string escape; + // Pad to the full length of the field + escape.assign((char*)buf, colWidthInBytes); + + boost::replace_all(escape, "\\", "\\\\"); + + fprintf(ci.filePtr(), "%c%.*s%c%c", + ci.enclosed_by(), + (int)escape.length(), escape.c_str(), + ci.enclosed_by(), ci.delimiter()); + } + else + { + std::string escape; + // Get the actual data length + bitmap_set_bit(m_field->table->read_set, m_field->field_index); + String attribute; + m_field->val_str(&attribute); + + escape.assign((char*)buf, attribute.length()); + boost::replace_all(escape, "\\", "\\\\"); + + fprintf(ci.filePtr(), "%c%.*s%c%c", + ci.enclosed_by(), + (int)escape.length(), escape.c_str(), + ci.enclosed_by(), ci.delimiter()); + } + } + + return colWidthInBytes; + } + + size_t ColWriteBatchVarchar(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + const uchar *buf0= buf; + uint32_t colWidthInBytes = m_type.colWidth * m_mbmaxlen; + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + fprintf(ci.filePtr(), "%c", ci.delimiter()); + if (colWidthInBytes < 256) + { + buf++; + } + else + { + buf = buf + 2 ; + } + } + else + { + int dataLength = 0; + + if (colWidthInBytes < 256) + { + dataLength = *(uint8_t*) buf; + buf++; + } + else + { + dataLength = *(uint16_t*) buf; + buf = buf + 2 ; + } + std::string escape; + escape.assign((char*)buf, dataLength); + boost::replace_all(escape, "\\", "\\\\"); + fprintf(ci.filePtr(), "%c%.*s%c%c", + ci.enclosed_by(), + (int)escape.length(), escape.c_str(), + ci.enclosed_by(), ci.delimiter()); + } + buf += colWidthInBytes; + return buf - buf0; + } + + size_t ColWriteBatchSInt64(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + fprintf(ci.filePtr(), "%lld%c", *((long long*)buf), ci.delimiter()); + return 8; + } + + size_t ColWriteBatchUInt64(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + fprintf(ci.filePtr(), "%llu%c", *((long long unsigned*)buf), ci.delimiter()); + return 8; + } + + + size_t ColWriteBatchSInt32(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + fprintf(ci.filePtr(), "%d%c", *((int32_t*)buf), ci.delimiter()); + return 4; + } + + size_t ColWriteBatchUInt32(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + fprintf(ci.filePtr(), "%u%c", *((uint32_t*)buf), ci.delimiter()); + return 4; + } + + size_t ColWriteBatchSInt16(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + fprintf(ci.filePtr(), "%d%c", *((int16_t*)buf), ci.delimiter()); + return 2; + } + + size_t ColWriteBatchUInt16(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + fprintf(ci.filePtr(), "%u%c", *((uint16_t*)buf), ci.delimiter()); + return 2; + } + + size_t ColWriteBatchSInt8(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + fprintf(ci.filePtr(), "%d%c", *((int8_t*)buf), ci.delimiter()); + return 1; + } + + size_t ColWriteBatchUInt8(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + fprintf(ci.filePtr(), "%u%c", *((uint8_t*)buf), ci.delimiter()); + return 1; + } + + + size_t ColWriteBatchXFloat(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + { + float val = *((float*)buf); + + if ((fabs(val) > (1.0 / IDB_pow[4])) && (fabs(val) < (float) IDB_pow[6])) + { + fprintf(ci.filePtr(), "%.7f%c", val, ci.delimiter()); + } + else + { + fprintf(ci.filePtr(), "%e%c", val, ci.delimiter()); + } + + //fprintf(ci.filePtr(), "%.7g|", *((float*)buf2)); + //printf("%.7f|", *((float*)buf2)); + } + + return 4; + } + + + size_t ColWriteBatchXDouble(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + { + fprintf(ci.filePtr(), "%.15g%c", *((double*)buf), ci.delimiter()); + //printf("%.15g|", *((double*)buf)); + } + + return 8; + } + + + size_t ColWriteBatchSLongDouble(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + fprintf(ci.filePtr(), "%.15Lg%c", *((long double*)buf), ci.delimiter()); + return sizeof(long double); + } + + + size_t ColWriteBatchXDecimal(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + uint bytesBefore = 1; + uint totalBytes = 9; + + switch (m_type.precision) + { + case 18: + case 17: + case 16: + { + totalBytes = 8; + break; + } + + case 15: + case 14: + { + totalBytes = 7; + break; + } + + case 13: + case 12: + { + totalBytes = 6; + break; + } + + case 11: + { + totalBytes = 5; + break; + } + + case 10: + { + totalBytes = 5; + break; + } + + case 9: + case 8: + case 7: + { + totalBytes = 4; + break; + } + + case 6: + case 5: + { + totalBytes = 3; + break; + } + + case 4: + case 3: + { + totalBytes = 2; + break; + } + + case 2: + case 1: + { + totalBytes = 1; + break; + } + + default: + break; + } + + switch (m_type.scale) + { + case 0: + { + bytesBefore = totalBytes; + break; + } + + case 1: //1 byte for digits after decimal point + { + if ((m_type.precision != 16) && (m_type.precision != 14) + && (m_type.precision != 12) && (m_type.precision != 10) + && (m_type.precision != 7) && (m_type.precision != 5) + && (m_type.precision != 3) && (m_type.precision != 1)) + totalBytes++; + + bytesBefore = totalBytes - 1; + break; + } + + case 2: //1 byte for digits after decimal point + { + if ((m_type.precision == 18) || (m_type.precision == 9)) + totalBytes++; + + bytesBefore = totalBytes - 1; + break; + } + + case 3: //2 bytes for digits after decimal point + { + if ((m_type.precision != 16) && (m_type.precision != 14) + && (m_type.precision != 12) && (m_type.precision != 7) + && (m_type.precision != 5) && (m_type.precision != 3)) + totalBytes++; + + bytesBefore = totalBytes - 2; + break; + } + + case 4: + { + if ((m_type.precision == 18) || (m_type.precision == 11) + || (m_type.precision == 9)) + totalBytes++; + + bytesBefore = totalBytes - 2; + break; + + } + + case 5: + { + if ((m_type.precision != 16) && (m_type.precision != 14) + && (m_type.precision != 7) && (m_type.precision != 5)) + totalBytes++; + + bytesBefore = totalBytes - 3; + break; + } + + case 6: + { + if ((m_type.precision == 18) || (m_type.precision == 13) + || (m_type.precision == 11) || (m_type.precision == 9)) + totalBytes++; + + bytesBefore = totalBytes - 3; + break; + } + + case 7: + { + if ((m_type.precision != 16) && (m_type.precision != 7)) + totalBytes++; + + bytesBefore = totalBytes - 4; + break; + } + + case 8: + { + if ((m_type.precision == 18) || (m_type.precision == 15) + || (m_type.precision == 13) || (m_type.precision == 11) + || (m_type.precision == 9)) + totalBytes++; + + bytesBefore = totalBytes - 4;; + break; + } + + case 9: + { + bytesBefore = totalBytes - 4;; + break; + } + + case 10: + { + if ((m_type.precision != 16) && (m_type.precision != 14) + && (m_type.precision != 12) && (m_type.precision != 10)) + totalBytes++; + + bytesBefore = totalBytes - 5;; + break; + } + + case 11: + { + if (m_type.precision == 18) + totalBytes++; + + bytesBefore = totalBytes - 5; + break; + } + + case 12: + { + if ((m_type.precision != 16) && (m_type.precision != 14) + && (m_type.precision != 12)) + totalBytes++; + + bytesBefore = totalBytes - 6; + break; + } + + case 13: + { + if (m_type.precision == 18) + totalBytes++; + + bytesBefore = totalBytes - 6; + break; + } + + case 14: + { + if ((m_type.precision != 16) && (m_type.precision != 14)) + totalBytes++; + + bytesBefore = totalBytes - 7; + break; + } + + case 15: + { + if (m_type.precision == 18) + totalBytes++; + + bytesBefore = totalBytes - 7; + break; + } + + case 16: + { + if (m_type.precision != 16) + totalBytes++; + + bytesBefore = totalBytes - 8; + break; + } + + case 17: + { + if (m_type.precision == 18) + totalBytes++; + + bytesBefore = totalBytes - 8; + break; + } + + case 18: + { + bytesBefore = totalBytes - 8; + break; + } + + default: + break; + } + + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + fprintf(ci.filePtr(), "%c", ci.delimiter()); + //printf("|"); + } + else + { + uint32_t mask [5] = {0, 0xFF, 0xFFFF, 0xFFFFFF, 0xFFFFFFFF}; + char neg = '-'; + + if (m_type.scale == 0) + { + const uchar* tmpBuf = buf; + //test flag bit for sign + bool posNum = tmpBuf[0] & (0x80); + uchar tmpChr = tmpBuf[0]; + tmpChr ^= 0x80; //flip the bit + int32_t tmp1 = tmpChr; + + if (totalBytes > 4) + { + for (uint i = 1; i < (totalBytes - 4); i++) + { + tmp1 = (tmp1 << 8) + tmpBuf[i]; + } + + if (( tmp1 != 0 ) && (tmp1 != -1)) + { + if (!posNum) + { + tmp1 = mask[totalBytes - 4] - tmp1; + + if (tmp1 != 0) + { + fprintf(ci.filePtr(), "%c", neg); + //printf("%c", neg); + } + } + + if (tmp1 != 0) + { + fprintf(ci.filePtr(), "%d", tmp1); + ////printf("%d", tmp1); + } + } + + int32_t tmp2 = tmpBuf[totalBytes - 4]; + + for (uint i = (totalBytes - 3); i < totalBytes; i++) + { + tmp2 = (tmp2 << 8) + tmpBuf[i]; + } + + if ( tmp1 != 0 ) + { + if (!posNum) + { + tmp2 = mask[4] - tmp2; + + if (tmp1 == -1) + { + fprintf(ci.filePtr(), "%c", neg); + fprintf(ci.filePtr(), "%d%c", tmp2, ci.delimiter()); + ////printf("%c", neg); + //////printf( "%d|", tmp2); + } + else + { + fprintf(ci.filePtr(), "%09u%c", tmp2, ci.delimiter()); + ////printf("%09u|", tmp2); + } + } + else + { + fprintf(ci.filePtr(), "%09u%c", tmp2, ci.delimiter()); + //printf("%09u|", tmp2); + } + } + else + { + if (!posNum) + { + tmp2 = mask[4] - tmp2; + fprintf(ci.filePtr(), "%c", neg); + //printf("%c", neg); + } + + fprintf(ci.filePtr(), "%d%c", tmp2, ci.delimiter()); + //printf("%d|", tmp2); + } + } + else + { + for (uint i = 1; i < totalBytes; i++) + { + tmp1 = (tmp1 << 8) + tmpBuf[i]; + } + + if (!posNum) + { + tmp1 = mask[totalBytes] - tmp1; + fprintf(ci.filePtr(), "%c", neg); + //printf("%c", neg); + } + + fprintf(ci.filePtr(), "%d%c", tmp1, ci.delimiter()); + //printf("%d|", tmp1); + } + } + else + { + const uchar* tmpBuf = buf; + //test flag bit for sign + bool posNum = tmpBuf[0] & (0x80); + uchar tmpChr = tmpBuf[0]; + tmpChr ^= 0x80; //flip the bit + int32_t tmp1 = tmpChr; + + //fetch the digits before decimal point + if (bytesBefore == 0) + { + if (!posNum) + { + fprintf(ci.filePtr(), "%c", neg); + //printf("%c", neg); + } + + fprintf(ci.filePtr(), "0."); + //printf("0."); + } + else if (bytesBefore > 4) + { + for (uint i = 1; i < (bytesBefore - 4); i++) + { + tmp1 = (tmp1 << 8) + tmpBuf[i]; + } + + if (!posNum) + { + tmp1 = mask[bytesBefore - 4] - tmp1; + } + + if (( tmp1 != 0 ) && (tmp1 != -1)) + { + if (!posNum) + { + fprintf(ci.filePtr(), "%c", neg); + //printf("%c", neg); + } + + fprintf(ci.filePtr(), "%d", tmp1); + //printf("%d", tmp1); + } + + tmpBuf += (bytesBefore - 4); + int32_t tmp2 = *((int32_t*)tmpBuf); + tmp2 = ntohl(tmp2); + + if ( tmp1 != 0 ) + { + if (!posNum) + { + tmp2 = mask[4] - tmp2; + } + + if (tmp1 == -1) + { + fprintf(ci.filePtr(), "%c", neg); + fprintf(ci.filePtr(), "%d.", tmp2); + //printf("%c", neg); + //printf("%d.", tmp2); + } + else + { + fprintf(ci.filePtr(), "%09u.", tmp2); + //printf("%09u.", tmp2); + } + } + else + { + if (!posNum) + { + tmp2 = mask[4] - tmp2; + fprintf(ci.filePtr(), "%c", neg); + //printf("%c", neg); + } + + fprintf(ci.filePtr(), "%d.", tmp2); + //printf("%d.", tmp2); + } + } + else + { + for (uint i = 1; i < bytesBefore; i++) + { + tmp1 = (tmp1 << 8) + tmpBuf[i]; + } + + if (!posNum) + { + tmp1 = mask[bytesBefore] - tmp1; + fprintf(ci.filePtr(), "%c", neg); + //printf("%c", neg); + } + + fprintf(ci.filePtr(), "%d.", tmp1); + //printf("%d.", tmp1); + } + + //fetch the digits after decimal point + int32_t tmp2 = 0; + + if (bytesBefore > 4) + tmpBuf += 4; + else + tmpBuf += bytesBefore; + + tmp2 = tmpBuf[0]; + + if ((totalBytes - bytesBefore) < 5) + { + for (uint j = 1; j < (totalBytes - bytesBefore); j++) + { + tmp2 = (tmp2 << 8) + tmpBuf[j]; + } + + int8_t digits = m_type.scale - 9; //9 digits is a 4 bytes chunk + + if ( digits <= 0 ) + digits = m_type.scale; + + if (!posNum) + { + tmp2 = mask[totalBytes - bytesBefore] - tmp2; + } + + fprintf(ci.filePtr(), "%0*u%c", digits, tmp2, ci.delimiter()); + //printf("%0*u|", digits, tmp2); + } + else + { + for (uint j = 1; j < 4; j++) + { + tmp2 = (tmp2 << 8) + tmpBuf[j]; + } + + if (!posNum) + { + tmp2 = mask[4] - tmp2; + } + + fprintf(ci.filePtr(), "%09u", tmp2); + //printf("%09u", tmp2); + + tmpBuf += 4; + int32_t tmp3 = tmpBuf[0]; + + for (uint j = 1; j < (totalBytes - bytesBefore - 4); j++) + { + tmp3 = (tmp3 << 8) + tmpBuf[j]; + } + + int8_t digits = m_type.scale - 9; //9 digits is a 4 bytes chunk + + if ( digits < 0 ) + digits = m_type.scale; + + if (!posNum) + { + tmp3 = mask[totalBytes - bytesBefore - 4] - tmp3; + } + + fprintf(ci.filePtr(), "%0*u%c", digits, tmp3, ci.delimiter()); + //printf("%0*u|", digits, tmp3); + } + } + } + + return totalBytes; + } + + + size_t ColWriteBatchVarbinary(const uchar *buf0, bool nullVal, ColBatchWriter &ci) override + { + const uchar *buf= buf0; + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + fprintf(ci.filePtr(), "%c", ci.delimiter()); + + if (m_type.colWidth < 256) + { + buf++; + } + else + { + buf = buf + 2; + } + } + else + { + uint16_t dataLength = 0; + + if (m_type.colWidth < 256) + { + dataLength = *(int8_t*) buf; + buf++; + } + else + { + dataLength = *(int16_t*) buf; + buf = buf + 2 ; + } + + const uchar* tmpBuf = buf; + + for (int32_t i = 0; i < dataLength; i++) + { + fprintf(ci.filePtr(), "%02x", *(uint8_t*)tmpBuf); + tmpBuf++; + } + + fprintf(ci.filePtr(), "%c", ci.delimiter()); + + } + + return buf - buf0; + } + + + size_t ColWriteBatchBlob(const uchar *buf0, bool nullVal, ColBatchWriter &ci) override + { + const uchar *buf= buf0; + // MCOL-4005 Note that we don't handle nulls as a special + // case here as we do for other datatypes, the below works + // as expected for nulls. + uint32_t dataLength = 0; + uintptr_t* dataptr; + uchar* ucharptr; + bool isBlob = m_type.colDataType == CalpontSystemCatalog::BLOB; + uint colWidthInBytes = isBlob ? m_type.colWidth : m_type.colWidth * m_mbmaxlen; + + if (!isBlob && m_field->char_length() == 65535) + { + // Special case for TEXT field without default length, + // such as: + // CREATE TABLE mcol4364 (a TEXT); + // Here, char_length() represents the number of bytes, + // not number of characters. + dataLength = *(uint16_t*) buf; + buf += 2; + } + else if (colWidthInBytes < 256) + { + dataLength = *(uint8_t*) buf; + buf++; + } + else if (colWidthInBytes < 65536) + { + dataLength = *(uint16_t*) buf; + buf += 2; + } + else if (colWidthInBytes < 16777216) + { + dataLength = *(uint16_t*) buf; + dataLength |= ((int) buf[2]) << 16; + buf += 3; + } + else + { + dataLength = *(uint32_t*) buf; + buf += 4; + } + + // buf contains pointer to blob, for example: + // (gdb) p (char*)*(uintptr_t*)buf + // $43 = 0x7f68500c58f8 "hello world" + + dataptr = (uintptr_t*)buf; + ucharptr = (uchar*)*dataptr; + buf += sizeof(uintptr_t); + + if (isBlob) + { + for (uint32_t i = 0; i < dataLength; i++) + { + fprintf(ci.filePtr(), "%02x", *(uint8_t*)ucharptr); + ucharptr++; + } + + fprintf(ci.filePtr(), "%c", ci.delimiter()); + } + else + { + // TEXT Column + std::string escape; + escape.assign((char*)ucharptr, dataLength); + boost::replace_all(escape, "\\", "\\\\"); + fprintf(ci.filePtr(), "%c%.*s%c%c", + ci.enclosed_by(), + (int)escape.length(), escape.c_str(), + ci.enclosed_by(), ci.delimiter()); + } + + return buf - buf0; + } + +}; + +} // end of namespace datatypes + +#endif + +// vim:ts=2 sw=2: diff --git a/dbcon/mysql/ha_mcs_ddl.cpp b/dbcon/mysql/ha_mcs_ddl.cpp index 82ea00a17..e1eca830f 100644 --- a/dbcon/mysql/ha_mcs_ddl.cpp +++ b/dbcon/mysql/ha_mcs_ddl.cpp @@ -61,9 +61,6 @@ using namespace ddlpackage; #include "ddlpackageprocessor.h" using namespace ddlpackageprocessor; -#include "dataconvert.h" -using namespace dataconvert; - #include "bytestream.h" using namespace messageqcpp; @@ -133,131 +130,16 @@ static void decode_file_path(const char *path, char *decoded_dbname, decode_objectname(decoded_tbname, tbname_start, FN_REFLEN); } -uint32_t convertDataType(int dataType) + +CalpontSystemCatalog::ColDataType convertDataType(const ddlpackage::ColumnType &ct) { - uint32_t calpontDataType; - - switch (dataType) - { - case ddlpackage::DDL_CHAR: - calpontDataType = CalpontSystemCatalog::CHAR; - break; - - case ddlpackage::DDL_VARCHAR: - calpontDataType = CalpontSystemCatalog::VARCHAR; - break; - - case ddlpackage::DDL_VARBINARY: - calpontDataType = CalpontSystemCatalog::VARBINARY; - break; - - case ddlpackage::DDL_BIT: - calpontDataType = CalpontSystemCatalog::BIT; - break; - - case ddlpackage::DDL_REAL: - case ddlpackage::DDL_DECIMAL: - case ddlpackage::DDL_NUMERIC: - case ddlpackage::DDL_NUMBER: - calpontDataType = CalpontSystemCatalog::DECIMAL; - break; - - case ddlpackage::DDL_FLOAT: - calpontDataType = CalpontSystemCatalog::FLOAT; - break; - - case ddlpackage::DDL_DOUBLE: - calpontDataType = CalpontSystemCatalog::DOUBLE; - break; - - case ddlpackage::DDL_INT: - case ddlpackage::DDL_INTEGER: - calpontDataType = CalpontSystemCatalog::INT; - break; - - case ddlpackage::DDL_BIGINT: - calpontDataType = CalpontSystemCatalog::BIGINT; - break; - - case ddlpackage::DDL_MEDINT: - calpontDataType = CalpontSystemCatalog::MEDINT; - break; - - case ddlpackage::DDL_SMALLINT: - calpontDataType = CalpontSystemCatalog::SMALLINT; - break; - - case ddlpackage::DDL_TINYINT: - calpontDataType = CalpontSystemCatalog::TINYINT; - break; - - case ddlpackage::DDL_DATE: - calpontDataType = CalpontSystemCatalog::DATE; - break; - - case ddlpackage::DDL_DATETIME: - calpontDataType = CalpontSystemCatalog::DATETIME; - break; - - case ddlpackage::DDL_TIME: - calpontDataType = CalpontSystemCatalog::TIME; - break; - - case ddlpackage::DDL_TIMESTAMP: - calpontDataType = CalpontSystemCatalog::TIMESTAMP; - break; - - case ddlpackage::DDL_CLOB: - calpontDataType = CalpontSystemCatalog::CLOB; - break; - - case ddlpackage::DDL_BLOB: - calpontDataType = CalpontSystemCatalog::BLOB; - break; - - case ddlpackage::DDL_TEXT: - calpontDataType = CalpontSystemCatalog::TEXT; - break; - - case ddlpackage::DDL_UNSIGNED_TINYINT: - calpontDataType = CalpontSystemCatalog::UTINYINT; - break; - - case ddlpackage::DDL_UNSIGNED_SMALLINT: - calpontDataType = CalpontSystemCatalog::USMALLINT; - break; - - case ddlpackage::DDL_UNSIGNED_MEDINT: - calpontDataType = CalpontSystemCatalog::UMEDINT; - break; - - case ddlpackage::DDL_UNSIGNED_INT: - calpontDataType = CalpontSystemCatalog::UINT; - break; - - case ddlpackage::DDL_UNSIGNED_BIGINT: - calpontDataType = CalpontSystemCatalog::UBIGINT; - break; - - case ddlpackage::DDL_UNSIGNED_DECIMAL: - case ddlpackage::DDL_UNSIGNED_NUMERIC: - calpontDataType = CalpontSystemCatalog::UDECIMAL; - break; - - case ddlpackage::DDL_UNSIGNED_FLOAT: - calpontDataType = CalpontSystemCatalog::UFLOAT; - break; - - case ddlpackage::DDL_UNSIGNED_DOUBLE: - calpontDataType = CalpontSystemCatalog::UDOUBLE; - break; - - default: - throw runtime_error("Unsupported datatype!"); - - } - - return calpontDataType; + const datatypes::TypeHandler *h= datatypes::TypeHandler::find_by_ddltype(ct); + if (!h) + { + throw runtime_error("Unsupported datatype!"); + return CalpontSystemCatalog::UNDEFINED; + } + return h->code(); } @@ -1003,10 +885,10 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl if (!createTable->fTableDef->fColumns[i]->fDefaultValue->fNull) { //validate the default value, if out of range, just error out - uint32_t dataType; + CalpontSystemCatalog::ColDataType dataType; dataType = convertDataType(createTable->fTableDef->fColumns[i]->fType->fType); CalpontSystemCatalog::ColType colType; - colType.colDataType = (CalpontSystemCatalog::ColDataType) dataType; + colType.colDataType = dataType; colType.colWidth = createTable->fTableDef->fColumns[i]->fType->fLength; colType.precision = createTable->fTableDef->fColumns[i]->fType->fPrecision; colType.scale = createTable->fTableDef->fColumns[i]->fType->fScale; @@ -1015,7 +897,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl try { - convertedVal = DataConvert::convertColumnData(colType, createTable->fTableDef->fColumns[i]->fDefaultValue->fValue, pushWarning, thd->variables.time_zone->get_name()->ptr(), false, false, false); + convertedVal = colType.convertColumnData(createTable->fTableDef->fColumns[i]->fDefaultValue->fValue, pushWarning, thd->variables.time_zone->get_name()->ptr(), false, false, false); } catch (std::exception&) { @@ -1379,10 +1261,10 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl } //validate the default value, if out of range, just error out - uint32_t dataType; + CalpontSystemCatalog::ColDataType dataType; dataType = convertDataType(addColumnPtr->fColumnDef->fType->fType); CalpontSystemCatalog::ColType colType; - colType.colDataType = (CalpontSystemCatalog::ColDataType) dataType; + colType.colDataType = dataType; colType.colWidth = addColumnPtr->fColumnDef->fType->fLength; colType.precision = addColumnPtr->fColumnDef->fType->fPrecision; colType.scale = addColumnPtr->fColumnDef->fType->fScale; @@ -1391,7 +1273,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl try { - convertedVal = DataConvert::convertColumnData(colType, addColumnPtr->fColumnDef->fDefaultValue->fValue, pushWarning, thd->variables.time_zone->get_name()->ptr(), false, false, false); + convertedVal = colType.convertColumnData(addColumnPtr->fColumnDef->fDefaultValue->fValue, pushWarning, thd->variables.time_zone->get_name()->ptr(), false, false, false); } catch (std::exception&) { @@ -1733,10 +1615,10 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl } //validate the default value, if out of range, just error out - uint32_t dataType; + CalpontSystemCatalog::ColDataType dataType; dataType = convertDataType(addColumnsPtr->fColumns[0]->fType->fType); CalpontSystemCatalog::ColType colType; - colType.colDataType = (CalpontSystemCatalog::ColDataType) dataType; + colType.colDataType = dataType; colType.colWidth = addColumnsPtr->fColumns[0]->fType->fLength; colType.precision = addColumnsPtr->fColumns[0]->fType->fPrecision; colType.scale = addColumnsPtr->fColumns[0]->fType->fScale; @@ -1745,7 +1627,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl try { - convertedVal = DataConvert::convertColumnData(colType, addColumnsPtr->fColumns[0]->fDefaultValue->fValue, pushWarning, thd->variables.time_zone->get_name()->ptr(), false, false, false); + convertedVal = colType.convertColumnData(addColumnsPtr->fColumns[0]->fDefaultValue->fValue, pushWarning, thd->variables.time_zone->get_name()->ptr(), false, false, false); } catch (std::exception&) { @@ -2295,6 +2177,22 @@ static bool get_field_default_value(THD *thd, Field *field, String *def_value, return has_default; } +/* + Utility function search for ZEROFILL +*/ + +bool hasZerofillDecimal(TABLE *table_arg) +{ + for (Field **field= table_arg->field; *field; field++) + { + if (((*field)->flags & ZEROFILL_FLAG) + && typeid (**field) == typeid(Field_new_decimal)) + return true; + } + + return false; +} + int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* create_info, cal_connection_info& ci) { #ifdef MCS_DEBUG @@ -2574,6 +2472,14 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea cout << "ha_mcs_impl_create_: ProcessDDL error, now in state NOT_ALTER" << endl; #endif } + else + { + if (hasZerofillDecimal(table_arg)) + { + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, + WARN_OPTION_IGNORED, "ZEROFILL is ignored in ColumnStore"); + } + } return rc; } diff --git a/dbcon/mysql/ha_mcs_dml.cpp b/dbcon/mysql/ha_mcs_dml.cpp index 06d1129fe..68ccff713 100644 --- a/dbcon/mysql/ha_mcs_dml.cpp +++ b/dbcon/mysql/ha_mcs_dml.cpp @@ -75,6 +75,8 @@ using namespace joblist; #include "dbrm.h" +#include "ha_mcs_datatype.h" + namespace { #define BATCH_INSERT_GROUP_ROWS_FOR_CACHE 100000 @@ -669,6 +671,7 @@ int ha_mcs_impl_write_row_(const uchar* buf, TABLE* table, cal_connection_info& } } + int ha_mcs_impl_write_batch_row_(const uchar* buf, TABLE* table, cal_impl_if::cal_connection_info& ci) { ByteStream rowData; @@ -681,7 +684,6 @@ int ha_mcs_impl_write_batch_row_(const uchar* buf, TABLE* table, cal_impl_if::ca uint16_t colpos = 0; buf = buf + ci.headerLength; // Number of bytes used for null bits. //@Bug 6122 if all columns have not null constraint, there is no information in the header - std::string escape; char nullBits = *bufHdr++; if (!ci.useXbit) @@ -730,1078 +732,22 @@ int ha_mcs_impl_write_batch_row_(const uchar* buf, TABLE* table, cal_impl_if::ca nullVal = false; } - switch (ci.columnTypes[colpos].colDataType) + const CalpontSystemCatalog::ColType &colType= ci.columnTypes[colpos]; + const datatypes::TypeHandler *h= colType.typeHandler(); + if (h) // QQ: error reporting { - case CalpontSystemCatalog::DATE: //date fetch - { - - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - { - fprintf(ci.filePtr, "%c", ci.delimiter); - } - else - { - const uchar* tmp1 = buf; - uint32_t tmp = (tmp1[2] << 16) + (tmp1[1] << 8) + tmp1[0]; - - int day = tmp & 0x0000001fl; - int month = (tmp >> 5) & 0x0000000fl; - int year = tmp >> 9; - fprintf(ci.filePtr, "%04d-%02d-%02d%c", year, month, day, ci.delimiter); - } - - buf += 3; - break; - } - - case CalpontSystemCatalog::DATETIME: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - { - fprintf(ci.filePtr, "%c", ci.delimiter); - - if (table->field[colpos]->real_type() == MYSQL_TYPE_DATETIME2) - buf += table->field[colpos]->pack_length(); - else - buf += 8; - } - else - { - if (table->field[colpos]->real_type() == MYSQL_TYPE_DATETIME2) - { - // mariadb 10.1 compatibility -- MYSQL_TYPE_DATETIME2 introduced in mysql 5.6 - MYSQL_TIME ltime; - const uchar* pos = buf; - longlong tmp = my_datetime_packed_from_binary(pos, table->field[colpos]->decimals()); - TIME_from_longlong_datetime_packed(<ime, tmp); - - if (!ltime.second_part) - { - fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d%c", - ltime.year, ltime.month, ltime.day, - ltime.hour, ltime.minute, ltime.second, ci.delimiter); - } - else - { - fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d.%ld%c", - ltime.year, ltime.month, ltime.day, - ltime.hour, ltime.minute, ltime.second, - ltime.second_part, ci.delimiter); - } - - buf += table->field[colpos]->pack_length(); - } - else - { - long long value = *((long long*) buf); - long datePart = (long) (value / 1000000ll); - int day = datePart % 100; - int month = (datePart / 100) % 100; - int year = datePart / 10000; - fprintf(ci.filePtr, "%04d-%02d-%02d ", year, month, day); - - long timePart = (long) (value - (long long) datePart * 1000000ll); - int second = timePart % 100; - int min = (timePart / 100) % 100; - int hour = timePart / 10000; - fprintf(ci.filePtr, "%02d:%02d:%02d%c", hour, min, second, ci.delimiter); - buf += 8; - } - } - - break; - } - - case CalpontSystemCatalog::TIME: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - { - fprintf(ci.filePtr, "%c", ci.delimiter); - - buf += table->field[colpos]->pack_length(); - } - else - { - MYSQL_TIME ltime; - const uchar* pos = buf; - longlong tmp = my_time_packed_from_binary(pos, table->field[colpos]->decimals()); - TIME_from_longlong_time_packed(<ime, tmp); - - if (ltime.neg) - { - fprintf(ci.filePtr, "-"); - } - - if (!ltime.second_part) - { - fprintf(ci.filePtr, "%02d:%02d:%02d%c", - ltime.hour, ltime.minute, ltime.second, ci.delimiter); - } - else - { - fprintf(ci.filePtr, "%02d:%02d:%02d.%ld%c", - ltime.hour, ltime.minute, ltime.second, - ltime.second_part, ci.delimiter); - } - - buf += table->field[colpos]->pack_length(); - } - - break; - } - - case CalpontSystemCatalog::TIMESTAMP: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - { - fprintf(ci.filePtr, "%c", ci.delimiter); - } - else - { - const uchar* pos = buf; - struct timeval tm; - my_timestamp_from_binary(&tm, pos, table->field[colpos]->decimals()); - - MySQLTime time; - gmtSecToMySQLTime(tm.tv_sec, time, current_thd->variables.time_zone->get_name()->ptr()); - - if (!tm.tv_usec) - { - fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d%c", - time.year, time.month, time.day, - time.hour, time.minute, time.second, ci.delimiter); - } - else - { - fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d.%ld%c", - time.year, time.month, time.day, - time.hour, time.minute, time.second, - tm.tv_usec, ci.delimiter); - } - } - - buf += table->field[colpos]->pack_length(); - - break; - } - - case CalpontSystemCatalog::CHAR: - { - Field* field = table->field[colpos]; - - uint32_t colWidthInBytes = - ci.columnTypes[colpos].colWidth * field->charset()->mbmaxlen; - - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - { - fprintf(ci.filePtr, "%c", ci.delimiter); - } - else - { - if (current_thd->variables.sql_mode & MODE_PAD_CHAR_TO_FULL_LENGTH) - { - // Pad to the full length of the field - escape.assign((char*)buf, colWidthInBytes); - - boost::replace_all(escape, "\\", "\\\\"); - - fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, (int)escape.length(), - escape.c_str(), ci.enclosed_by, ci.delimiter); - } - else - { - // Get the actual data length - Field* field = table->field[colpos]; - bitmap_set_bit(table->read_set, field->field_index); - String attribute; - field->val_str(&attribute); - - escape.assign((char*)buf, attribute.length()); - boost::replace_all(escape, "\\", "\\\\"); - - fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, (int)escape.length(), - escape.c_str(), ci.enclosed_by, ci.delimiter); - } - } - - buf += colWidthInBytes; - - break; - } - - case CalpontSystemCatalog::VARCHAR: - { - Field* field = table->field[colpos]; - - uint32_t colWidthInBytes = - ci.columnTypes[colpos].colWidth * field->charset()->mbmaxlen; - - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - { - fprintf(ci.filePtr, "%c", ci.delimiter); - - if (colWidthInBytes < 256) - { - buf++; - } - else - { - buf = buf + 2 ; - } - } - else - { - // Maximum number of bytes allowed for a VARCHAR - // field is 65532, so the max length fits in 2 bytes. - // dataLength is length in bytes, not length in chars - uint16_t dataLength = 0; - - if (colWidthInBytes < 256) - { - dataLength = *(uint8_t*) buf; - buf++; - } - else - { - dataLength = *(uint16_t*) buf; - buf = buf + 2 ; - } - - escape.assign((char*)buf, dataLength); - boost::replace_all(escape, "\\", "\\\\"); - fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, (int)escape.length(), escape.c_str(), ci.enclosed_by, ci.delimiter); - } - - buf += colWidthInBytes; - - break; - } - - case CalpontSystemCatalog::BIGINT: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - fprintf(ci.filePtr, "%lld%c", *((long long*)buf), ci.delimiter); - - buf += 8; - break; - } - - case CalpontSystemCatalog::UBIGINT: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - fprintf(ci.filePtr, "%llu%c", *((long long unsigned*)buf), ci.delimiter); - - buf += 8; - break; - } - - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::MEDINT: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - fprintf(ci.filePtr, "%d%c", *((int32_t*)buf), ci.delimiter); - - buf += 4; - break; - } - - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UMEDINT: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - { - fprintf(ci.filePtr, "%u%c", *((uint32_t*)buf), ci.delimiter); - //printf("%u|", *((uint32_t*)buf)); - //cout << *((uint32_t*)buf) << endl; - } - - buf += 4; - break; - } - - case CalpontSystemCatalog::SMALLINT: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - fprintf(ci.filePtr, "%d%c", *((int16_t*)buf), ci.delimiter); - - buf += 2; - break; - } - - case CalpontSystemCatalog::USMALLINT: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - fprintf(ci.filePtr, "%u%c", *((uint16_t*)buf), ci.delimiter); - - buf += 2; - break; - } - - case CalpontSystemCatalog::TINYINT: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - fprintf(ci.filePtr, "%d%c", *((int8_t*)buf), ci.delimiter); - - buf += 1; - break; - } - - case CalpontSystemCatalog::UTINYINT: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - fprintf(ci.filePtr, "%u%c", *((uint8_t*)buf), ci.delimiter); - - buf += 1; - break; - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - { - float val = *((float*)buf); - - if ((fabs(val) > (1.0 / IDB_pow[4])) && (fabs(val) < (float) IDB_pow[6])) - { - fprintf(ci.filePtr, "%.7f%c", val, ci.delimiter); - } - else - { - fprintf(ci.filePtr, "%e%c", val, ci.delimiter); - } - - - //fprintf(ci.filePtr, "%.7g|", *((float*)buf)); - //printf("%.7f|", *((float*)buf)); - } - - buf += 4; - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - { - fprintf(ci.filePtr, "%.15g%c", *((double*)buf), ci.delimiter); - //printf("%.15g|", *((double*)buf)); - } - - buf += 8; - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - { - fprintf(ci.filePtr, "%.15Lg%c", *((long double*)buf), ci.delimiter); - //printf("%.15g|", *((double*)buf)); - } - - buf += 8; - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - uint bytesBefore = 1; - uint totalBytes = 9; - - switch (ci.columnTypes[colpos].precision) - { - case 18: - case 17: - case 16: - { - totalBytes = 8; - break; - } - - case 15: - case 14: - { - totalBytes = 7; - break; - } - - case 13: - case 12: - { - totalBytes = 6; - break; - } - - case 11: - { - totalBytes = 5; - break; - } - - case 10: - { - totalBytes = 5; - break; - } - - case 9: - case 8: - case 7: - { - totalBytes = 4; - break; - } - - case 6: - case 5: - { - totalBytes = 3; - break; - } - - case 4: - case 3: - { - totalBytes = 2; - break; - } - - case 2: - case 1: - { - totalBytes = 1; - break; - } - - default: - break; - } - - switch (ci.columnTypes[colpos].scale) - { - case 0: - { - bytesBefore = totalBytes; - break; - } - - case 1: //1 byte for digits after decimal point - { - if ((ci.columnTypes[colpos].precision != 16) && (ci.columnTypes[colpos].precision != 14) - && (ci.columnTypes[colpos].precision != 12) && (ci.columnTypes[colpos].precision != 10) - && (ci.columnTypes[colpos].precision != 7) && (ci.columnTypes[colpos].precision != 5) - && (ci.columnTypes[colpos].precision != 3) && (ci.columnTypes[colpos].precision != 1)) - totalBytes++; - - bytesBefore = totalBytes - 1; - break; - } - - case 2: //1 byte for digits after decimal point - { - if ((ci.columnTypes[colpos].precision == 18) || (ci.columnTypes[colpos].precision == 9)) - totalBytes++; - - bytesBefore = totalBytes - 1; - break; - } - - case 3: //2 bytes for digits after decimal point - { - if ((ci.columnTypes[colpos].precision != 16) && (ci.columnTypes[colpos].precision != 14) - && (ci.columnTypes[colpos].precision != 12) && (ci.columnTypes[colpos].precision != 7) - && (ci.columnTypes[colpos].precision != 5) && (ci.columnTypes[colpos].precision != 3)) - totalBytes++; - - bytesBefore = totalBytes - 2; - break; - } - - case 4: - { - if ((ci.columnTypes[colpos].precision == 18) || (ci.columnTypes[colpos].precision == 11) - || (ci.columnTypes[colpos].precision == 9)) - totalBytes++; - - bytesBefore = totalBytes - 2; - break; - - } - - case 5: - { - if ((ci.columnTypes[colpos].precision != 16) && (ci.columnTypes[colpos].precision != 14) - && (ci.columnTypes[colpos].precision != 7) && (ci.columnTypes[colpos].precision != 5)) - totalBytes++; - - bytesBefore = totalBytes - 3; - break; - } - - case 6: - { - if ((ci.columnTypes[colpos].precision == 18) || (ci.columnTypes[colpos].precision == 13) - || (ci.columnTypes[colpos].precision == 11) || (ci.columnTypes[colpos].precision == 9)) - totalBytes++; - - bytesBefore = totalBytes - 3; - break; - } - - case 7: - { - if ((ci.columnTypes[colpos].precision != 16) && (ci.columnTypes[colpos].precision != 7)) - totalBytes++; - - bytesBefore = totalBytes - 4; - break; - } - - case 8: - { - if ((ci.columnTypes[colpos].precision == 18) || (ci.columnTypes[colpos].precision == 15) - || (ci.columnTypes[colpos].precision == 13) || (ci.columnTypes[colpos].precision == 11) - || (ci.columnTypes[colpos].precision == 9)) - totalBytes++; - - bytesBefore = totalBytes - 4;; - break; - } - - case 9: - { - bytesBefore = totalBytes - 4;; - break; - } - - case 10: - { - if ((ci.columnTypes[colpos].precision != 16) && (ci.columnTypes[colpos].precision != 14) - && (ci.columnTypes[colpos].precision != 12) && (ci.columnTypes[colpos].precision != 10)) - totalBytes++; - - bytesBefore = totalBytes - 5;; - break; - } - - case 11: - { - if (ci.columnTypes[colpos].precision == 18) - totalBytes++; - - bytesBefore = totalBytes - 5; - break; - } - - case 12: - { - if ((ci.columnTypes[colpos].precision != 16) && (ci.columnTypes[colpos].precision != 14) - && (ci.columnTypes[colpos].precision != 12)) - totalBytes++; - - bytesBefore = totalBytes - 6; - break; - } - - case 13: - { - if (ci.columnTypes[colpos].precision == 18) - totalBytes++; - - bytesBefore = totalBytes - 6; - break; - } - - case 14: - { - if ((ci.columnTypes[colpos].precision != 16) && (ci.columnTypes[colpos].precision != 14)) - totalBytes++; - - bytesBefore = totalBytes - 7; - break; - } - - case 15: - { - if (ci.columnTypes[colpos].precision == 18) - totalBytes++; - - bytesBefore = totalBytes - 7; - break; - } - - case 16: - { - if (ci.columnTypes[colpos].precision != 16) - totalBytes++; - - bytesBefore = totalBytes - 8; - break; - } - - case 17: - { - if (ci.columnTypes[colpos].precision == 18) - totalBytes++; - - bytesBefore = totalBytes - 8; - break; - } - - case 18: - { - bytesBefore = totalBytes - 8; - break; - } - - default: - break; - } - - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - { - fprintf(ci.filePtr, "%c", ci.delimiter); - //printf("|"); - } - else - { - uint32_t mask [5] = {0, 0xFF, 0xFFFF, 0xFFFFFF, 0xFFFFFFFF}; - char neg = '-'; - - if (ci.columnTypes[colpos].scale == 0) - { - const uchar* tmpBuf = buf; - //test flag bit for sign - bool posNum = tmpBuf[0] & (0x80); - uchar tmpChr = tmpBuf[0]; - tmpChr ^= 0x80; //flip the bit - int32_t tmp1 = tmpChr; - - if (totalBytes > 4) - { - for (uint i = 1; i < (totalBytes - 4); i++) - { - tmp1 = (tmp1 << 8) + tmpBuf[i]; - } - - if (( tmp1 != 0 ) && (tmp1 != -1)) - { - if (!posNum) - { - tmp1 = mask[totalBytes - 4] - tmp1; - - if (tmp1 != 0) - { - fprintf(ci.filePtr, "%c", neg); - //printf("%c", neg); - } - } - - if (tmp1 != 0) - { - fprintf(ci.filePtr, "%d", tmp1); - ////printf("%d", tmp1); - } - } - - int32_t tmp2 = tmpBuf[totalBytes - 4]; - - for (uint i = (totalBytes - 3); i < totalBytes; i++) - { - tmp2 = (tmp2 << 8) + tmpBuf[i]; - } - - if ( tmp1 != 0 ) - { - if (!posNum) - { - tmp2 = mask[4] - tmp2; - - if (tmp1 == -1) - { - fprintf(ci.filePtr, "%c", neg); - fprintf(ci.filePtr, "%d%c", tmp2, ci.delimiter); - ////printf("%c", neg); - //////printf( "%d|", tmp2); - } - else - { - fprintf(ci.filePtr, "%09u%c", tmp2, ci.delimiter); - ////printf("%09u|", tmp2); - } - } - else - { - fprintf(ci.filePtr, "%09u%c", tmp2, ci.delimiter); - //printf("%09u|", tmp2); - } - } - else - { - if (!posNum) - { - tmp2 = mask[4] - tmp2; - fprintf(ci.filePtr, "%c", neg); - //printf("%c", neg); - } - - fprintf(ci.filePtr, "%d%c", tmp2, ci.delimiter); - //printf("%d|", tmp2); - } - } - else - { - for (uint i = 1; i < totalBytes; i++) - { - tmp1 = (tmp1 << 8) + tmpBuf[i]; - } - - if (!posNum) - { - tmp1 = mask[totalBytes] - tmp1; - fprintf(ci.filePtr, "%c", neg); - //printf("%c", neg); - } - - fprintf(ci.filePtr, "%d%c", tmp1, ci.delimiter); - //printf("%d|", tmp1); - } - } - else - { - const uchar* tmpBuf = buf; - //test flag bit for sign - bool posNum = tmpBuf[0] & (0x80); - uchar tmpChr = tmpBuf[0]; - tmpChr ^= 0x80; //flip the bit - int32_t tmp1 = tmpChr; - - //fetch the digits before decimal point - if (bytesBefore == 0) - { - if (!posNum) - { - fprintf(ci.filePtr, "%c", neg); - //printf("%c", neg); - } - - fprintf(ci.filePtr, "0."); - //printf("0."); - } - else if (bytesBefore > 4) - { - for (uint i = 1; i < (bytesBefore - 4); i++) - { - tmp1 = (tmp1 << 8) + tmpBuf[i]; - } - - if (!posNum) - { - tmp1 = mask[bytesBefore - 4] - tmp1; - } - - if (( tmp1 != 0 ) && (tmp1 != -1)) - { - if (!posNum) - { - fprintf(ci.filePtr, "%c", neg); - //printf("%c", neg); - } - - fprintf(ci.filePtr, "%d", tmp1); - //printf("%d", tmp1); - } - - tmpBuf += (bytesBefore - 4); - int32_t tmp2 = *((int32_t*)tmpBuf); - tmp2 = ntohl(tmp2); - - if ( tmp1 != 0 ) - { - if (!posNum) - { - tmp2 = mask[4] - tmp2; - } - - if (tmp1 == -1) - { - fprintf(ci.filePtr, "%c", neg); - fprintf(ci.filePtr, "%d.", tmp2); - //printf("%c", neg); - //printf("%d.", tmp2); - } - else - { - fprintf(ci.filePtr, "%09u.", tmp2); - //printf("%09u.", tmp2); - } - } - else - { - if (!posNum) - { - tmp2 = mask[4] - tmp2; - fprintf(ci.filePtr, "%c", neg); - //printf("%c", neg); - } - - fprintf(ci.filePtr, "%d.", tmp2); - //printf("%d.", tmp2); - } - } - else - { - for (uint i = 1; i < bytesBefore; i++) - { - tmp1 = (tmp1 << 8) + tmpBuf[i]; - } - - if (!posNum) - { - tmp1 = mask[bytesBefore] - tmp1; - fprintf(ci.filePtr, "%c", neg); - //printf("%c", neg); - } - - fprintf(ci.filePtr, "%d.", tmp1); - //printf("%d.", tmp1); - } - - //fetch the digits after decimal point - int32_t tmp2 = 0; - - if (bytesBefore > 4) - tmpBuf += 4; - else - tmpBuf += bytesBefore; - - tmp2 = tmpBuf[0]; - - if ((totalBytes - bytesBefore) < 5) - { - for (uint j = 1; j < (totalBytes - bytesBefore); j++) - { - tmp2 = (tmp2 << 8) + tmpBuf[j]; - } - - int8_t digits = ci.columnTypes[colpos].scale - 9; //9 digits is a 4 bytes chunk - - if ( digits <= 0 ) - digits = ci.columnTypes[colpos].scale; - - if (!posNum) - { - tmp2 = mask[totalBytes - bytesBefore] - tmp2; - } - - fprintf(ci.filePtr, "%0*u%c", digits, tmp2, ci.delimiter); - //printf("%0*u|", digits, tmp2); - } - else - { - for (uint j = 1; j < 4; j++) - { - tmp2 = (tmp2 << 8) + tmpBuf[j]; - } - - if (!posNum) - { - tmp2 = mask[4] - tmp2; - } - - fprintf(ci.filePtr, "%09u", tmp2); - //printf("%09u", tmp2); - - tmpBuf += 4; - int32_t tmp3 = tmpBuf[0]; - - for (uint j = 1; j < (totalBytes - bytesBefore - 4); j++) - { - tmp3 = (tmp3 << 8) + tmpBuf[j]; - } - - int8_t digits = ci.columnTypes[colpos].scale - 9; //9 digits is a 4 bytes chunk - - if ( digits < 0 ) - digits = ci.columnTypes[colpos].scale; - - if (!posNum) - { - tmp3 = mask[totalBytes - bytesBefore - 4] - tmp3; - } - - fprintf(ci.filePtr, "%0*u%c", digits, tmp3, ci.delimiter); - //printf("%0*u|", digits, tmp3); - } - } - } - - buf += totalBytes; - break; - } - - case CalpontSystemCatalog::VARBINARY: - { - // For a VARBINARY field, ci.columnTypes[colpos].colWidth == colWidthInBytes - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - { - fprintf(ci.filePtr, "%c", ci.delimiter); - - if (ci.columnTypes[colpos].colWidth < 256) - { - buf++; - } - else - { - buf = buf + 2; - } - } - else - { - // Maximum number of bytes allowed for a VARBINARY - // field is 65532, so the max length fits in 2 bytes. - // dataLength is length in bytes, not length in chars - uint16_t dataLength = 0; - - if (ci.columnTypes[colpos].colWidth < 256) - { - dataLength = *(uint8_t*) buf; - buf++; - } - else - { - dataLength = *(uint16_t*) buf; - buf = buf + 2 ; - } - - const uchar* tmpBuf = buf; - - for (int32_t i = 0; i < dataLength; i++) - { - fprintf(ci.filePtr, "%02x", *(uint8_t*)tmpBuf); - tmpBuf++; - } - - fprintf(ci.filePtr, "%c", ci.delimiter); - } - - buf += ci.columnTypes[colpos].colWidth; - - break; - } - - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - { - // MCOL-4005 Note that we don't handle nulls as a special - // case here as we do for other datatypes, the below works - // as expected for nulls. - // dataLength is length in bytes, not length in chars - uint32_t dataLength = 0; - uintptr_t* dataptr; - uchar* ucharptr; - - bool isBlob = - ci.columnTypes[colpos].colDataType == CalpontSystemCatalog::BLOB; - - Field* field = table->field[colpos]; - - uint32_t colWidthInBytes = isBlob ? ci.columnTypes[colpos].colWidth : - ci.columnTypes[colpos].colWidth * field->charset()->mbmaxlen; - - if (!isBlob && field->char_length() == 65535) - { - // Special case for TEXT field without default length, - // such as: - // CREATE TABLE mcol4364 (a TEXT); - // Here, char_length() represents the number of bytes, - // not number of characters. - dataLength = *(uint16_t*) buf; - buf += 2; - } - else if (colWidthInBytes < 256) - { - dataLength = *(uint8_t*) buf; - buf++; - } - else if (colWidthInBytes < 65536) - { - dataLength = *(uint16_t*) buf; - buf += 2; - } - else if (colWidthInBytes < 16777216) - { - dataLength = *(uint16_t*) buf; - dataLength |= ((int) buf[2]) << 16; - buf += 3; - } - else - { - dataLength = *(uint32_t*) buf; - buf += 4; - } - - // buf contains pointer to blob, for example: - // (gdb) p (char*)*(uintptr_t*)buf - // $43 = 0x7f68500c58f8 "hello world" - - dataptr = (uintptr_t*)buf; - ucharptr = (uchar*)*dataptr; - buf += sizeof(uintptr_t); - - if (isBlob) - { - for (uint32_t i = 0; i < dataLength; i++) - { - fprintf(ci.filePtr, "%02x", *(uint8_t*)ucharptr); - ucharptr++; - } - - fprintf(ci.filePtr, "%c", ci.delimiter); - } - else - { - // TEXT Column - escape.assign((char*)ucharptr, dataLength); - boost::replace_all(escape, "\\", "\\\\"); - fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, (int)escape.length(), escape.c_str(), ci.enclosed_by, ci.delimiter); - } - - break; - - } - - default: // treat as int64 - { - break; - } + datatypes::ColBatchWriter writer(ci.filePtr, + ci.delimiter, + ci.enclosed_by); + Field* fieldPtr= table->field[colpos]; + uint32_t mbmaxlen = (fieldPtr->charset() && fieldPtr->charset()->mbmaxlen) + ? fieldPtr->charset()->mbmaxlen : 0; + datatypes::WriteBatchFieldMariaDB field(fieldPtr, + colType, + mbmaxlen); + idbassert(table == table->field[colpos]->table); + buf+= h->ColWriteBatch(&field, buf, nullVal, writer); } - colpos++; } } diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 8a78fbddd..a5daf7048 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -3105,14 +3105,13 @@ CalpontSystemCatalog::ColType colType_MysqlToIDB (const Item* item) case DECIMAL_RESULT: { Item_decimal* idp = (Item_decimal*)item; - ct.colDataType = CalpontSystemCatalog::DECIMAL; - ct.colWidth = 8; - ct.scale = idp->decimals; - if (ct.scale == 0) - ct.precision = idp->max_length - 1; - else - ct.precision = idp->max_length - idp->decimals; + ct.colDataType = CalpontSystemCatalog::DECIMAL; + + unsigned int precision = idp->decimal_precision(); + unsigned int scale = idp->decimal_scale(); + + ct.setDecimalScalePrecision(precision, scale); break; } @@ -3138,7 +3137,7 @@ ReturnedColumn* buildReturnedColumn( { ReturnedColumn* rc = NULL; - if ( gwi.thd) + if (gwi.thd) { //if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI )) { @@ -3174,7 +3173,8 @@ ReturnedColumn* buildReturnedColumn( if (item->unsigned_flag) { - rc = new ConstantColumn((uint64_t)item->val_uint(), ConstantColumn::NUM); + rc = new ConstantColumn((uint64_t)item->val_uint(), ConstantColumn::NUM, + (int8_t) item->decimal_scale(), (uint8_t) item->decimal_precision()); } else { @@ -3441,6 +3441,7 @@ ArithmeticColumn* buildArithmeticColumn( Item** sfitempp = item->arguments(); ArithmeticOperator* aop = new ArithmeticOperator(item->func_name()); aop->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + aop->setOverflowCheck(get_decimal_overflow_check(gwi.thd)); ParseTree* pt = new ParseTree(aop); //ReturnedColumn *lhs = 0, *rhs = 0; ParseTree* lhs = 0, *rhs = 0; @@ -3599,16 +3600,53 @@ ArithmeticColumn* buildArithmeticColumn( pt->right(rhs); } - //aop->resultType(colType_MysqlToIDB(item)); // @bug5715. Use InfiniDB adjusted coltype for result type. // decimal arithmetic operation gives double result when the session variable is set. - //idbassert(pt->left() && pt->right() && pt->left()->data() && pt->right()->data()); - CalpontSystemCatalog::ColType mysql_type = colType_MysqlToIDB(item); + CalpontSystemCatalog::ColType mysqlType = colType_MysqlToIDB(item); + + const CalpontSystemCatalog::ColType& leftColType = pt->left()->data()->resultType(); + const CalpontSystemCatalog::ColType& rightColType = pt->right()->data()->resultType(); + + // Only tinker with the type if all columns involved are decimal + if (datatypes::isDecimalOperands(mysqlType.colDataType, + leftColType.colDataType, rightColType.colDataType)) + { + int32_t leftColWidth = leftColType.colWidth; + int32_t rightColWidth = rightColType.colWidth; + + if (leftColWidth == datatypes::MAXDECIMALWIDTH || + rightColWidth == datatypes::MAXDECIMALWIDTH) + { + mysqlType.colWidth = datatypes::MAXDECIMALWIDTH; + + string funcName = item->func_name(); + + int32_t scale1 = leftColType.scale; + int32_t scale2 = rightColType.scale; + + if (funcName == "/" && + (mysqlType.scale - (scale1 - scale2)) > datatypes::INT128MAXPRECISION) + { + Item_decimal* idp = (Item_decimal*)item; + + unsigned int precision = idp->decimal_precision(); + unsigned int scale = idp->decimal_scale(); + + mysqlType.setDecimalScalePrecisionHeuristic(precision, scale); + + if (mysqlType.scale < scale1) + mysqlType.scale = scale1; + + if (mysqlType.precision < mysqlType.scale) + mysqlType.precision = mysqlType.scale; + } + } + } if (get_double_for_decimal_math(current_thd) == true) - aop->adjustResultType(mysql_type); + aop->adjustResultType(mysqlType); else - aop->resultType(mysql_type); + aop->resultType(mysqlType); // adjust decimal result type according to internalDecimalScale if (gwi.internalDecimalScale >= 0 && aop->resultType().colDataType == CalpontSystemCatalog::DECIMAL) @@ -4043,6 +4081,20 @@ ReturnedColumn* buildFunctionColumn( fc->functionParms(funcParms); fc->resultType(colType_MysqlToIDB(ifp)); + // if the result type is DECIMAL and any function parameter is a wide decimal + // column, set the result colwidth to wide + if (fc->resultType().colDataType == CalpontSystemCatalog::DECIMAL) + { + for (size_t i = 0; i < funcParms.size(); i++) + { + if (funcParms[i]->data()->resultType().isWideDecimalType()) + { + fc->resultType().colWidth = datatypes::MAXDECIMALWIDTH; + break; + } + } + } + // MySQL give string result type for date function, but has the flag set. // we should set the result type to be datetime for comparision. if (ifp->field_type() == MYSQL_TYPE_DATETIME || @@ -4358,6 +4410,14 @@ ConstantColumn* buildDecimalColumn(Item* item, gp_walk_info& gwi) i = 1; } + bool specialPrecision = false; + + // handle the case when the constant value is 0.12345678901234567890123456789012345678 + // In this case idp->decimal_precision() = 39, but we can + if (((i + 1) < str->length()) && + str->ptr()[i] == '0' && str->ptr()[i + 1] == '.') + specialPrecision = true; + for (; i < str->length(); i++) { if (str->ptr()[i] == '.') @@ -4366,8 +4426,18 @@ ConstantColumn* buildDecimalColumn(Item* item, gp_walk_info& gwi) columnstore_decimal_val << str->ptr()[i]; } - columnstore_decimal.value = strtoll(columnstore_decimal_val.str().c_str(), 0, 10); + if (idp->decimal_precision() <= datatypes::INT64MAXPRECISION) + columnstore_decimal.value = strtoll(columnstore_decimal_val.str().c_str(), 0, 10); + else if (idp->decimal_precision() <= datatypes::INT128MAXPRECISION || + (idp->decimal_precision() <= datatypes::INT128MAXPRECISION + 1 && + specialPrecision)) + { + bool dummy = false; + columnstore_decimal.s128Value = + dataconvert::strtoll128(columnstore_decimal_val.str().c_str(), dummy, 0); + } + // TODO MCOL-641 Add support here if (gwi.internalDecimalScale >= 0 && idp->decimals > (uint)gwi.internalDecimalScale) { columnstore_decimal.scale = gwi.internalDecimalScale; @@ -4375,9 +4445,10 @@ ConstantColumn* buildDecimalColumn(Item* item, gp_walk_info& gwi) columnstore_decimal.value = (int64_t)(val > 0 ? val + 0.5 : val - 0.5); } else - columnstore_decimal.scale = idp->decimals; + columnstore_decimal.scale = idp->decimal_scale(); - columnstore_decimal.precision = idp->max_length - idp->decimals; + columnstore_decimal.precision = (idp->decimal_precision() > datatypes::INT128MAXPRECISION) ? + datatypes::INT128MAXPRECISION : idp->decimal_precision(); ConstantColumn* cc = new ConstantColumn(valStr, columnstore_decimal); cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); cc->charsetNumber(idp->collation.collation->number); @@ -4404,18 +4475,18 @@ SimpleColumn* buildSimpleColumn(Item_field* ifp, gp_walk_info& gwi) return buildSimpleColFromDerivedTable(gwi, ifp); CalpontSystemCatalog::ColType ct; - bool columnStore = true; + datatypes::SimpleColumnParam prm(gwi.sessionid, true); try { // check foreign engine if (ifp->cached_table && ifp->cached_table->table) - columnStore = isMCSTable(ifp->cached_table->table); + prm.columnStore(isMCSTable(ifp->cached_table->table)); // @bug4509. ifp->cached_table could be null for myisam sometimes else if (ifp->field && ifp->field->table) - columnStore = isMCSTable(ifp->field->table); + prm.columnStore(isMCSTable(ifp->field->table)); - if (columnStore) + if (prm.columnStore()) { ct = gwi.csc->colType( gwi.csc->lookupOID(make_tcn(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str))); @@ -4432,75 +4503,12 @@ SimpleColumn* buildSimpleColumn(Item_field* ifp, gp_walk_info& gwi) return NULL; } - SimpleColumn* sc = NULL; + const datatypes::DatabaseQualifiedColumnName name(ifp->db_name.str, + bestTableName(ifp), + ifp->field_name.str); + const datatypes::TypeHandler *h= ct.typeHandler(); + SimpleColumn *sc = h->newSimpleColumn(name, ct, prm); - switch (ct.colDataType) - { - case CalpontSystemCatalog::TINYINT: - if (ct.scale == 0) - sc = new SimpleColumn_INT<1>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - else - { - sc = new SimpleColumn_Decimal<1>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - ct.colDataType = CalpontSystemCatalog::DECIMAL; - } - - break; - - case CalpontSystemCatalog::SMALLINT: - if (ct.scale == 0) - sc = new SimpleColumn_INT<2>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - else - { - sc = new SimpleColumn_Decimal<2>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - ct.colDataType = CalpontSystemCatalog::DECIMAL; - } - - break; - - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::MEDINT: - if (ct.scale == 0) - sc = new SimpleColumn_INT<4>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - else - { - sc = new SimpleColumn_Decimal<4>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - ct.colDataType = CalpontSystemCatalog::DECIMAL; - } - - break; - - case CalpontSystemCatalog::BIGINT: - if (ct.scale == 0) - sc = new SimpleColumn_INT<8>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - else - { - sc = new SimpleColumn_Decimal<8>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - ct.colDataType = CalpontSystemCatalog::DECIMAL; - } - - break; - - case CalpontSystemCatalog::UTINYINT: - sc = new SimpleColumn_UINT<1>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - break; - - case CalpontSystemCatalog::USMALLINT: - sc = new SimpleColumn_UINT<2>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - break; - - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UMEDINT: - sc = new SimpleColumn_UINT<4>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - break; - - case CalpontSystemCatalog::UBIGINT: - sc = new SimpleColumn_UINT<8>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - break; - - default: - sc = new SimpleColumn(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - } sc->resultType(ct); sc->charsetNumber(ifp->collation.collation->number); string tbname(ifp->table_name.str); @@ -4517,10 +4525,10 @@ SimpleColumn* buildSimpleColumn(Item_field* ifp, gp_walk_info& gwi) sc->viewName(getViewName(ifp->cached_table), lower_case_table_names); sc->alias(ifp->name.str); - sc->isColumnStore(columnStore); + sc->isColumnStore(prm.columnStore()); sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - if (!columnStore && ifp->field) + if (!prm.columnStore() && ifp->field) sc->oid(ifp->field->field_index + 1); // ExeMgr requires offset started from 1 if (ifp->depended_from) @@ -4583,6 +4591,9 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) vector selCols; vector orderCols; bool bIsConst = false; + unsigned int constValPrecision = 0; + unsigned int constValScale = 0; + bool hasDecimalConst = false; if (get_fe_conn_info_ptr() == NULL) set_fe_conn_info_ptr((void*)new cal_connection_info()); @@ -4590,7 +4601,6 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) Item_sum* isp = reinterpret_cast(item); Item** sfitempp = isp->get_orig_args(); -// Item** sfitempp = isp->arguments(); SRCP parm; // @bug4756 @@ -4767,6 +4777,13 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) parm.reset(buildReturnedColumn(sfitemp, gwi, gwi.fatalParseError)); ac->constCol(parm); bIsConst = true; + if (sfitemp->cmp_type() == DECIMAL_RESULT) + { + hasDecimalConst = true; + Item_decimal* idp = (Item_decimal*)sfitemp; + constValPrecision = idp->decimal_precision(); + constValScale = idp->decimal_scale(); + } break; } default: @@ -4810,7 +4827,6 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) if ((fc && fc->functionParms().empty()) || !fc) { - //ac->aggOp(AggregateColumn::COUNT_ASTERISK); ReturnedColumn* rc = buildReturnedColumn(sfitemp, gwi, gwi.fatalParseError); if (dynamic_cast(rc)) @@ -4889,6 +4905,9 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) } } + bool isAvg = (isp->sum_func() == Item_sum::AVG_FUNC || + isp->sum_func() == Item_sum::AVG_DISTINCT_FUNC); + // Get result type // Modified for MCOL-1201 multi-argument aggregate if (!bIsConst && ac->aggParms().size() > 0) @@ -4897,14 +4916,31 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) // use the first parm for result type. parm = ac->aggParms()[0]; - if (isp->sum_func() == Item_sum::AVG_FUNC || - isp->sum_func() == Item_sum::AVG_DISTINCT_FUNC) + if (isAvg || isp->sum_func() == Item_sum::SUM_FUNC || + isp->sum_func() == Item_sum::SUM_DISTINCT_FUNC) { CalpontSystemCatalog::ColType ct = parm->resultType(); - ct.colDataType = CalpontSystemCatalog::LONGDOUBLE; - ct.colWidth = sizeof(long double); - ct.scale += 4; - ct.precision = -1; + if (ct.isWideDecimalType()) + { + uint32_t precision = ct.precision; + uint32_t scale = ct.scale; + if (isAvg) + { + datatypes::Decimal::setScalePrecision4Avg(precision, scale); + } + ct.precision = precision; + ct.scale = scale; + } + else + { + ct.colDataType = CalpontSystemCatalog::LONGDOUBLE; + ct.colWidth = sizeof(long double); + if (isAvg) + { + ct.scale += datatypes::MAXSCALEINC4AVG; + } + ct.precision = datatypes::IGNOREPRECISION; + } ac->resultType(ct); } else if (isp->sum_func() == Item_sum::COUNT_FUNC || @@ -4916,15 +4952,6 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) ct.scale = parm->resultType().scale; ac->resultType(ct); } - else if (isp->sum_func() == Item_sum::SUM_FUNC || - isp->sum_func() == Item_sum::SUM_DISTINCT_FUNC) - { - CalpontSystemCatalog::ColType ct = parm->resultType(); - ct.colDataType = CalpontSystemCatalog::LONGDOUBLE; - ct.colWidth = sizeof(long double); - ct.precision = -1; - ac->resultType(ct); - } else if (isp->sum_func() == Item_sum::STD_FUNC || isp->sum_func() == Item_sum::VARIANCE_FUNC) { @@ -4958,13 +4985,27 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) ac->resultType(parm->resultType()); } } + else if (bIsConst && hasDecimalConst && isAvg) + { + CalpontSystemCatalog::ColType ct = parm->resultType(); + if (datatypes::Decimal::isWideDecimalTypeByPrecision(constValPrecision)) + { + ct.precision = constValPrecision; + ct.scale = constValScale; + ct.colWidth = datatypes::MAXDECIMALWIDTH; + } + ac->resultType(ct); + } else { ac->resultType(colType_MysqlToIDB(isp)); } // adjust decimal result type according to internalDecimalScale - if (gwi.internalDecimalScale >= 0 && ac->resultType().colDataType == CalpontSystemCatalog::DECIMAL) + bool isWideDecimal = ac->resultType().isWideDecimalType(); + // This must be also valid for UDECIMAL + if (!isWideDecimal && gwi.internalDecimalScale >= 0 + && ac->resultType().colDataType == CalpontSystemCatalog::DECIMAL) { CalpontSystemCatalog::ColType ct = ac->resultType(); ct.scale = gwi.internalDecimalScale; @@ -7308,7 +7349,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, gwi.returnedCols[i]->hasAggregate(true); } - gwi.returnedCols[i]->resultType(dataconvert::DataConvert::convertUnionColType(coltypes)); + gwi.returnedCols[i]->resultType(CalpontSystemCatalog::ColType::convertUnionColType(coltypes)); } } @@ -9058,7 +9099,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro gwi.returnedCols[i]->hasAggregate(true); } - gwi.returnedCols[i]->resultType(dataconvert::DataConvert::convertUnionColType(coltypes)); + gwi.returnedCols[i]->resultType(CalpontSystemCatalog::ColType::convertUnionColType(coltypes)); } } diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 2f8cbe220..d1868f52d 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -56,6 +56,7 @@ using namespace std; #include #include +#include "mcs_basic_types.h" #include "idb_mysql.h" #define NEED_CALPONT_INTERFACE @@ -138,11 +139,14 @@ using namespace funcexp; #include "columnstoreversion.h" #include "ha_mcs_sysvars.h" +#include "ha_mcs_datatype.h" + namespace cal_impl_if { extern bool nonConstFunc(Item_func* ifp); } + namespace { // Calpont vtable non-support error message @@ -247,106 +251,6 @@ void force_close_fep_conn(THD *thd, cal_connection_info* ci, bool check_prev_rc ci->cal_conn_hndl = 0; } -void storeNumericField(Field** f, int64_t value, CalpontSystemCatalog::ColType& ct) -{ - // unset null bit first - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - // For unsigned, use the ColType returned in the row rather than the - // unsigned_flag set by mysql. This is because mysql gets it wrong for SUM() - // Hopefully, in all other cases we get it right. - switch ((*f)->type()) - { - case MYSQL_TYPE_NEWDECIMAL: - { - // @bug4388 stick to InfiniDB's scale in case mysql gives wrong scale due - // to create vtable limitation. - //if (f2->dec < ct.scale) - // f2->dec = ct.scale; - - char buf[256]; - dataconvert::DataConvert::decimalToString(value, (unsigned)ct.scale, buf, 256, ct.colDataType); - (*f)->store(buf, strlen(buf), (*f)->charset()); - break; - } - - case MYSQL_TYPE_TINY: //TINYINT type - { - Field_tiny* f2 = (Field_tiny*)*f; - longlong int_val = (longlong)value; - (*f)->store(int_val, f2->unsigned_flag); - break; - } - - case MYSQL_TYPE_SHORT: //SMALLINT type - { - Field_short* f2 = (Field_short*)*f; - longlong int_val = (longlong)value; - (*f)->store(int_val, f2->unsigned_flag); - break; - } - - case MYSQL_TYPE_INT24: //MEDINT type - { - Field_medium* f2 = (Field_medium*)*f; - longlong int_val = (longlong)value; - (*f)->store(int_val, f2->unsigned_flag); - break; - } - - case MYSQL_TYPE_LONG: //INT type - { - Field_long* f2 = (Field_long*)*f; - longlong int_val = (longlong)value; - (*f)->store(int_val, f2->unsigned_flag); - break; - } - - case MYSQL_TYPE_LONGLONG: //BIGINT type - { - Field_longlong* f2 = (Field_longlong*)*f; - longlong int_val = (longlong)value; - (*f)->store(int_val, f2->unsigned_flag); - break; - } - - case MYSQL_TYPE_FLOAT: // FLOAT type - { - float float_val = *(float*)(&value); - (*f)->store(float_val); - break; - } - - case MYSQL_TYPE_DOUBLE: // DOUBLE type - { - double double_val = *(double*)(&value); - (*f)->store(double_val); - break; - } - - case MYSQL_TYPE_VARCHAR: - { - char tmp[25]; - if (ct.colDataType == CalpontSystemCatalog::DECIMAL) - dataconvert::DataConvert::decimalToString(value, (unsigned)ct.scale, tmp, 25, ct.colDataType); - else - snprintf(tmp, 25, "%lld", (long long)value); - - (*f)->store(tmp, strlen(tmp), (*f)->charset()); - break; - } - - default: - { - Field_longlong* f2 = (Field_longlong*)*f; - longlong int_val = (longlong)value; - (*f)->store(int_val, f2->unsigned_flag); - break; - } - } -} - // // @bug 2244. Log exception related to lost connection to ExeMgr. // Log exception error from calls to sm::tpl_scan_fetch in fetchNextRow() @@ -387,19 +291,6 @@ void tpl_scan_fetch_LogException( cal_table_info& ti, cal_connection_info* ci, s sesID << "; " << connHndl << "; rowsReturned: " << rowsRet << endl; } -const char hexdig[] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', }; - -int vbin2hex(const uint8_t* p, const unsigned l, char* o) -{ - for (unsigned i = 0; i < l; i++, p++) - { - *o++ = hexdig[*p >> 4]; - *o++ = hexdig[*p & 0xf]; - } - - return 0; -} - // Table Map is used by both cond_push and table mode processing // Entries made by cond_push don't have csep though. // When @@ -417,6 +308,7 @@ bool onlyOneTableinTM(cal_impl_if::cal_connection_info* ci) return true; } + int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool handler_flag = false) { int rc = HA_ERR_END_OF_FILE; @@ -469,9 +361,6 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h } std::vector& colTypes = ti.tpl_scan_ctx->ctp; - int64_t intColVal = 0; - uint64_t uintColVal = 0; - char tmp[256]; RowGroup* rowGroup = ti.tpl_scan_ctx->rowGroup; @@ -544,286 +433,25 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h colType.colDataType == CalpontSystemCatalog::VARCHAR || colType.colDataType == CalpontSystemCatalog::VARBINARY) { - (*f)->store(tmp, 0, (*f)->charset()); + (*f)->store("", 0, (*f)->charset()); } continue; } - // fetch and store data - switch (colType.colDataType) + const datatypes::TypeHandler *h= colType.typeHandler(); + if (!h) { - case CalpontSystemCatalog::DATE: - { - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - intColVal = row.getUintField<4>(s); - DataConvert::dateToString(intColVal, tmp, 255); - (*f)->store(tmp, strlen(tmp), (*f)->charset()); - break; - } - - case CalpontSystemCatalog::DATETIME: - { - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - intColVal = row.getUintField<8>(s); - DataConvert::datetimeToString(intColVal, tmp, 255, colType.precision); - (*f)->store(tmp, strlen(tmp), (*f)->charset()); - break; - } - - case CalpontSystemCatalog::TIME: - { - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - intColVal = row.getUintField<8>(s); - DataConvert::timeToString(intColVal, tmp, 255, colType.precision); - (*f)->store(tmp, strlen(tmp), (*f)->charset()); - break; - } - - case CalpontSystemCatalog::TIMESTAMP: - { - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - intColVal = row.getUintField<8>(s); - DataConvert::timestampToString(intColVal, tmp, 255, current_thd->variables.time_zone->get_name()->ptr(), colType.precision); - (*f)->store(tmp, strlen(tmp), (*f)->charset()); - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - { - switch (colType.colWidth) - { - case 1: - intColVal = row.getUintField<1>(s); - (*f)->store((char*)(&intColVal), strlen((char*)(&intColVal)), (*f)->charset()); - break; - - case 2: - intColVal = row.getUintField<2>(s); - (*f)->store((char*)(&intColVal), strlen((char*)(&intColVal)), (*f)->charset()); - break; - - case 4: - intColVal = row.getUintField<4>(s); - (*f)->store((char*)(&intColVal), strlen((char*)(&intColVal)), (*f)->charset()); - break; - - case 8: - //make sure we don't send strlen off into the weeds... - intColVal = row.getUintField<8>(s); - memcpy(tmp, &intColVal, 8); - tmp[8] = 0; - (*f)->store(tmp, strlen(tmp), (*f)->charset()); - break; - - default: - (*f)->store((const char*)row.getStringPointer(s), row.getStringLength(s), (*f)->charset()); - } - - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - break; - } - - case CalpontSystemCatalog::VARBINARY: - { - if (get_varbin_always_hex(current_thd)) - { - uint32_t l; - const uint8_t* p = row.getVarBinaryField(l, s); - uint32_t ll = l * 2; - boost::scoped_array sca(new char[ll]); - vbin2hex(p, l, sca.get()); - (*f)->store(sca.get(), ll, (*f)->charset()); - } - else - (*f)->store((const char*)row.getVarBinaryField(s), row.getVarBinaryLength(s), (*f)->charset()); - - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - break; - } - - case CalpontSystemCatalog::BIGINT: - { - intColVal = row.getIntField<8>(s); - storeNumericField(f, intColVal, colType); - break; - } - - case CalpontSystemCatalog::UBIGINT: - { - uintColVal = row.getUintField<8>(s); - storeNumericField(f, uintColVal, colType); - break; - } - - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::MEDINT: - { - intColVal = row.getIntField<4>(s); - storeNumericField(f, intColVal, colType); - break; - } - - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UMEDINT: - { - uintColVal = row.getUintField<4>(s); - storeNumericField(f, uintColVal, colType); - break; - } - - case CalpontSystemCatalog::SMALLINT: - { - intColVal = row.getIntField<2>(s); - storeNumericField(f, intColVal, colType); - break; - } - - case CalpontSystemCatalog::USMALLINT: - { - uintColVal = row.getUintField<2>(s); - storeNumericField(f, uintColVal, colType); - break; - } - - case CalpontSystemCatalog::TINYINT: - { - intColVal = row.getIntField<1>(s); - storeNumericField(f, intColVal, colType); - break; - } - - case CalpontSystemCatalog::UTINYINT: - { - uintColVal = row.getUintField<1>(s); - storeNumericField(f, uintColVal, colType); - break; - } - - //In this case, we're trying to load a double output column with float data. This is the - // case when you do sum(floatcol), e.g. - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - float dl = row.getFloatField(s); - - if (dl == std::numeric_limits::infinity()) - continue; - - // bug 3485, reserve enough space for the longest float value - // -3.402823466E+38 to -1.175494351E-38, 0, and - // 1.175494351E-38 to 3.402823466E+38. - (*f)->field_length = 40; - (*f)->store(dl); - - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - double dl = row.getDoubleField(s); - - if (dl == std::numeric_limits::infinity()) - continue; - - if ((*f)->type() == MYSQL_TYPE_NEWDECIMAL) - { - char buf[310]; - // reserve enough space for the longest double value - // -1.7976931348623157E+308 to -2.2250738585072014E-308, 0, and - // 2.2250738585072014E-308 to 1.7976931348623157E+308. - snprintf(buf, 310, "%.18g", dl); - (*f)->store(buf, strlen(buf), (*f)->charset()); - } - else - { - // The server converts dl=-0 to dl=0 in (*f)->store(). - // This happens in the call to truncate_double(). - // This is an unexpected behaviour, so we directly store the - // double value using the lower level float8store() function. - // TODO Remove this when (*f)->store() handles this properly. - (*f)->field_length = 310; - if (dl == 0) - float8store((*f)->ptr,dl); - else - (*f)->store(dl); - } - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - long double dl = row.getLongDoubleField(s); - if (dl == std::numeric_limits::infinity()) - { - continue; - } - - if ((*f)->type() == MYSQL_TYPE_NEWDECIMAL) - { - char buf[310]; - snprintf(buf, 310, "%.20Lg", dl); - (*f)->store(buf, strlen(buf), (*f)->charset()); - } - else - { - // reserve enough space for the longest double value - // -1.7976931348623157E+308 to -2.2250738585072014E-308, 0, and - // 2.2250738585072014E-308 to 1.7976931348623157E+308. - (*f)->field_length = 310; - (*f)->store(static_cast(dl)); - } - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - intColVal = row.getIntField(s); - storeNumericField(f, intColVal, colType); - break; - } - - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - { - Field_blob* f2 = (Field_blob*)*f; - f2->set_ptr(row.getVarBinaryLength(s), (unsigned char*)row.getVarBinaryField(s)); - - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - break; - } - - default: // treat as int64 - { - intColVal = row.getUintField<8>(s); - storeNumericField(f, intColVal, colType); - break; - } + idbassert(0); + (*f)->reset(); + (*f)->set_null(); + } + else + { + // fetch and store data + (*f)->set_notnull(); + datatypes::StoreFieldMariaDB mf(*f, colType); + h->storeValueToField(row, s, &mf); } } @@ -1452,6 +1080,17 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c isFromCol = true; columnAssignmentPtr->fFromCol = true; Item_field* setIt = reinterpret_cast (value); + + // Minor optimization: + // do not perform updates of the form "update t1 set a = a;" + if (!strcmp(item->name.str, setIt->name.str) + && item->table_name.str && setIt->table_name.str && !strcmp(item->table_name.str, setIt->table_name.str) + && item->db_name.str && setIt->db_name.str && !strcmp(item->db_name.str, setIt->db_name.str)) + { + delete columnAssignmentPtr; + continue; + } + string sectableName = string(setIt->table_name.str); if ( setIt->db_name.str ) //derived table @@ -1559,6 +1198,15 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c ci->stats.fQueryType = updateCP->queryType(); } + // Exit early if there is nothing to update + if (colAssignmentListPtr->empty() && + (((thd->lex)->sql_command == SQLCOM_UPDATE) || + ((thd->lex)->sql_command == SQLCOM_UPDATE_MULTI))) + { + ci->affectedRows = 0; + return 0; + } + //save table oid for commit/rollback to use uint32_t sessionID = tid2sid(thd->thread_id); boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); @@ -1595,7 +1243,6 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c TableName* qualifiedTablName = new TableName(); - UpdateSqlStatement updateStmt; //@Bug 2753. To make sure the momory is freed. updateStmt.fColAssignmentListPtr = colAssignmentListPtr; @@ -2286,9 +1933,9 @@ int ha_mcs_impl_discover_existence(const char* schema, const char* name) int ha_mcs_impl_direct_update_delete_rows(bool execute, ha_rows *affected_rows, const std::vector& condStack) { THD* thd = current_thd; - int rc = 0; cal_impl_if::gp_walk_info gwi; gwi.thd = thd; + int rc = 0; if (thd->slave_thread && !get_replication_slave(thd) && ( thd->lex->sql_command == SQLCOM_INSERT || diff --git a/dbcon/mysql/ha_mcs_partition.cpp b/dbcon/mysql/ha_mcs_partition.cpp index 47dff51f1..c057eecdf 100644 --- a/dbcon/mysql/ha_mcs_partition.cpp +++ b/dbcon/mysql/ha_mcs_partition.cpp @@ -68,8 +68,28 @@ using namespace boost; namespace { -const uint8_t ROUND_POS = 0x01; -const uint8_t ROUND_NEG = 0x80; + +datatypes::SimpleValue getStartVal(const datatypes::SessionParam &sp, + const CalpontSystemCatalog::ColType &ct, + const char *val, + datatypes::round_style_t & rfMin) +{ + const datatypes::TypeHandler *h= ct.typeHandler(); + return val ? h->toSimpleValue(sp, ct, val, rfMin) : + h->getMinValueSimple(); +} + + +datatypes::SimpleValue getEndVal(const datatypes::SessionParam &sp, + const CalpontSystemCatalog::ColType &ct, + const char *val, + datatypes::round_style_t & rfMax) +{ + const datatypes::TypeHandler *h= ct.typeHandler(); + return val ? h->toSimpleValue(sp, ct, val, rfMax) : + h->getMaxValueSimple(); +} + //convenience fcn inline uint32_t tid2sid(const uint32_t tid) @@ -100,348 +120,27 @@ void push_warnings(THD* thd, string& warnings) } } + string name(CalpontSystemCatalog::ColType& ct) { - switch (ct.colDataType) - { - case CalpontSystemCatalog::INT: - return "INT"; - - case CalpontSystemCatalog::TINYINT: - return "TINYINT"; - - case CalpontSystemCatalog::MEDINT: - return "MEDINT"; - - case CalpontSystemCatalog::SMALLINT: - return "SMALLINT"; - - case CalpontSystemCatalog::BIGINT: - return "BIGINT"; - - case CalpontSystemCatalog::DATE: - return "DATE"; - - case CalpontSystemCatalog::DATETIME: - return "DATETIME"; - - case CalpontSystemCatalog::TIME: - return "TIME"; - - case CalpontSystemCatalog::TIMESTAMP: - return "TIMESTAMP"; - - case CalpontSystemCatalog::DECIMAL: - return "DECIMAL"; - - case CalpontSystemCatalog::CHAR: - { - ostringstream oss; - oss << "CHAR(" << ct.colWidth << ")"; - return oss.str(); - } - - case CalpontSystemCatalog::VARCHAR: - { - ostringstream oss; - oss << "VARCHAR(" << ct.colWidth << ")"; - return oss.str(); - } - - case CalpontSystemCatalog::FLOAT: - return "FLOAT"; - - case CalpontSystemCatalog::DOUBLE: - return "DOUBLE"; - - case CalpontSystemCatalog::BIT: - return "BIT"; - - case CalpontSystemCatalog::VARBINARY: - return "VARBINARY"; - - case CalpontSystemCatalog::BLOB: - return "BLOB"; - - case CalpontSystemCatalog::TEXT: - return "TEXT"; - - case CalpontSystemCatalog::CLOB: - return "CLOB"; - - case CalpontSystemCatalog::UINT: - return "UINT"; - - case CalpontSystemCatalog::UTINYINT: - return "UTINYINT"; - - case CalpontSystemCatalog::UMEDINT: - return "UMEDINT"; - - case CalpontSystemCatalog::USMALLINT: - return "USMALLINT"; - - case CalpontSystemCatalog::UBIGINT: - return "UBIGINT"; - - case CalpontSystemCatalog::UDECIMAL: - return "UDECIMAL"; - - case CalpontSystemCatalog::UFLOAT: - return "UFLOAT"; - - case CalpontSystemCatalog::UDOUBLE: - return "UDOUBLE"; - - case CalpontSystemCatalog::LONGDOUBLE: - return "LONGDOUBLE"; - - default: - return "Unknown Type"; - } + const datatypes::TypeHandler *h= ct.typeHandler(); + if (!h) + return "Unknown Type"; + return h->print(ct); } + bool CP_type(CalpontSystemCatalog::ColType& ct) { - if (ct.colDataType == CalpontSystemCatalog::INT || - ct.colDataType == CalpontSystemCatalog::TINYINT || - ct.colDataType == CalpontSystemCatalog::MEDINT || - ct.colDataType == CalpontSystemCatalog::SMALLINT || - ct.colDataType == CalpontSystemCatalog::BIGINT || - ct.colDataType == CalpontSystemCatalog::DATE || - ct.colDataType == CalpontSystemCatalog::DATETIME || - ct.colDataType == CalpontSystemCatalog::TIME || - ct.colDataType == CalpontSystemCatalog::TIMESTAMP || - ct.colDataType == CalpontSystemCatalog::DECIMAL || - ct.colDataType == CalpontSystemCatalog::UTINYINT || - ct.colDataType == CalpontSystemCatalog::USMALLINT || - ct.colDataType == CalpontSystemCatalog::UMEDINT || - ct.colDataType == CalpontSystemCatalog::UINT || - ct.colDataType == CalpontSystemCatalog::UBIGINT || - ct.colDataType == CalpontSystemCatalog::UDECIMAL || - (ct.colDataType == CalpontSystemCatalog::CHAR && ct.colWidth <= 8) || - (ct.colDataType == CalpontSystemCatalog::VARCHAR && ct.colWidth <= 7)) - { - return true; - } - - return false; + const datatypes::TypeHandler *h= ct.typeHandler(); + if (!h) + return false; + return h->CP_type(ct); } -const uint64_t ET_DISABLED = 0x0002; -const uint64_t CPINVALID = 0x0004; -struct PartitionInfo -{ - int64_t min; - int64_t max; - uint64_t status; - PartitionInfo(): min((uint64_t)0x8000000000000001ULL), - max((uint64_t) - 0x8000000000000001LL), - status(0) {}; -}; +typedef map PartitionMap; -typedef map PartitionMap; - -const string format(int64_t v, CalpontSystemCatalog::ColType& ct) -{ - ostringstream oss; - - switch (ct.colDataType) - { - case CalpontSystemCatalog::DATE: - oss << DataConvert::dateToString(v); - break; - - case CalpontSystemCatalog::DATETIME: - oss << DataConvert::datetimeToString(v); - break; - - case CalpontSystemCatalog::TIMESTAMP: - oss << DataConvert::timestampToString(v, current_thd->variables.time_zone->get_name()->ptr()); - break; - - case CalpontSystemCatalog::TIME: - oss << DataConvert::timeToString(v); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - { - // swap again to retain the string byte order - uint64_t tmp = uint64ToStr(v); - oss << (char*)(&tmp); - break; - } - - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - if (ct.scale > 0) - { - double d = ((double)(v) / (double)pow((double)10, ct.scale)); - oss << setprecision(ct.scale) << fixed << d; - } - else - { - oss << v; - } - - break; - } - - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - oss << static_cast(v); - break; - - case CalpontSystemCatalog::VARBINARY: - oss << "N/A"; - break; - - default: - oss << v; - break; - } - - return oss.str(); -} - -int64_t IDB_format(char* str, CalpontSystemCatalog::ColType& ct, uint8_t& rf) -{ - int64_t v = 0; - bool pushWarning = false; - rf = 0; - boost::any anyVal = DataConvert::convertColumnData(ct, str, pushWarning, current_thd->variables.time_zone->get_name()->ptr(), false, true, false); - - switch (ct.colDataType) - { - case CalpontSystemCatalog::BIT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::TINYINT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::UTINYINT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::SMALLINT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::USMALLINT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: -#ifdef _MSC_VER - v = boost::any_cast(anyVal); -#else - v = boost::any_cast(anyVal); -#endif - break; - - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::BIGINT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::UBIGINT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::CLOB: - { - string i = boost::any_cast(anyVal); - // bug 1932, pad nulls up to the size of v - i.resize(sizeof(v), 0); - v = uint64ToStr(*((uint64_t*) i.data())); - - if (pushWarning) - rf = ROUND_POS; - } - break; - - case CalpontSystemCatalog::DATE: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::DATETIME: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::TIME: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - v = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) - v = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::FOUR_BYTE) -#ifdef _MSC_VER - v = boost::any_cast(anyVal); - -#else - v = boost::any_cast(anyVal); -#endif - else - v = boost::any_cast(anyVal); - - break; - - default: - break; - } - - if ((ct.colDataType == CalpontSystemCatalog::TINYINT || - ct.colDataType == CalpontSystemCatalog::SMALLINT || - ct.colDataType == CalpontSystemCatalog::MEDINT || - ct.colDataType == CalpontSystemCatalog::INT || - ct.colDataType == CalpontSystemCatalog::BIGINT || - ct.colDataType == CalpontSystemCatalog::DECIMAL || - ct.colDataType == CalpontSystemCatalog::UDECIMAL) && - pushWarning) - { - // get rid of leading white spaces and parentheses - string data(str); - size_t fpos = data.find_first_of(" \t()"); - - while (string::npos != fpos) - { - data.erase(fpos, 1); - fpos = data.find_first_of(" \t()"); - } - - rf = (data[0] == '-') ? ROUND_NEG : ROUND_POS; - } - - return v; -} void parsePartitionString(UDF_ARGS* args, @@ -616,6 +315,35 @@ int processPartition ( SqlStatement* stmt) return rc; } + +static void addPartition(const CalpontSystemCatalog::ColType& ct, + DBRM &em, + const BRM::EMEntry &entry, + PartitionMap &partMap, + const LogicalPartition &logicalPartNum) +{ + const datatypes::TypeHandler *h= ct.typeHandler(); + int state; + datatypes::MinMaxPartitionInfo partInfo= h->getExtentPartitionInfo(ct, em, entry, &state); + + PartitionMap::iterator mapit = partMap.find(logicalPartNum); + if (mapit == partMap.end()) + { + if (state != CP_VALID) + partInfo.set_invalid(); + + partMap[logicalPartNum] = partInfo; + } + else + { + if (mapit->second.is_invalid()) + return; + + mapit->second.MinMaxInfo::operator=(h->widenMinMaxInfo(ct, mapit->second, partInfo)); + } +} + + void partitionByValue_common(UDF_ARGS* args, // input string& errMsg, // output CalpontSystemCatalog::TableName& tableName, // output @@ -627,12 +355,7 @@ void partitionByValue_common(UDF_ARGS* args, // input vector entries; vector::iterator iter; PartitionMap partMap; - PartitionMap::iterator mapit; - int32_t seqNum; string schema, table, column; - CalpontSystemCatalog::ColType ct; - int64_t startVal, endVal; - uint8_t rfMin = 0, rfMax = 0; if (args->arg_count == 5) { @@ -680,7 +403,12 @@ void partitionByValue_common(UDF_ARGS* args, // input CalpontSystemCatalog::TableColName tcn = make_tcn(schema, table, column, lower_case_table_names); csc->identity(CalpontSystemCatalog::FE); OID_t oid = csc->lookupOID(tcn); - ct = csc->colType(oid); + CalpontSystemCatalog::ColType ct = csc->colType(oid); + datatypes::SessionParam sp(current_thd->variables.time_zone->get_name()->ptr()); + datatypes::SimpleValue startVal; + datatypes::SimpleValue endVal; + datatypes::round_style_t rfMin = datatypes::round_style_t::NONE; + datatypes::round_style_t rfMax = datatypes::round_style_t::NONE; if (oid == -1) { @@ -702,162 +430,39 @@ void partitionByValue_common(UDF_ARGS* args, // input if (args->arg_count == 4) { - if (!args->args[2]) - { - if (isUnsigned(ct.colDataType)) - { - startVal = 0; - } - else - { - startVal = numeric_limits::min(); - } - } - else - { - startVal = IDB_format((char*) args->args[2], ct, rfMin); - } - - if (!args->args[3]) - { - if (isUnsigned(ct.colDataType)) - { - endVal = static_cast(numeric_limits::max()); - } - else - { - endVal = numeric_limits::max(); - } - } - else - { - endVal = IDB_format((char*) args->args[3], ct, rfMax); - } + startVal = getStartVal(sp, ct, args->args[2], rfMin); + endVal = getEndVal(sp, ct, args->args[3], rfMax); } else { - if (!args->args[3]) - { - if (isUnsigned(ct.colDataType)) - { - startVal = 0; - } - else - { - startVal = numeric_limits::min(); - } - } - else - { - startVal = IDB_format((char*) args->args[3], ct, rfMin); - } - - if (!args->args[4]) - { - if (isUnsigned(ct.colDataType)) - { - endVal = static_cast(numeric_limits::max()); - } - else - { - endVal = numeric_limits::max(); - } - } - else - { - endVal = IDB_format((char*) args->args[4], ct, rfMax); - } + startVal = getStartVal(sp, ct, args->args[3], rfMin); + endVal = getEndVal(sp, ct, args->args[4], rfMax); } CHECK(em.getExtents(oid, entries, false, false, true)); if (entries.size() > 0) { - LogicalPartition logicalPartNum; for (iter = entries.begin(); iter != entries.end(); ++iter) { - PartitionInfo partInfo; + LogicalPartition logicalPartNum; logicalPartNum.dbroot = (*iter).dbRoot; logicalPartNum.pp = (*iter).partitionNum; logicalPartNum.seg = (*iter).segmentNum; - - if (iter->status == EXTENTOUTOFSERVICE) - partInfo.status |= ET_DISABLED; - - mapit = partMap.find(logicalPartNum); - int state = em.getExtentMaxMin(iter->range.start, partInfo.max, partInfo.min, seqNum); - - // char column order swap - if ((ct.colDataType == CalpontSystemCatalog::CHAR && ct.colWidth <= 8) || - (ct.colDataType == CalpontSystemCatalog::VARCHAR && ct.colWidth <= 7)) - { - partInfo.max = uint64ToStr(partInfo.max); - partInfo.min = uint64ToStr(partInfo.min); - } - - if (mapit == partMap.end()) - { - if (state != CP_VALID) - partInfo.status |= CPINVALID; - - partMap[logicalPartNum] = partInfo; - } - else - { - if (mapit->second.status & CPINVALID) - continue; - - if (isUnsigned(ct.colDataType)) - { - mapit->second.min = - (static_cast(partInfo.min) < static_cast(mapit->second.min) ? partInfo.min : mapit->second.min); - mapit->second.max = - (static_cast(partInfo.max) > static_cast(mapit->second.max) ? partInfo.max : mapit->second.max); - } - else - { - mapit->second.min = (partInfo.min < mapit->second.min ? partInfo.min : mapit->second.min); - mapit->second.max = (partInfo.max > mapit->second.max ? partInfo.max : mapit->second.max); - } - } + addPartition(ct, em, *iter, partMap, logicalPartNum); } // check col value range - for (mapit = partMap.begin(); mapit != partMap.end(); ++mapit) + for (PartitionMap::iterator mapit = partMap.begin(); mapit != partMap.end(); ++mapit) { + if (mapit->second.is_invalid()) + continue; + + const datatypes::TypeHandler *h= ct.typeHandler(); // @bug 4595. check empty/null case - if (isUnsigned(ct.colDataType)) - { - if (!(mapit->second.status & CPINVALID) && - static_cast(mapit->second.min) >= static_cast(startVal) && - static_cast(mapit->second.max) <= static_cast(endVal) && - !(static_cast(mapit->second.min) == numeric_limits::max() && - static_cast(mapit->second.max == 0))) - { - if (rfMin == ROUND_POS && mapit->second.min == startVal) - continue; - - if (rfMax == ROUND_NEG && mapit->second.max == endVal) - continue; - - partSet.insert(mapit->first); - } - } - else - { - if (!(mapit->second.status & CPINVALID) && mapit->second.min >= startVal && mapit->second.max <= endVal && - !(mapit->second.min == numeric_limits::max() && mapit->second.max == numeric_limits::min())) - { - if (rfMin == ROUND_POS && mapit->second.min == startVal) - continue; - - if (rfMax == ROUND_NEG && mapit->second.max == endVal) - continue; - - partSet.insert(mapit->first); - } - } + if (h->isSuitablePartition(ct, mapit->second, startVal, rfMin, endVal, rfMax)) + partSet.insert(mapit->first); } } } @@ -1007,7 +612,6 @@ extern "C" vector::iterator iter; vector::iterator end; PartitionMap partMap; - int32_t seqNum; string schema, table, column; CalpontSystemCatalog::ColType ct; string errMsg; @@ -1066,40 +670,10 @@ extern "C" for (; iter != end; ++iter) { - PartitionInfo partInfo; logicalPartNum.dbroot = (*iter).dbRoot; logicalPartNum.pp = (*iter).partitionNum; logicalPartNum.seg = (*iter).segmentNum; - - if (iter->status == EXTENTOUTOFSERVICE) - partInfo.status |= ET_DISABLED; - - mapit = partMap.find(logicalPartNum); - int state = em.getExtentMaxMin(iter->range.start, partInfo.max, partInfo.min, seqNum); - - // char column order swap for compare - if ((ct.colDataType == CalpontSystemCatalog::CHAR && ct.colWidth <= 8) || - (ct.colDataType == CalpontSystemCatalog::VARCHAR && ct.colWidth <= 7)) - { - partInfo.max = uint64ToStr(partInfo.max); - partInfo.min = uint64ToStr(partInfo.min); - } - - if (mapit == partMap.end()) - { - if (state != CP_VALID) - partInfo.status |= CPINVALID; - - partMap[logicalPartNum] = partInfo; - } - else - { - if (mapit->second.status & CPINVALID) - continue; - - mapit->second.min = (partInfo.min < mapit->second.min ? partInfo.min : mapit->second.min); - mapit->second.max = (partInfo.max > mapit->second.max ? partInfo.max : mapit->second.max); - } + addPartition(ct, em, *iter, partMap, logicalPartNum); } } } @@ -1116,22 +690,14 @@ extern "C" return result; } + const datatypes::TypeHandler *h= ct.typeHandler(); + uint8_t valueCharLength= h->PartitionValueCharLength(ct); ostringstream output; output.setf(ios::left, ios::adjustfield); + output << setw(10) << "Part#" - << setw(30) << "Min" - << setw(30) << "Max" << "Status"; - - int64_t maxLimit = numeric_limits::max(); - int64_t minLimit = numeric_limits::min(); - - // char column order swap for compare in subsequent loop - if ((ct.colDataType == CalpontSystemCatalog::CHAR && ct.colWidth <= 8) || - (ct.colDataType == CalpontSystemCatalog::VARCHAR && ct.colWidth <= 7)) - { - maxLimit = uint64ToStr(maxLimit); - minLimit = uint64ToStr(minLimit); - } + << setw(valueCharLength) << "Min" + << setw(valueCharLength) << "Max" << "Status"; PartitionMap::const_iterator partIt; @@ -1141,30 +707,18 @@ extern "C" oss << partIt->first; output << "\n " << setw(10) << oss.str(); - if (partIt->second.status & CPINVALID) + if (partIt->second.is_invalid()) { - output << setw(30) << "N/A" << setw(30) << "N/A"; + output << setw(valueCharLength) << "N/A" + << setw(valueCharLength) << "N/A"; } else { - if ((isUnsigned(ct.colDataType))) - { - if (static_cast(partIt->second.min) == numeric_limits::max() - && static_cast(partIt->second.max) == numeric_limits::min()) - output << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; - else - output << setw(30) << format(partIt->second.min, ct) << setw(30) << format(partIt->second.max, ct); - } - else - { - if (partIt->second.min == maxLimit && partIt->second.max == minLimit) - output << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; - else - output << setw(30) << format(partIt->second.min, ct) << setw(30) << format(partIt->second.max, ct); - } + const datatypes::TypeHandler *h= ct.typeHandler(); + oss << h->formatPartitionInfo(ct, partIt->second); } - if (partIt->second.status & ET_DISABLED) + if (partIt->second.is_disabled()) output << "Disabled"; else output << "Enabled"; @@ -1705,6 +1259,9 @@ extern "C" delete initid->ptr; } + + + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -1718,12 +1275,14 @@ extern "C" vector::iterator end; PartitionMap partMap; PartitionMap::iterator mapit; - int32_t seqNum; string schema, table, column; CalpontSystemCatalog::ColType ct; string errMsg; - int64_t startVal, endVal; - uint8_t rfMin = 0, rfMax = 0; + datatypes::SessionParam sp(current_thd->variables.time_zone->get_name()->ptr()); + datatypes::SimpleValue startVal; + datatypes::SimpleValue endVal; + datatypes::round_style_t rfMin = datatypes::round_style_t::NONE; + datatypes::round_style_t rfMax = datatypes::round_style_t::NONE; try { @@ -1779,71 +1338,13 @@ extern "C" if (args->arg_count == 4) { - if (!args->args[2]) - { - if (isUnsigned(ct.colDataType)) - { - startVal = 0; - } - else - { - startVal = numeric_limits::min(); - } - } - else - { - startVal = IDB_format((char*) args->args[2], ct, rfMin); - } - - if (!args->args[3]) - { - if (isUnsigned(ct.colDataType)) - { - endVal = static_cast(numeric_limits::max()); - } - else - { - endVal = numeric_limits::max(); - } - } - else - { - endVal = IDB_format((char*) args->args[3], ct, rfMax); - } + startVal= getStartVal(sp, ct, args->args[2], rfMin); + endVal= getEndVal(sp, ct, args->args[3], rfMax); } else { - if (!args->args[3]) - { - if (isUnsigned(ct.colDataType)) - { - startVal = 0; - } - else - { - startVal = numeric_limits::min(); - } - } - else - { - startVal = IDB_format((char*) args->args[3], ct, rfMin); - } - - if (!args->args[4]) - { - if (isUnsigned(ct.colDataType)) - { - endVal = static_cast(numeric_limits::max()); - } - else - { - endVal = numeric_limits::max(); - } - } - else - { - endVal = IDB_format((char*) args->args[4], ct, rfMax); - } + startVal= getStartVal(sp, ct, args->args[3], rfMin); + endVal= getEndVal(sp, ct, args->args[4], rfMax); } CHECK(em.getExtents(oid, entries, false, false, true)); @@ -1856,50 +1357,10 @@ extern "C" for (; iter != end; ++iter) { - PartitionInfo partInfo; logicalPartNum.dbroot = (*iter).dbRoot; logicalPartNum.pp = (*iter).partitionNum; logicalPartNum.seg = (*iter).segmentNum; - - if (iter->status == EXTENTOUTOFSERVICE) - partInfo.status |= ET_DISABLED; - - mapit = partMap.find(logicalPartNum); - int state = em.getExtentMaxMin(iter->range.start, partInfo.max, partInfo.min, seqNum); - - // char column order swap - if ((ct.colDataType == CalpontSystemCatalog::CHAR && ct.colWidth <= 8) || - (ct.colDataType == CalpontSystemCatalog::VARCHAR && ct.colWidth <= 7)) - { - partInfo.max = uint64ToStr(partInfo.max); - partInfo.min = uint64ToStr(partInfo.min); - } - - if (mapit == partMap.end()) - { - if (state != CP_VALID) - partInfo.status |= CPINVALID; - - partMap[logicalPartNum] = partInfo; - } - else - { - if (mapit->second.status & CPINVALID) - continue; - - if (isUnsigned(ct.colDataType)) - { - mapit->second.min = - (static_cast(partInfo.min) < static_cast(mapit->second.min) ? partInfo.min : mapit->second.min); - mapit->second.max = - (static_cast(partInfo.max) > static_cast(mapit->second.max) ? partInfo.max : mapit->second.max); - } - else - { - mapit->second.min = (partInfo.min < mapit->second.min ? partInfo.min : mapit->second.min); - mapit->second.max = (partInfo.max > mapit->second.max ? partInfo.max : mapit->second.max); - } - } + addPartition(ct, em, *iter, partMap, logicalPartNum); } } } @@ -1931,59 +1392,41 @@ extern "C" for (mapit = partMap.begin(); mapit != partMap.end(); ++mapit) { - // @bug 4595. check empty/null case - if (!(mapit->second.status & CPINVALID) && mapit->second.min >= startVal && mapit->second.max <= endVal && - !(mapit->second.min == numeric_limits::max() && mapit->second.max == numeric_limits::min())) + const datatypes::TypeHandler *h= ct.typeHandler(); + uint8_t valueCharLength= h->PartitionValueCharLength(ct); + if (mapit->second.is_invalid()) { - if (rfMin == ROUND_POS && mapit->second.min == startVal) - continue; - - if (rfMax == ROUND_NEG && mapit->second.max == endVal) - continue; - - // print header - if (noPartFound) - { - output.setf(ios::left, ios::adjustfield); - output << setw(10) << "Part#" - << setw(30) << "Min" - << setw(30) << "Max" << "Status"; - } - - noPartFound = false; - - // print part info - ostringstream oss; - oss << mapit->first; - output << "\n " << setw(10) << oss.str(); - - if (mapit->second.status & CPINVALID) - { - output << setw(30) << "N/A" << setw(30) << "N/A"; - } - else - { - if ((isUnsigned(ct.colDataType))) - { - if (static_cast(mapit->second.min) > static_cast(mapit->second.max)) - output << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; - else - output << setw(30) << format(mapit->second.min, ct) << setw(30) << format(mapit->second.max, ct); - } - else - { - if (mapit->second.min > mapit->second.max) - output << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; - else - output << setw(30) << format(mapit->second.min, ct) << setw(30) << format(mapit->second.max, ct); - } - } - - if (mapit->second.status & ET_DISABLED) - output << "Disabled"; - else - output << "Enabled"; + output << setw(valueCharLength) << "N/A" + << setw(valueCharLength) << "N/A"; + continue; } + // @bug 4595. check empty/null case + string tmp= h->PrintPartitionValue(ct, mapit->second, + startVal, rfMin, + endVal, rfMax); + if (tmp == "") + continue; + + // print header + if (noPartFound) + { + output.setf(ios::left, ios::adjustfield); + output << setw(10) << "Part#" + << setw(valueCharLength) << "Min" + << setw(valueCharLength) << "Max" << "Status"; + noPartFound = false; + } + + // print part info + ostringstream oss; + oss << mapit->first; + output << "\n " << setw(10) << oss.str() << tmp; + + if (mapit->second.is_disabled()) + output << "Disabled"; + else + output << "Enabled"; + } if (noPartFound) diff --git a/dbcon/mysql/ha_mcs_sysvars.cpp b/dbcon/mysql/ha_mcs_sysvars.cpp index 02f89ab50..fc037005b 100644 --- a/dbcon/mysql/ha_mcs_sysvars.cpp +++ b/dbcon/mysql/ha_mcs_sysvars.cpp @@ -159,6 +159,15 @@ static MYSQL_THDVAR_BOOL( 0 ); +static MYSQL_THDVAR_BOOL( + decimal_overflow_check, + PLUGIN_VAR_NOCMDARG, + "Enable/disable for ColumnStore to check for overflow in arithmetic operation.", + NULL, + NULL, + 0 +); + static MYSQL_THDVAR_BOOL( ordered_only, PLUGIN_VAR_NOCMDARG, @@ -353,6 +362,7 @@ st_mysql_sys_var* mcs_system_variables[] = MYSQL_SYSVAR(diskjoin_bucketsize), MYSQL_SYSVAR(um_mem_limit), MYSQL_SYSVAR(double_for_decimal_math), + MYSQL_SYSVAR(decimal_overflow_check), MYSQL_SYSVAR(local_query), MYSQL_SYSVAR(use_import_for_batchinsert), MYSQL_SYSVAR(import_for_batchinsert_delimiter), @@ -557,6 +567,15 @@ void set_double_for_decimal_math(THD* thd, bool value) THDVAR(thd, double_for_decimal_math) = value; } +bool get_decimal_overflow_check(THD* thd) +{ + return ( thd == NULL ) ? false : THDVAR(thd, decimal_overflow_check); +} +void set_decimal_overflow_check(THD* thd, bool value) +{ + THDVAR(thd, decimal_overflow_check) = value; +} + ulong get_local_query(THD* thd) { return ( thd == NULL ) ? 0 : THDVAR(thd, local_query); diff --git a/dbcon/mysql/ha_mcs_sysvars.h b/dbcon/mysql/ha_mcs_sysvars.h index 1bb01307f..b8e3884b7 100644 --- a/dbcon/mysql/ha_mcs_sysvars.h +++ b/dbcon/mysql/ha_mcs_sysvars.h @@ -101,6 +101,9 @@ void set_varbin_always_hex(THD* thd, bool value); bool get_double_for_decimal_math(THD* thd); void set_double_for_decimal_math(THD* thd, bool value); +bool get_decimal_overflow_check(THD* thd); +void set_decimal_overflow_check(THD* thd, bool value); + ulong get_local_query(THD* thd); void set_local_query(THD* thd, ulong value); diff --git a/dbcon/mysql/is_columnstore_extents.cpp b/dbcon/mysql/is_columnstore_extents.cpp index 188ce31dc..f1ab1f3a7 100644 --- a/dbcon/mysql/is_columnstore_extents.cpp +++ b/dbcon/mysql/is_columnstore_extents.cpp @@ -27,6 +27,8 @@ #include "dbrm.h" #include "objectidmanager.h" #include "is_columnstore.h" +#include "mcs_decimal.h" +#include "dataconvert.h" // Required declaration as it isn't in a MairaDB include bool schema_table_store_record(THD* thd, TABLE* table); @@ -37,8 +39,10 @@ ST_FIELD_INFO is_columnstore_extents_fields[] = Show::Column("OBJECT_TYPE", Show::Varchar(64), NOT_NULL), // 1 Show::Column("LOGICAL_BLOCK_START", Show::SLonglong(0), NOT_NULL), // 2 Show::Column("LOGICAL_BLOCK_END", Show::SLonglong(0), NOT_NULL), // 3 - Show::Column("MIN_VALUE", Show::SLonglong(0), NULLABLE), // 4 - Show::Column("MAX_VALUE", Show::SLonglong(0), NULLABLE), // 5 + // length=3800 here because sql/sql_i_s.h sets + // decimal_precision() as (length / 100) % 100). Not sure why. + Show::Column("MIN_VALUE", Show::Decimal(3800), NULLABLE), // 4 + Show::Column("MAX_VALUE", Show::Decimal(3800), NULLABLE), // 5 Show::Column("WIDTH", Show::ULong(0), NOT_NULL), // 6 Show::Column("DBROOT", Show::ULong(0), NOT_NULL), // 7 Show::Column("PARTITION_ID", Show::ULong(0), NOT_NULL), // 8 @@ -75,26 +79,59 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM* emp, TABLE* table, THD* th { table->field[1]->store("Column", strlen("Column"), cs); - if (iter->partition.cprange.lo_val == std::numeric_limits::max() || - iter->partition.cprange.lo_val <= (std::numeric_limits::min() + 2)) + if (iter->colWid != datatypes::MAXDECIMALWIDTH) { - table->field[4]->set_null(); - } - else - { - table->field[4]->set_notnull(); - table->field[4]->store(iter->partition.cprange.lo_val); - } + if (iter->partition.cprange.loVal == std::numeric_limits::max() || + iter->partition.cprange.loVal <= (std::numeric_limits::min() + 1)) + { + table->field[4]->set_null(); + } + else + { + table->field[4]->set_notnull(); + table->field[4]->store(iter->partition.cprange.loVal); + } - if (iter->partition.cprange.hi_val == std::numeric_limits::max() || - iter->partition.cprange.hi_val <= (std::numeric_limits::min() + 2)) - { - table->field[5]->set_null(); + if (iter->partition.cprange.hiVal <= (std::numeric_limits::min() + 1)) + { + table->field[5]->set_null(); + } + else + { + table->field[5]->set_notnull(); + table->field[5]->store(iter->partition.cprange.hiVal); + } } else { - table->field[5]->set_notnull(); - table->field[5]->store(iter->partition.cprange.hi_val); + if (iter->partition.cprange.bigLoVal == utils::maxInt128 || + iter->partition.cprange.bigLoVal <= (utils::minInt128 + 1)) + { + table->field[4]->set_null(); + } + else + { + table->field[4]->set_notnull(); + std::string decAsAStr = datatypes::TSInt128(iter->partition.cprange.bigLoVal) + .toString(); + table->field[4]->store(decAsAStr.c_str(), + decAsAStr.length(), + table->field[4]->charset()); + } + + if (iter->partition.cprange.bigHiVal <= (utils::minInt128 + 1)) + { + table->field[5]->set_null(); + } + else + { + table->field[5]->set_notnull(); + std::string decAsAStr = datatypes::TSInt128(iter->partition.cprange.bigHiVal) + .toString(); + table->field[5]->store(decAsAStr.c_str(), + decAsAStr.length(), + table->field[5]->charset()); + } } table->field[6]->store(iter->colWid); diff --git a/debian/mariadb-plugin-columnstore.install b/debian/mariadb-plugin-columnstore.install index 3f4cc792b..b74b47b8f 100644 --- a/debian/mariadb-plugin-columnstore.install +++ b/debian/mariadb-plugin-columnstore.install @@ -121,6 +121,7 @@ usr/lib/*/libwindowfunction.so usr/lib/*/libwriteengine.so usr/lib/*/libwriteengineclient.so usr/lib/*/libwriteengineredistribute.so +usr/lib/*/libdatatypes.so usr/lib/mysql/plugin/ha_columnstore.so usr/lib/mysql/plugin/libregr_mysql.so usr/lib/mysql/plugin/libudf_mysql.so diff --git a/primitives/linux-port/CMakeLists.txt b/primitives/linux-port/CMakeLists.txt index 08ca261f8..569169d12 100644 --- a/primitives/linux-port/CMakeLists.txt +++ b/primitives/linux-port/CMakeLists.txt @@ -16,3 +16,9 @@ target_link_libraries(processor ${NETSNMP_LIBRARIES}) INSTALL (TARGETS processor DESTINATION ${ENGINE_LIBDIR}) +#if (WITH_PP_SCAN_UT) +# add_executable(pp_scan_unittest pp-scan-unittest.cpp) +# target_link_libraries(pp_scan_unittest ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) +# install(TARGETS pp_scan_unittest DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) +#endif() + diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index a8b3dc838..72a0d747f 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -1,4 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2016-2020 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -40,6 +41,8 @@ using namespace boost; #include "stats.h" #include "primproc.h" #include "dataconvert.h" +#include "mcs_decimal.h" + using namespace logging; using namespace dbbc; using namespace primitives; @@ -273,6 +276,22 @@ inline bool colStrCompare_(uint64_t val1, uint64_t val2, uint8_t COP, const idb_ template inline bool isEmptyVal(uint8_t type, const uint8_t* val8); +template<> +inline bool isEmptyVal<32>(uint8_t type, const uint8_t* ival) // For BINARY +{ + std::cout << __func__ << " WARNING!!! Not implemented for 32 byte data types." << std::endl; + return false; +} + +template<> +inline bool isEmptyVal<16>(uint8_t type, const uint8_t* ival) // For BINARY +{ + const int128_t* val = reinterpret_cast(ival); + // Wide-DECIMAL supplies a universal NULL/EMPTY magics for all 16 byte + // data types. + return *val == datatypes::Decimal128Empty; +} + template<> inline bool isEmptyVal<8>(uint8_t type, const uint8_t* ival) { @@ -394,6 +413,24 @@ inline bool isEmptyVal<1>(uint8_t type, const uint8_t* ival) template inline bool isNullVal(uint8_t type, const uint8_t* val8); +template<> +inline bool isNullVal<16>(uint8_t type, const uint8_t* ival) +{ + const int128_t* val = reinterpret_cast(ival); + // Wide-DECIMAL supplies a universal NULL/EMPTY magics for all 16 byte + // data types. + return *val == datatypes::Decimal128Null; +} + +template<> +inline bool isNullVal<32>(uint8_t type, const uint8_t* ival) +{ + + std::cout << __func__ << " WARNING!!! Not implemented for 32 byte data types." +<< std::endl; + return false; +} + template<> inline bool isNullVal<8>(uint8_t type, const uint8_t* ival) { @@ -521,6 +558,9 @@ inline bool isNullVal(uint32_t length, uint8_t type, const uint8_t* val8) { switch (length) { + case 16: + return isNullVal<16>(type, val8); + case 8: return isNullVal<8>(type, val8); @@ -533,7 +573,8 @@ inline bool isNullVal(uint32_t length, uint8_t type, const uint8_t* val8) case 1: return isNullVal<1>(type, val8); }; - + std::cout << __func__ << " WARNING!!! Not implemented for " << length << " bytes data types." << std::endl; + return false; } @@ -576,7 +617,7 @@ inline bool isMinMaxValid(const NewColRequestHeader* in) case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - return (in->DataSize <= 8); + return (in->DataSize <= datatypes::MAXDECIMALWIDTH); default: return false; @@ -597,8 +638,6 @@ inline string fixChar(int64_t intval) inline bool colCompare(int64_t val1, int64_t val2, uint8_t COP, uint8_t rf, int type, uint8_t width, const idb_regex_t& regex, bool isNull = false) { -// cout << "comparing " << hex << val1 << " to " << val2 << endl; - if (COMPARE_NIL == COP) return false; //@bug 425 added isNull condition @@ -647,9 +686,22 @@ inline bool colCompare(int64_t val1, int64_t val2, uint8_t COP, uint8_t rf, int } } +inline bool colCompare(int128_t val1, int128_t val2, uint8_t COP, uint8_t rf, int type, uint8_t width, bool isNull = false) +{ + if (COMPARE_NIL == COP) return false; + + /* isNullVal should work on the normalized value on little endian machines */ + bool val2Null = isNullVal(width, type, (uint8_t*) &val2); + + if (isNull == val2Null || (val2Null && COP == COMPARE_NE)) + return colCompare_(val1, val2, COP, rf); + else + return false; +} + inline bool colCompareUnsigned(uint64_t val1, uint64_t val2, uint8_t COP, uint8_t rf, int type, uint8_t width, const idb_regex_t& regex, bool isNull = false) { -// cout << "comparing unsigned" << hex << val1 << " to " << val2 << endl; + // cout << "comparing unsigned" << hex << val1 << " to " << val2 << endl; if (COMPARE_NIL == COP) return false; @@ -703,7 +755,19 @@ inline void store(const NewColRequestHeader* in, switch (in->DataSize) { + case 32: + std::cout << __func__ << " WARNING!!! Not implemented for 32 byte data types." << std::endl; + break; + + case 16: + ptr2 += (rid << 4); + memcpy(ptr1, ptr2, 16); + break; + default: + std::cout << __func__ << " WARNING!!! unspecified column width." << std::endl; + // fallthrough + case 8: ptr2 += (rid << 3); memcpy(ptr1, ptr2, 8); @@ -724,7 +788,6 @@ inline void store(const NewColRequestHeader* in, memcpy(ptr1, ptr2, 1); break; } - *written += in->DataSize; } @@ -811,6 +874,66 @@ inline uint64_t nextUnsignedColValue(int type, return -1; } } +template +inline uint8_t* nextBinColValue(int type, + const uint16_t* ridArray, + int NVALS, + int* index, + bool* done, + bool* isNull, + bool* isEmpty, + uint16_t* rid, + uint8_t OutputType, uint8_t* val8, unsigned itemsPerBlk) +{ + if (ridArray == NULL) + { + while (static_cast(*index) < itemsPerBlk && + isEmptyVal(type, &val8[*index * W]) && + (OutputType & OT_RID)) + { + (*index)++; + } + + + if (static_cast(*index) >= itemsPerBlk) + { + *done = true; + return NULL; + } + *rid = (*index)++; + } + else + { + while (*index < NVALS && + isEmptyVal(type, &val8[ridArray[*index] * W])) + { + (*index)++; + } + + if (*index >= NVALS) + { + *done = true; + return NULL; + } + *rid = ridArray[(*index)++]; + } + + uint32_t curValueOffset = *rid * W; + + *isNull = isNullVal(type, &val8[curValueOffset]); + *isEmpty = isEmptyVal(type, &val8[curValueOffset]); + //cout << "nextColBinValue " << *index << " rowid " << *rid << endl; + // at this point, nextRid is the index to return, and index is... + // if RIDs are not specified, nextRid + 1, + // if RIDs are specified, it's the next index in the rid array. + return &val8[curValueOffset]; + +#ifdef PRIM_DEBUG + throw logic_error("PrimitiveProcessor::nextColBinValue() bad width"); +#endif + return NULL; +} + template inline int64_t nextColValue(int type, @@ -1426,6 +1549,225 @@ inline void p_Col_ridArray(NewColRequestHeader* in, #endif } +// There are number of hardcoded type-dependant objects +// that effectively makes this template int128-based only. +// Use type based template method for Min,Max values. +// prestored_set_128 must be a template with a type arg. +template +inline void p_Col_bin_ridArray(NewColRequestHeader* in, + NewColResultHeader* out, + unsigned outSize, + unsigned* written, int* block, Stats* fStatsPtr, unsigned itemsPerBlk, + boost::shared_ptr parsedColumnFilter) +{ + uint16_t* ridArray = 0; + uint8_t* in8 = reinterpret_cast(in); + const uint8_t filterSize = sizeof(uint8_t) + sizeof(uint8_t) + W; + idb_regex_t placeholderRegex; + placeholderRegex.used = false; + + if (in->NVALS > 0) + ridArray = reinterpret_cast(&in8[sizeof(NewColRequestHeader) + + (in->NOPS * filterSize)]); + + if (ridArray && 1 == in->sort ) + { + qsort(ridArray, in->NVALS, sizeof(uint16_t), compareBlock); + + if (fStatsPtr) +#ifdef _MSC_VER + fStatsPtr->markEvent(in->LBID, GetCurrentThreadId(), in->hdr.SessionID, 'O'); + +#else + fStatsPtr->markEvent(in->LBID, pthread_self(), in->hdr.SessionID, 'O'); +#endif + } + + // Set boolean indicating whether to capture the min and max values. + out->ValidMinMax = isMinMaxValid(in); + + if (out->ValidMinMax) + { + // Assume that isUnsigned returns true for 8-bytes DTs only + if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType)) + { + out->Min = -1; + out->Max = 0; + } + else + { + out->Min = datatypes::Decimal::maxInt128; + out->Max = datatypes::Decimal::minInt128; + } + } + else + { + out->Min = 0; + out->Max = 0; + } + + typedef char binWtype [W]; + + const ColArgs* args = NULL; + binWtype* bval; + int nextRidIndex = 0, argIndex = 0; + bool done = false, cmp = false, isNull = false, isEmpty = false; + uint16_t rid = 0; + prestored_set_t_128::const_iterator it; + + binWtype* argVals = (binWtype*)alloca(in->NOPS * W); + uint8_t* std_cops = (uint8_t*)alloca(in->NOPS * sizeof(uint8_t)); + uint8_t* std_rfs = (uint8_t*)alloca(in->NOPS * sizeof(uint8_t)); + uint8_t* cops = NULL; + uint8_t* rfs = NULL; + + scoped_array std_regex; + idb_regex_t* regex = NULL; + + // no pre-parsed column filter is set, parse the filter in the message + if (parsedColumnFilter.get() == NULL) { + std_regex.reset(new idb_regex_t[in->NOPS]); + regex = &(std_regex[0]); + + cops = std_cops; + rfs = std_rfs; + + for (argIndex = 0; argIndex < in->NOPS; argIndex++) { + args = reinterpret_cast (&in8[sizeof (NewColRequestHeader) + + (argIndex * filterSize)]); + cops[argIndex] = args->COP; + rfs[argIndex] = args->rf; + + memcpy(argVals[argIndex],args->val, W); + regex[argIndex].used = false; + } + } + // we have a pre-parsed filter, and it's in the form of op and value arrays + else if (parsedColumnFilter->columnFilterMode == TWO_ARRAYS) + { + argVals = (binWtype*) parsedColumnFilter->prestored_argVals128.get(); + cops = parsedColumnFilter->prestored_cops.get(); + rfs = parsedColumnFilter->prestored_rfs.get(); + regex = parsedColumnFilter->prestored_regex.get(); + } + + // else we have a pre-parsed filter, and it's an unordered set for quick == comparisons + + bval = (binWtype*)nextBinColValue(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull, + &isEmpty, &rid, in->OutputType, reinterpret_cast(block), itemsPerBlk); + + T val; + + while (!done) + { + val = *reinterpret_cast(bval); + + if (cops == NULL) // implies parsedColumnFilter && columnFilterMode == SET + { + /* bug 1920: ignore NULLs in the set and in the column data */ + if (!(isNull && in->BOP == BOP_AND)) + { + + it = parsedColumnFilter->prestored_set_128->find(val); + + + if (in->BOP == BOP_OR) + { + // assume COP == COMPARE_EQ + if (it != parsedColumnFilter->prestored_set_128->end()) + { + store(in, out, outSize, written, rid, reinterpret_cast(block)); + } + } + else if (in->BOP == BOP_AND) + { + // assume COP == COMPARE_NE + if (it == parsedColumnFilter->prestored_set_128->end()) + { + store(in, out, outSize, written, rid, reinterpret_cast(block)); + } + } + } + } + else + { + for (argIndex = 0; argIndex < in->NOPS; argIndex++) + { + T filterVal = *reinterpret_cast(argVals[argIndex]); + + cmp = colCompare(val, filterVal, cops[argIndex], + rfs[argIndex], in->DataType, W, isNull); + + if (in->NOPS == 1) + { + if (cmp == true) + { + store(in, out, outSize, written, rid, reinterpret_cast(block)); + } + break; + } + else if (in->BOP == BOP_AND && cmp == false) + { + break; + } + else if (in->BOP == BOP_OR && cmp == true) + { + store(in, out, outSize, written, rid, reinterpret_cast(block)); + break; + } + } + + if ((argIndex == in->NOPS && in->BOP == BOP_AND) || in->NOPS == 0) + { + store(in, out, outSize, written, rid, reinterpret_cast(block)); + } + } + + // Set the min and max if necessary. Ignore nulls. + if (out->ValidMinMax && !isNull && !isEmpty) + { + + if (in->DataType == CalpontSystemCatalog::CHAR || in->DataType == CalpontSystemCatalog::VARCHAR) + { + // !!! colCompare is overloaded with int128_t only yet. + if (colCompare(out->Min, val, COMPARE_GT, false, in->DataType, W, placeholderRegex)) + { + out->Min = val; + } + + if (colCompare(out->Max, val, COMPARE_LT, false, in->DataType, W, placeholderRegex)) + { + out->Max = val; + } + } + else + { + if (out->Min > val) + { + out->Min = val; + } + + if (out->Max < val) + { + out->Max = val; + } + } + } + + bval = (binWtype*)nextBinColValue(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull, + &isEmpty, &rid, in->OutputType, reinterpret_cast(block), itemsPerBlk); + + } + + if (fStatsPtr) +#ifdef _MSC_VER + fStatsPtr->markEvent(in->LBID, GetCurrentThreadId(), in->hdr.SessionID, 'K'); + +#else + fStatsPtr->markEvent(in->LBID, pthread_self(), in->hdr.SessionID, 'K'); +#endif +} + } //namespace anon namespace primitives @@ -1492,6 +1834,14 @@ void PrimitiveProcessor::p_Col(NewColRequestHeader* in, NewColResultHeader* out, p_Col_ridArray<1>(in, out, outSize, written, block, fStatsPtr, itemsPerBlk, parsedColumnFilter); break; + case 16: + p_Col_bin_ridArray<16, int128_t>(in, out, outSize, written, block, fStatsPtr, itemsPerBlk, parsedColumnFilter); + break; + + case 32: + std::cout << __func__ << " WARNING!!! Not implemented for 32 byte data types." << std::endl; + // fallthrough + default: idbassert(0); break; @@ -1521,7 +1871,10 @@ boost::shared_ptr parseColumnFilter ret.reset(new ParsedColumnFilter()); ret->columnFilterMode = TWO_ARRAYS; - ret->prestored_argVals.reset(new int64_t[filterCount]); + if (colWidth == datatypes::MAXDECIMALWIDTH) + ret->prestored_argVals128.reset(new int128_t[filterCount]); + else + ret->prestored_argVals.reset(new int64_t[filterCount]); ret->prestored_cops.reset(new uint8_t[filterCount]); ret->prestored_rfs.reset(new uint8_t[filterCount]); ret->prestored_regex.reset(new idb_regex_t[filterCount]); @@ -1614,6 +1967,13 @@ boost::shared_ptr parseColumnFilter case 8: ret->prestored_argVals[argIndex] = *reinterpret_cast(args->val); break; + + case 16: + { + datatypes::TSInt128::assignPtrPtr(&(ret->prestored_argVals128[argIndex]), + args->val); + break; + } } } @@ -1642,12 +2002,24 @@ boost::shared_ptr parseColumnFilter if (convertToSet) { ret->columnFilterMode = UNORDERED_SET; - ret->prestored_set.reset(new prestored_set_t()); + if (colWidth == datatypes::MAXDECIMALWIDTH) + { + ret->prestored_set_128.reset(new prestored_set_t_128()); - // @bug 2584, use COMPARE_NIL for "= null" to allow "is null" in OR expression - for (argIndex = 0; argIndex < filterCount; argIndex++) - if (ret->prestored_rfs[argIndex] == 0) - ret->prestored_set->insert(ret->prestored_argVals[argIndex]); + // @bug 2584, use COMPARE_NIL for "= null" to allow "is null" in OR expression + for (argIndex = 0; argIndex < filterCount; argIndex++) + if (ret->prestored_rfs[argIndex] == 0) + ret->prestored_set_128->insert(ret->prestored_argVals128[argIndex]); + } + else + { + ret->prestored_set.reset(new prestored_set_t()); + + // @bug 2584, use COMPARE_NIL for "= null" to allow "is null" in OR expression + for (argIndex = 0; argIndex < filterCount; argIndex++) + if (ret->prestored_rfs[argIndex] == 0) + ret->prestored_set->insert(ret->prestored_argVals[argIndex]); + } } return ret; diff --git a/primitives/linux-port/tdriver.cpp b/primitives/linux-port/pp-scan-unittest.cpp similarity index 96% rename from primitives/linux-port/tdriver.cpp rename to primitives/linux-port/pp-scan-unittest.cpp index 0d7a00c9b..0d80141a5 100644 --- a/primitives/linux-port/tdriver.cpp +++ b/primitives/linux-port/pp-scan-unittest.cpp @@ -39,9 +39,9 @@ #include #include "primitiveprocessor.h" +using namespace primitives; using namespace std; - int done; void alarm_handler(int sig) @@ -87,7 +87,6 @@ class PrimTest : public CppUnit::TestFixture CPPUNIT_TEST(p_IdxList_1); CPPUNIT_TEST(p_IdxList_2); - // whole block tests CPPUNIT_TEST(p_Col_1); CPPUNIT_TEST(p_Col_2); @@ -162,7 +161,11 @@ class PrimTest : public CppUnit::TestFixture // CPPUNIT_TEST(p_Dictionary_like_prefixbench_1); // CPPUNIT_TEST(p_Dictionary_like_substrbench_1); - + +// binary data type + CPPUNIT_TEST(p_Col_bin_16); + CPPUNIT_TEST(p_Col_bin_32); + CPPUNIT_TEST_SUITE_END(); private: @@ -3744,6 +3747,178 @@ public: close(fd); } + + template struct binary; + typedef binary<16> binary16; + typedef binary<32> binary32; + template + struct binary { + unsigned char data[W]; // May be ok for empty value ? + void operator=(uint64_t v) {*((uint64_t *) data) = v; memset(data + 8, 0, W - 8);} + inline uint8_t& operator[](const int index) {return *((uint8_t*) (data + index));} + inline uint64_t& uint64(const int index) {return *((uint64_t*) (data + (index << 3)));} + }; + + void p_Col_bin_16() + { + PrimitiveProcessor pp; + uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + NewColResultHeader* out; + ColArgs* args; + binary16* results; + uint32_t written, i; + int fd; + binary16 tmp; + binary16* bin16 = (binary16*) block; + + for(int i = 0; i < BLOCK_SIZE/16; i++) + { + bin16[i] = 0; + } + + bin16[0].uint64(0) = 10UL; + + bin16[1].uint64(0) = 1000UL; + + bin16[3].uint64(0) = 1000UL; + bin16[3].uint64(1) = 1; + + bin16[4].uint64(0) = 256; + bin16[4].uint64(1) = 1; + + typedef char bin16_t[16]; + + *(uint64_t*)(((bin16_t*)block) + 5) = 500; + + *(uint64_t*)&((bin16_t*)block)[6] = 501; + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); + + in->DataSize = sizeof(binary16); + in->DataType = execplan::CalpontSystemCatalog::BINARY; + in->OutputType = OT_DATAVALUE; + in->NOPS = 3; + in->BOP = BOP_OR; + in->NVALS = 0; + + tmp = 10; + args->COP = COMPARE_EQ; + memcpy(args->val, &tmp, in->DataSize); + args = reinterpret_cast (args->val + in->DataSize); + + args->COP = COMPARE_EQ; + tmp = 1000; + memcpy(args->val, &tmp, in->DataSize); + + args = reinterpret_cast (args->val + in->DataSize); + tmp.uint64(0) = 256; + tmp.uint64(1) = 1; + args->COP = COMPARE_EQ; + memcpy(args->val, &tmp, in->DataSize); + + pp.setBlockPtr((int*) block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + results = reinterpret_cast(&output[sizeof(NewColResultHeader)]); +// cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT_EQUAL((uint16_t)3, out->NVALS); + CPPUNIT_ASSERT_EQUAL((u_int64_t)10, results[0].uint64(0)); + CPPUNIT_ASSERT_EQUAL((u_int64_t)1000, results[1].uint64(0)); + for (i = 0; i < out->NVALS; i++) { + printf("Result %d Value %016X%016X\n",i ,results[i].uint64(1),results[i].uint64(0) ); +// CPPUNIT_ASSERT(results[i] == (uint32_t) (i < 10 ? i : i - 10 + 1001)); + } + } + + void p_Col_bin_32() + { + PrimitiveProcessor pp; + uint8_t input[2 * BLOCK_SIZE], output[8 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + NewColResultHeader* out; + ColArgs* args; + binary32* results; + uint32_t written, i; + int fd; + binary32 tmp; + binary32* bin32 = (binary32*) block; + + for(int i = 0; i < BLOCK_SIZE/32; i++) + { + bin32[i].uint64(0) = 0; + } + + bin32[0].uint64(0) = 10UL; + + bin32[1].uint64(0) = 1000UL; + + bin32[3].uint64(0) = 1000UL; + bin32[3].uint64(1) = 1; + + bin32[4].uint64(0) = 256; + bin32[4].uint64(1) = 254; + bin32[4].uint64(2) = 253; + bin32[4].uint64(3) = 252; + + typedef char bin32_t[32]; + + *(uint64_t*)(((bin32_t*)block) + 5) = 500; + + *(uint64_t*)&((bin32_t*)block)[6] = 501; + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); + + in->DataSize = sizeof(binary32); + in->DataType = execplan::CalpontSystemCatalog::BINARY; + in->OutputType = OT_DATAVALUE; + in->NOPS = 3; + in->BOP = BOP_OR; + in->NVALS = 0; + + tmp = 10; + args->COP = COMPARE_EQ; + memcpy(args->val, &tmp, in->DataSize); + args = reinterpret_cast (args->val + in->DataSize); + + args->COP = COMPARE_EQ; + tmp = 1000; + memcpy(args->val, &tmp, in->DataSize); + + args = reinterpret_cast (args->val + in->DataSize); + tmp.uint64(0) = 256; + tmp.uint64(1) = 254; + tmp.uint64(2) = 253; + tmp.uint64(3) = 252; + + args->COP = COMPARE_EQ; + memcpy(args->val, &tmp, in->DataSize); + + pp.setBlockPtr((int*) block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + results = reinterpret_cast(&output[sizeof(NewColResultHeader)]); +// cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT_EQUAL((uint16_t)3, out->NVALS); +// CPPUNIT_ASSERT_EQUAL((u_int64_t)10, results[0].uint64(0)); +// CPPUNIT_ASSERT_EQUAL((u_int64_t)1000, results[1].uint64(0)); + for (i = 0; i < out->NVALS; i++) { + printf("Result %d Value %016X%016X%016X%016X\n",i ,results[i].uint64(3),results[i].uint64(2),results[i].uint64(1),results[i].uint64(0) ); +// CPPUNIT_ASSERT(results[i] == (uint32_t) (i < 10 ? i : i - 10 + 1001)); + } + } + + void p_Dictionary_1() { diff --git a/primitives/linux-port/primitiveprocessor.h b/primitives/linux-port/primitiveprocessor.h index 0e8a93396..4c674f95e 100644 --- a/primitives/linux-port/primitiveprocessor.h +++ b/primitives/linux-port/primitiveprocessor.h @@ -82,7 +82,26 @@ public: } }; +class pcfHasher128 +{ +public: + inline size_t operator()(const int128_t i) const + { + return *reinterpret_cast(&i); + } +}; + +class pcfEqual128 +{ +public: + inline bool operator()(const int128_t f1, const int128_t f2) const + { + return f1 == f2; + } +}; + typedef std::tr1::unordered_set prestored_set_t; +typedef std::tr1::unordered_set prestored_set_t_128; typedef std::tr1::unordered_set DictEqualityFilter; struct idb_regex_t @@ -109,9 +128,11 @@ struct ParsedColumnFilter { ColumnFilterMode columnFilterMode; boost::shared_array prestored_argVals; + boost::shared_array prestored_argVals128; boost::shared_array prestored_cops; boost::shared_array prestored_rfs; boost::shared_ptr prestored_set; + boost::shared_ptr prestored_set_128; boost::shared_array prestored_regex; uint8_t likeOps; @@ -308,6 +329,7 @@ private: int dict_OffsetIndex, currentOffsetIndex; // used by p_dictionary int fDebugLevel; dbbc::Stats* fStatsPtr; // pointer for pmstats + // To be removed b/c always true bool logicalBlockMode; boost::shared_ptr parsedColumnFilter; diff --git a/primitives/primproc/CMakeLists.txt b/primitives/primproc/CMakeLists.txt index aab0232ac..b8d2e3741 100644 --- a/primitives/primproc/CMakeLists.txt +++ b/primitives/primproc/CMakeLists.txt @@ -21,8 +21,6 @@ set(PrimProc_SRCS umsocketselector.cpp ../../utils/common/crashtrace.cpp) -#PrimProc_CXXFLAGS = $(march_flags) $(AM_CXXFLAGS) - add_executable(PrimProc ${PrimProc_SRCS}) add_dependencies(PrimProc loggingcpp) diff --git a/primitives/primproc/batchprimitiveprocessor.cpp b/primitives/primproc/batchprimitiveprocessor.cpp index 3750c0cde..869c9758f 100644 --- a/primitives/primproc/batchprimitiveprocessor.cpp +++ b/primitives/primproc/batchprimitiveprocessor.cpp @@ -1,5 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2016, 2017 MariaDB Corporation + Copyright (C) 2016-2020 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -54,6 +54,7 @@ using namespace boost; #include "MonitorProcMem.h" #include "threadnaming.h" #include "vlarray.h" +#include "widedecimalutils.h" #define MAX64 0x7fffffffffffffffLL #define MIN64 0x8000000000000000LL @@ -100,6 +101,7 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor() : baseRid(0), ridCount(0), needStrValues(false), + wideColumnsWidths(0), filterCount(0), projectCount(0), sendRidsAtDelivery(false), @@ -111,6 +113,7 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor() : minVal(MAX64), maxVal(MIN64), lbidForCP(0), + hasWideColumnOut(false), busyLoaderCount(0), physIO(0), cachedIO(0), @@ -144,6 +147,7 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor(ByteStream& b, double prefetch, baseRid(0), ridCount(0), needStrValues(false), + wideColumnsWidths(0), filterCount(0), projectCount(0), sendRidsAtDelivery(false), @@ -155,6 +159,7 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor(ByteStream& b, double prefetch, minVal(MAX64), maxVal(MIN64), lbidForCP(0), + hasWideColumnOut(false), busyLoaderCount(0), physIO(0), cachedIO(0), @@ -216,6 +221,7 @@ void BatchPrimitiveProcessor::initBPP(ByteStream& bs) { uint32_t i; uint8_t tmp8; + uint16_t tmp16; Command::CommandType type; bs.advance(sizeof(ISMPacketHeader)); // skip the header @@ -227,21 +233,25 @@ void BatchPrimitiveProcessor::initBPP(ByteStream& bs) bs >> uniqueID; bs >> versionInfo; - bs >> tmp8; - needStrValues = tmp8 & NEED_STR_VALUES; - gotAbsRids = tmp8 & GOT_ABS_RIDS; - gotValues = tmp8 & GOT_VALUES; - LBIDTrace = tmp8 & LBID_TRACE; - sendRidsAtDelivery = tmp8 & SEND_RIDS_AT_DELIVERY; - doJoin = tmp8 & HAS_JOINER; - hasRowGroup = tmp8 & HAS_ROWGROUP; - getTupleJoinRowGroupData = tmp8 & JOIN_ROWGROUP_DATA; + bs >> tmp16; + needStrValues = tmp16 & NEED_STR_VALUES; + gotAbsRids = tmp16 & GOT_ABS_RIDS; + gotValues = tmp16 & GOT_VALUES; + LBIDTrace = tmp16 & LBID_TRACE; + sendRidsAtDelivery = tmp16 & SEND_RIDS_AT_DELIVERY; + doJoin = tmp16 & HAS_JOINER; + hasRowGroup = tmp16 & HAS_ROWGROUP; + getTupleJoinRowGroupData = tmp16 & JOIN_ROWGROUP_DATA; + bool hasWideColumnsIn = tmp16 & HAS_WIDE_COLUMNS; // This used to signify that there was input row data from previous jobsteps, and // it never quite worked right. No need to fix it or update it; all BPP's have started // with a scan for years. Took it out. assert(!hasRowGroup); + if (hasWideColumnsIn) + bs >> wideColumnsWidths; + bs >> bop; bs >> forHJ; @@ -1016,6 +1026,8 @@ void BatchPrimitiveProcessor::initProcessor() fFiltRidCount[i] = 0; fFiltCmdRids[i].reset(new uint16_t[LOGICAL_BLOCK_RIDS]); fFiltCmdValues[i].reset(new int64_t[LOGICAL_BLOCK_RIDS]); + if (wideColumnsWidths | datatypes::MAXDECIMALWIDTH) + fFiltCmdBinaryValues[i].reset(new int128_t[LOGICAL_BLOCK_RIDS]); if (filtOnString) fFiltStrValues[i].reset(new string[LOGICAL_BLOCK_RIDS]); } @@ -1082,8 +1094,16 @@ void BatchPrimitiveProcessor::initProcessor() fAggregator->setInputOutput(fe2 ? fe2Output : outputRG, &fAggregateRG); } - minVal = MAX64; - maxVal = MIN64; + if (LIKELY(!hasWideColumnOut)) + { + minVal = MAX64; + maxVal = MIN64; + } + else + { + max128Val = datatypes::Decimal::minInt128; + min128Val = datatypes::Decimal::maxInt128; + } // @bug 1269, initialize data used by execute() for async loading blocks // +1 for the scan filter step with no predicate, if any @@ -1970,8 +1990,19 @@ void BatchPrimitiveProcessor::writeProjectionPreamble() { *serialized << (uint8_t) 1; *serialized << lbidForCP; - *serialized << (uint64_t) minVal; - *serialized << (uint64_t) maxVal; + if (UNLIKELY(hasWideColumnOut)) + { + // PSA width + *serialized << (uint8_t) wideColumnWidthOut; + *serialized << min128Val; + *serialized << max128Val; + } + else + { + *serialized << (uint8_t) utils::MAXLEGACYWIDTH; // width of min/max value + *serialized << (uint64_t) minVal; + *serialized << (uint64_t) maxVal; + } } else { @@ -2060,8 +2091,22 @@ void BatchPrimitiveProcessor::makeResponse() { *serialized << (uint8_t) 1; *serialized << lbidForCP; - *serialized << (uint64_t) minVal; - *serialized << (uint64_t) maxVal; + + if (UNLIKELY(hasWideColumnOut)) + { + // PSA width + // Remove the assert for >16 bytes DTs. + assert(wideColumnWidthOut == datatypes::MAXDECIMALWIDTH); + *serialized << (uint8_t) wideColumnWidthOut; + *serialized << min128Val; + *serialized << max128Val; + } + else + { + *serialized << (uint8_t) utils::MAXLEGACYWIDTH; // width of min/max value + *serialized << (uint64_t) minVal; + *serialized << (uint64_t) maxVal; + } } else { @@ -2164,8 +2209,18 @@ int BatchPrimitiveProcessor::operator()() } allocLargeBuffers(); - minVal = MAX64; - maxVal = MIN64; + + if (LIKELY(!hasWideColumnOut)) + { + minVal = MAX64; + maxVal = MIN64; + } + else + { + max128Val = datatypes::Decimal::minInt128; + min128Val = datatypes::Decimal::maxInt128; + } + validCPData = false; #ifdef PRIMPROC_STOPWATCH stopwatch->start("BPP() execute"); @@ -2300,6 +2355,7 @@ SBPP BatchPrimitiveProcessor::duplicate() bpp->stepID = stepID; bpp->uniqueID = uniqueID; bpp->needStrValues = needStrValues; + bpp->wideColumnsWidths = wideColumnsWidths; bpp->gotAbsRids = gotAbsRids; bpp->gotValues = gotValues; bpp->LBIDTrace = LBIDTrace; diff --git a/primitives/primproc/batchprimitiveprocessor.h b/primitives/primproc/batchprimitiveprocessor.h index d56481a5f..fd1b3992e 100644 --- a/primitives/primproc/batchprimitiveprocessor.h +++ b/primitives/primproc/batchprimitiveprocessor.h @@ -52,6 +52,7 @@ #include "rowaggregation.h" #include "funcexpwrapper.h" #include "bppsendthread.h" +#include "columnwidth.h" namespace primitiveprocessor { @@ -204,15 +205,16 @@ private: uint64_t baseRid; // first rid of the logical block uint16_t relRids[LOGICAL_BLOCK_RIDS]; - int64_t values[LOGICAL_BLOCK_RIDS]; + int64_t values[LOGICAL_BLOCK_RIDS]; + int128_t wide128Values[LOGICAL_BLOCK_RIDS]; boost::scoped_array absRids; boost::scoped_array strValues; uint16_t ridCount; bool needStrValues; + uint16_t wideColumnsWidths; /* Common space for primitive data */ - static const uint32_t BUFFER_SIZE = 65536; - uint8_t blockData[BLOCK_SIZE * 8]; + uint8_t blockData[BLOCK_SIZE * utils::MAXCOLUMNWIDTH]; boost::scoped_array outputMsg; uint32_t outMsgSize; @@ -228,9 +230,21 @@ private: bool hasScan; bool validCPData; - int64_t minVal, maxVal; // CP data from a scanned column - uint64_t lbidForCP; + // CP data from a scanned column + union + { + int128_t min128Val; + int64_t minVal; + }; + union + { + int128_t max128Val; + int64_t maxVal; + }; + uint64_t lbidForCP; + bool hasWideColumnOut; + uint8_t wideColumnWidthOut; // IO counters boost::mutex counterLock; uint32_t busyLoaderCount; @@ -262,6 +276,7 @@ private: bool filtOnString; boost::scoped_array fFiltCmdRids[2]; boost::scoped_array fFiltCmdValues[2]; + boost::scoped_array fFiltCmdBinaryValues[2]; boost::scoped_array fFiltStrValues[2]; uint64_t fFiltRidCount[2]; diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index db90b3577..ab6fbe223 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -40,6 +40,7 @@ using namespace std; #include "primitiveserver.h" #include "primproc.h" #include "stats.h" +#include "datatypes/mcs_int128.h" using namespace messageqcpp; using namespace rowgroup; @@ -47,6 +48,8 @@ using namespace rowgroup; #include "messageids.h" using namespace logging; +#include "emptyvaluemanip.h" + #ifdef _MSC_VER #define llabs labs #endif @@ -68,7 +71,6 @@ ColumnCommand::~ColumnCommand() { } void ColumnCommand::_execute() { -// cout << "CC: executing" << endl; if (_isScan) makeScanMsg(); else if (bpp->ridCount == 0) // this would cause a scan @@ -90,11 +92,20 @@ void ColumnCommand::_execute() void ColumnCommand::execute() { if (fFilterFeeder == LEFT_FEEDER) + { values = bpp->fFiltCmdValues[0].get(); + wide128Values = bpp->fFiltCmdBinaryValues[0].get(); + } else if (fFilterFeeder == RIGHT_FEEDER) + { values = bpp->fFiltCmdValues[1].get(); + wide128Values = bpp->fFiltCmdBinaryValues[1].get(); + } else + { values = bpp->values; + wide128Values = bpp->wide128Values; + } _execute(); } @@ -142,8 +153,10 @@ void ColumnCommand::loadData() bool lastBlockReached = false; oidLastLbid = getLastLbid(); uint32_t blocksToLoad = 0; - BRM::LBID_t* lbids = (BRM::LBID_t*) alloca(8 * sizeof(BRM::LBID_t)); - uint8_t** blockPtrs = (uint8_t**) alloca(8 * sizeof(uint8_t*)); + // The number of elements allocated equals to the number of + // iteratations of the first loop here. + BRM::LBID_t* lbids = (BRM::LBID_t*) alloca(colType.colWidth * sizeof(BRM::LBID_t)); + uint8_t** blockPtrs = (uint8_t**) alloca(colType.colWidth * sizeof(uint8_t*)); int i; @@ -165,6 +178,7 @@ void ColumnCommand::loadData() { // fill remaining blocks with empty values when col scan int blockLen = BLOCK_SIZE / colType.colWidth; + ByteStream::hexbyte* hPtr = NULL; ByteStream::octbyte* oPtr = NULL; ByteStream::quadbyte* qPtr = NULL; ByteStream::byte* bPtr = NULL; @@ -183,29 +197,43 @@ void ColumnCommand::loadData() if (colType.colWidth == 8) oPtr = reinterpret_cast(&bpp->blockData[i * BLOCK_SIZE]); + if (colType.colWidth == 16) + hPtr = reinterpret_cast(&bpp->blockData[i * BLOCK_SIZE]); + + for (int idx = 0; idx < blockLen; idx++) { if (bPtr && colType.colWidth == 1) { - ByteStream::byte b = getEmptyRowValue(colType.colDataType, colType.colWidth); + ByteStream::byte b; + utils::getEmptyRowValue(colType.colDataType, colType.colWidth, (uint8_t*)&b); bPtr[idx] = b; } //@Bug 1812. Added two bytes column handling else if (dPtr && colType.colWidth == 2) { - ByteStream::doublebyte d = getEmptyRowValue(colType.colDataType, colType.colWidth); + ByteStream::doublebyte d; + utils::getEmptyRowValue(colType.colDataType, colType.colWidth, (uint8_t*)&d); dPtr[idx] = d; } else if (qPtr && colType.colWidth == 4) { - ByteStream::quadbyte q = getEmptyRowValue(colType.colDataType, colType.colWidth); + ByteStream::quadbyte q; + utils::getEmptyRowValue(colType.colDataType, colType.colWidth, (uint8_t*)&q); qPtr[idx] = q; } else if (oPtr && colType.colWidth == 8) { - ByteStream::octbyte o = getEmptyRowValue(colType.colDataType, colType.colWidth); + ByteStream::octbyte o; + utils::getEmptyRowValue(colType.colDataType, colType.colWidth, (uint8_t*)&o); oPtr[idx] = o; } + else if (colType.colWidth == 16) + { + ByteStream::hexbyte h; + utils::getEmptyRowValue(colType.colDataType, colType.colWidth, (uint8_t*)&h); + datatypes::TSInt128::storeUnaligned(hPtr + idx, h); + } } }// else @@ -240,12 +268,11 @@ void ColumnCommand::issuePrimitive() loadData(); -// cout << "issuing primitive for LBID " << primMsg->LBID << endl; if (!suppressFilter) bpp->pp.setParsedColumnFilter(parsedColumnFilter); else bpp->pp.setParsedColumnFilter(emptyFilter); - + bpp->pp.p_Col(primMsg, outMsg, bpp->outMsgSize, (unsigned int*)&resultSize); /* Update CP data, the PseudoColumn code should always be !_isScan. Should be safe @@ -256,8 +283,30 @@ void ColumnCommand::issuePrimitive() //if (wasVersioned && outMsg->ValidMinMax) // cout << "CC: versioning overriding min max data\n"; bpp->lbidForCP = lbid; - bpp->maxVal = outMsg->Max; - bpp->minVal = outMsg->Min; + if (UNLIKELY(utils::isWide(colType.colWidth))) + { + if (colType.isWideDecimalType()) + { + bpp->hasWideColumnOut = true; + // colWidth is int32 and wideColumnWidthOut's + // value is expected to be at most uint8. + bpp->wideColumnWidthOut = colType.colWidth; + bpp->max128Val = outMsg->Max; + bpp->min128Val = outMsg->Min; + } + else + { + ostringstream os; + os << " WARNING!!! Not implemented for "; + os << primMsg->DataSize << " column."; + throw PrimitiveColumnProjectResultExcept(os.str()); + } + } + else + { + bpp->maxVal = static_cast(outMsg->Max); + bpp->minVal = static_cast(outMsg->Min); + } } } // issuePrimitive() @@ -268,11 +317,24 @@ void ColumnCommand::process_OT_BOTH() bpp->ridCount = outMsg->NVALS; bpp->ridMap = outMsg->RidFlags; -// cout << "rid Count is " << bpp->ridCount << endl; /* this is verbose and repetative to minimize the work per row */ switch (colType.colWidth) { + case 16: + for (i = 0, pos = sizeof(NewColResultHeader); i < outMsg->NVALS; ++i) + { + if (makeAbsRids) + bpp->absRids[i] = *((uint16_t*) &bpp->outputMsg[pos]) + bpp->baseRid; + + bpp->relRids[i] = *((uint16_t*) &bpp->outputMsg[pos]); + pos += 2; + datatypes::TSInt128::assignPtrPtr(&wide128Values[i], &bpp->outputMsg[pos]); + pos += 16; + } + + break; + case 8: for (i = 0, pos = sizeof(NewColResultHeader); i < outMsg->NVALS; ++i) { @@ -336,20 +398,23 @@ void ColumnCommand::process_OT_RID() memcpy(bpp->relRids, outMsg + 1, outMsg->NVALS << 1); bpp->ridCount = outMsg->NVALS; bpp->ridMap = outMsg->RidFlags; -// cout << "rid Count is " << bpp->ridCount << endl; } void ColumnCommand::process_OT_DATAVALUE() { bpp->ridCount = outMsg->NVALS; -// cout << "rid Count is " << bpp->ridCount << endl; switch (colType.colWidth) { + case 16: + { + memcpy(wide128Values, outMsg + 1, outMsg->NVALS << 4); + break; + } + case 8: { memcpy(values, outMsg + 1, outMsg->NVALS << 3); -// cout << " CC: first value is " << values[0] << endl; break; } @@ -427,8 +492,6 @@ void ColumnCommand::processResult() for (uint64_t i = 0; i < bpp->ridCount; i++) bpp->fFiltCmdRids[1][i] = bpp->relRids[i]; } - -// cout << "processed " << outMsg->NVALS << " rows" << endl; } void ColumnCommand::createCommand(ByteStream& bs) @@ -459,6 +522,9 @@ void ColumnCommand::createCommand(ByteStream& bs) bs >> BOP; bs >> filterCount; deserializeInlineVector(bs, lastLbid); + +// cout << __func__ << " colType.colWidth " << colType.colWidth << endl; + // cout << "lastLbid count=" << lastLbid.size() << endl; // for (uint32_t i = 0; i < lastLbid.size(); i++) // cout << " " << lastLbid[i]; @@ -488,7 +554,7 @@ void ColumnCommand::resetCommand(ByteStream& bs) void ColumnCommand::prep(int8_t outputType, bool absRids) { /* make the template NewColRequestHeader */ - + baseMsgLength = sizeof(NewColRequestHeader) + (suppressFilter ? 0 : filterString.length()); @@ -533,6 +599,8 @@ void ColumnCommand::prep(int8_t outputType, bool absRids) + // JFYI This switch results are used by index scan code that is unused + // as of 1.5 switch (colType.colWidth) { case 1: @@ -554,7 +622,11 @@ void ColumnCommand::prep(int8_t outputType, bool absRids) shift = 1; mask = 0x01; break; - + case 16: + shift = 1; + mask = 0x01; + break; + default: cout << "CC: colWidth is " << colType.colWidth << endl; throw logic_error("ColumnCommand: bad column width?"); @@ -737,10 +809,8 @@ void ColumnCommand::projectResultRG(RowGroup& rg, uint32_t pos) r.setUintField_offset<4>(*((uint32_t*) msg8), offset); r.nextRow(rowSize); } - break; } - case 8: { for (i = 0; i < outMsg->NVALS; ++i, msg8 += gapSize) @@ -748,9 +818,18 @@ void ColumnCommand::projectResultRG(RowGroup& rg, uint32_t pos) r.setUintField_offset<8>(*((uint64_t*) msg8), offset); r.nextRow(rowSize); } - break; } + case 16: + { + for (i = 0; i < outMsg->NVALS; ++i, msg8 += gapSize) + { + r.setBinaryField_offset((int128_t*)msg8, colType.colWidth, offset); + r.nextRow(rowSize); + } + break; + } + } } @@ -889,107 +968,6 @@ void ColumnCommand::enableFilters() prep(primMsg->OutputType, makeAbsRids); } - -/*********************************************************** -* DESCRIPTION: -* Get the value that represents empty row -* PARAMETERS: -* dataType - data type -* width - data width in byte -* RETURN: -* emptyVal - the value of empty row -***********************************************************/ -uint64_t ColumnCommand::getEmptyRowValue( const execplan::CalpontSystemCatalog::ColDataType dataType, const int width ) const -{ - uint64_t emptyVal = 0; - int offset; - - offset = ( dataType == execplan::CalpontSystemCatalog::VARCHAR ) ? -1 : 0; - - switch ( dataType ) - { - case execplan::CalpontSystemCatalog::TINYINT : - emptyVal = joblist::TINYINTEMPTYROW; - break; - - case execplan::CalpontSystemCatalog::SMALLINT: - emptyVal = joblist::SMALLINTEMPTYROW; - break; - - case execplan::CalpontSystemCatalog::MEDINT : - case execplan::CalpontSystemCatalog::INT : - emptyVal = joblist::INTEMPTYROW; - break; - - case execplan::CalpontSystemCatalog::BIGINT : - emptyVal = joblist::BIGINTEMPTYROW; - break; - - case execplan::CalpontSystemCatalog::UTINYINT : - emptyVal = joblist::UTINYINTEMPTYROW; - break; - - case execplan::CalpontSystemCatalog::USMALLINT: - emptyVal = joblist::USMALLINTEMPTYROW; - break; - - case execplan::CalpontSystemCatalog::UMEDINT : - case execplan::CalpontSystemCatalog::UINT : - emptyVal = joblist::UINTEMPTYROW; - break; - - case execplan::CalpontSystemCatalog::UBIGINT : - emptyVal = joblist::UBIGINTEMPTYROW; - break; - - case execplan::CalpontSystemCatalog::FLOAT : - case execplan::CalpontSystemCatalog::UFLOAT : - emptyVal = joblist::FLOATEMPTYROW; - break; - - case execplan::CalpontSystemCatalog::DOUBLE : - case execplan::CalpontSystemCatalog::UDOUBLE : - emptyVal = joblist::DOUBLEEMPTYROW; - break; - - case execplan::CalpontSystemCatalog::DECIMAL : - case execplan::CalpontSystemCatalog::UDECIMAL : - if ( width <= 1 ) - emptyVal = joblist::TINYINTEMPTYROW; - else if (width <= 2) - emptyVal = joblist::SMALLINTEMPTYROW; - else if ( width <= 4 ) - emptyVal = joblist::INTEMPTYROW; - else - emptyVal = joblist::BIGINTEMPTYROW; - - break; - - case execplan::CalpontSystemCatalog::CHAR : - case execplan::CalpontSystemCatalog::VARCHAR : - case execplan::CalpontSystemCatalog::DATE : - case execplan::CalpontSystemCatalog::DATETIME : - case execplan::CalpontSystemCatalog::TIMESTAMP : - case execplan::CalpontSystemCatalog::TIME : - case execplan::CalpontSystemCatalog::VARBINARY : - case execplan::CalpontSystemCatalog::BLOB : - case execplan::CalpontSystemCatalog::TEXT : - default: - emptyVal = joblist::CHAR1EMPTYROW; - - if ( width == (2 + offset) ) - emptyVal = joblist::CHAR2EMPTYROW; - else if ( width >= (3 + offset) && width <= ( 4 + offset ) ) - emptyVal = joblist::CHAR4EMPTYROW; - else if ( width >= (5 + offset) ) - emptyVal = joblist::CHAR8EMPTYROW; - - break; - } - - return emptyVal; -} - void ColumnCommand::getLBIDList(uint32_t loopCount, vector* lbids) { int64_t firstLBID = lbid, lastLBID = firstLBID + (loopCount * colType.colWidth) - 1, i; diff --git a/primitives/primproc/columncommand.h b/primitives/primproc/columncommand.h index e1d67aa0d..eb51db842 100644 --- a/primitives/primproc/columncommand.h +++ b/primitives/primproc/columncommand.h @@ -34,6 +34,8 @@ #include "command.h" #include "calpontsystemcatalog.h" +using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; + namespace primitiveprocessor { @@ -82,7 +84,6 @@ public: makeAbsRids = m; } bool willPrefetch(); - uint64_t getEmptyRowValue( const execplan::CalpontSystemCatalog::ColDataType dataType, const int width ) const; int64_t getLastLbid(); void getLBIDList(uint32_t loopCount, std::vector* lbids); @@ -146,6 +147,7 @@ private: uint16_t filterCount; bool makeAbsRids; int64_t* values; // this is usually bpp->values; RTSCommand needs to use a different container + int128_t* wide128Values; uint8_t mask, shift; // vars for the selective block loader diff --git a/primitives/primproc/filtercommand.cpp b/primitives/primproc/filtercommand.cpp index 99059f88f..ac9f8c79c 100644 --- a/primitives/primproc/filtercommand.cpp +++ b/primitives/primproc/filtercommand.cpp @@ -27,6 +27,7 @@ #include "dictstep.h" #include "filtercommand.h" #include "dataconvert.h" +#include "mcs_decimal.h" using namespace std; using namespace messageqcpp; @@ -174,7 +175,8 @@ Command* FilterCommand::makeFilterCommand(ByteStream& bs, vector& cmds } -FilterCommand::FilterCommand() : Command(FILTER_COMMAND), fBOP(0) +FilterCommand::FilterCommand() : Command(FILTER_COMMAND), fBOP(0), + hasWideColumns(false) { } @@ -247,6 +249,9 @@ void FilterCommand::setColTypes(const execplan::CalpontSystemCatalog::ColType& l { leftColType = left; rightColType = right; + + if (left.isWideDecimalType() || right.isWideDecimalType()) + hasWideColumns = true; } @@ -255,6 +260,13 @@ void FilterCommand::doFilter() bpp->ridMap = 0; bpp->ridCount = 0; + bool (FilterCommand::*compareFunc)(uint64_t, uint64_t); + + if (hasWideColumns) + compareFunc = &FilterCommand::binaryCompare; + else + compareFunc = &FilterCommand::compare; + // rids in [0] is used for scan [1], so [1] is a subset of [0], and same order. // -- see makeFilterCommand() above. for (uint64_t i = 0, j = 0; j < bpp->fFiltRidCount[1]; ) @@ -265,10 +277,13 @@ void FilterCommand::doFilter() } else { - if (compare(i, j) == true) + if ((this->*compareFunc)(i, j) == true) { bpp->relRids[bpp->ridCount] = bpp->fFiltCmdRids[0][i]; - bpp->values[bpp->ridCount] = bpp->fFiltCmdValues[0][i]; + if (leftColType.isWideDecimalType()) + bpp->wide128Values[bpp->ridCount] = bpp->fFiltCmdBinaryValues[0][i]; + else + bpp->values[bpp->ridCount] = bpp->fFiltCmdValues[0][i]; bpp->ridMap |= 1 << (bpp->relRids[bpp->ridCount] >> 10); bpp->ridCount++; } @@ -321,6 +336,70 @@ bool FilterCommand::compare(uint64_t i, uint64_t j) } } +bool FilterCommand::binaryCompare(uint64_t i, uint64_t j) +{ + // We type-promote to int128_t if either of the columns are + // not int128_t + int128_t leftVal, rightVal; + + if (leftColType.isWideDecimalType()) + { + if (execplan::isNull(bpp->fFiltCmdBinaryValues[0][i], leftColType)) + return false; + leftVal = bpp->fFiltCmdBinaryValues[0][i]; + } + else + { + if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType)) + return false; + leftVal = bpp->fFiltCmdValues[0][i]; + } + + if (rightColType.isWideDecimalType()) + { + if (execplan::isNull(bpp->fFiltCmdBinaryValues[1][j], rightColType)) + return false; + rightVal = bpp->fFiltCmdBinaryValues[1][j]; + } + else + { + if (execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType)) + return false; + rightVal = bpp->fFiltCmdValues[1][j]; + } + + switch (fBOP) + { + case COMPARE_GT: + return leftVal > rightVal; + break; + + case COMPARE_LT: + return leftVal < rightVal; + break; + + case COMPARE_EQ: + return leftVal == rightVal; + break; + + case COMPARE_GE: + return leftVal >= rightVal; + break; + + case COMPARE_LE: + return leftVal <= rightVal; + break; + + case COMPARE_NE: + return leftVal != rightVal; + break; + + default: + return false; + break; + } +} + bool FilterCommand::operator==(const FilterCommand& c) const { diff --git a/primitives/primproc/filtercommand.h b/primitives/primproc/filtercommand.h index be4a7931f..263523bef 100644 --- a/primitives/primproc/filtercommand.h +++ b/primitives/primproc/filtercommand.h @@ -76,9 +76,14 @@ protected: // compare method, take the indices to the values array virtual bool compare(uint64_t, uint64_t); + // compare method, take the indices to the values array + virtual bool binaryCompare(uint64_t, uint64_t); + // binary operator uint8_t fBOP; + bool hasWideColumns; + // column type for null check execplan::CalpontSystemCatalog::ColType leftColType; execplan::CalpontSystemCatalog::ColType rightColType; diff --git a/primitives/primproc/passthrucommand.cpp b/primitives/primproc/passthrucommand.cpp index 407c257dc..7f345d6d8 100644 --- a/primitives/primproc/passthrucommand.cpp +++ b/primitives/primproc/passthrucommand.cpp @@ -77,6 +77,10 @@ void PassThruCommand::project() switch (colWidth) { + case 16: + bpp->serialized->append((uint8_t*) bpp->wide128Values, bpp->ridCount << 4); + break; + case 8: bpp->serialized->append((uint8_t*) bpp->values, bpp->ridCount << 3); break; @@ -118,7 +122,6 @@ void PassThruCommand::projectIntoRowGroup(RowGroup& rg, uint32_t col) case 1: for (i = 0; i < bpp->ridCount; i++) { -// cout << "PTC: " << bpp->values[i] << endl; r.setUintField_offset<1>(bpp->values[i], offset); r.nextRow(rowSize); } @@ -128,7 +131,6 @@ void PassThruCommand::projectIntoRowGroup(RowGroup& rg, uint32_t col) case 2: for (i = 0; i < bpp->ridCount; i++) { -// cout << "PTC: " << bpp->values[i] << endl; r.setUintField_offset<2>(bpp->values[i], offset); r.nextRow(rowSize); } @@ -147,12 +149,17 @@ void PassThruCommand::projectIntoRowGroup(RowGroup& rg, uint32_t col) case 8: for (i = 0; i < bpp->ridCount; i++) { -// cout << "PTC: " << bpp->values[i] << endl; r.setUintField_offset<8>(bpp->values[i], offset); r.nextRow(rowSize); } break; + case 16: + for (i = 0; i < bpp->ridCount; i++) + { + r.setBinaryField_offset(&bpp->wide128Values[i], 16, offset); + r.nextRow(rowSize); + } } } diff --git a/primitives/primproc/primproc.cpp b/primitives/primproc/primproc.cpp index f282f8a7a..883503bf7 100644 --- a/primitives/primproc/primproc.cpp +++ b/primitives/primproc/primproc.cpp @@ -512,8 +512,9 @@ int ServicePrimProc::Child() // set to smallest extent size // do not allow to read beyond the end of an extent const int MaxReadAheadSz = (extentRows) / BLOCK_SIZE; - //defaultBufferSize = 512 * 1024; // @bug 2627 - changed default dict buffer from 256K to 512K, allows for cols w/ length of 61. - defaultBufferSize = 100 * 1024; // 1/17/12 - made the dict buffer dynamic, max size for a numeric col is 80k + ovrhd + // Max size for a primitive message column buffer. Must be big enough + // to accomodate 8192 values of the widest non-dict column + 8192 RIDs + ohead. + defaultBufferSize = 150 * 1024; // This parm controls whether we rotate through the output sockets diff --git a/primitives/primproc/pseudocc.cpp b/primitives/primproc/pseudocc.cpp index 1ee33a04b..588723452 100644 --- a/primitives/primproc/pseudocc.cpp +++ b/primitives/primproc/pseudocc.cpp @@ -42,6 +42,7 @@ SCommand PseudoCC::duplicate() ret.reset(pseudo); pseudo->function = function; pseudo->valueFromUM = valueFromUM; + pseudo->bigValueFromUM = bigValueFromUM; ColumnCommand::duplicate(pseudo); return ret; } @@ -55,9 +56,21 @@ void PseudoCC::createCommand(messageqcpp::ByteStream& bs) void PseudoCC::resetCommand(messageqcpp::ByteStream& bs) { - if (function == PSEUDO_EXTENTMAX || function == PSEUDO_EXTENTMIN || function == PSEUDO_EXTENTID) + if (function == PSEUDO_EXTENTMAX || function == PSEUDO_EXTENTMIN) + { + if (!colType.isWideDecimalType()) + bs >> valueFromUM; + else + bs >> bigValueFromUM; + } + else if (function == PSEUDO_EXTENTID) + { bs >> valueFromUM; + if (colType.isWideDecimalType()) + bigValueFromUM = valueFromUM; + } + ColumnCommand::resetCommand(bs); } @@ -92,6 +105,10 @@ void PseudoCC::loadData() loadPMNumber(); break; + case 16: + loadPMNumber(); + break; + default: cout << "PC::loadData(): bad column width" << endl; break; @@ -118,6 +135,10 @@ void PseudoCC::loadData() loadRIDs(); break; + case 16: + loadRIDs(); + break; + default: cout << "PC::loadData(): bad column width" << endl; break; @@ -144,6 +165,10 @@ void PseudoCC::loadData() loadSegmentNum(); break; + case 16: + loadSegmentNum(); + break; + default: cout << "PC::loadData(): bad column width" << endl; break; @@ -170,6 +195,10 @@ void PseudoCC::loadData() loadPartitionNum(); break; + case 16: + loadPartitionNum(); + break; + default: cout << "PC::loadData(): bad column width" << endl; break; @@ -196,6 +225,10 @@ void PseudoCC::loadData() loadLBID(); break; + case 16: + loadLBID(); + break; + default: cout << "PC::loadData(): bad column width" << endl; break; @@ -222,6 +255,10 @@ void PseudoCC::loadData() loadDBRootNum(); break; + case 16: + loadDBRootNum(); + break; + default: cout << "PC::loadData(): bad column width" << endl; break; @@ -251,6 +288,10 @@ void PseudoCC::loadData() loadSingleValue(valueFromUM); break; + case 16: + loadSingleValue(bigValueFromUM); + break; + default: cout << "PC::loadData(): bad column width" << endl; break; diff --git a/primitives/primproc/pseudocc.h b/primitives/primproc/pseudocc.h index 1a4d84c44..d59cc3360 100644 --- a/primitives/primproc/pseudocc.h +++ b/primitives/primproc/pseudocc.h @@ -49,6 +49,7 @@ private: uint32_t function; uint64_t valueFromUM; + uint128_t bigValueFromUM; }; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 000000000..1b10181e8 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,31 @@ +include_directories( ${ENGINE_COMMON_INCLUDES} ) + +if (WITH_ROWGROUP_UT) + add_executable(rowgroup_tests rowgroup-tests.cpp) + target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) + install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) +endif() + +if (WITH_ARITHMETICOPERATOR_UT) + add_executable(arithmeticoperator_tests arithmeticoperator-tests.cpp) + target_link_libraries(arithmeticoperator_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) + install(TARGETS arithmeticoperator_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) +endif() + +if (WITH_CSDECIMAL_UT) + add_executable(mcs_decimal_tests mcs_decimal-tests.cpp) + target_link_libraries(mcs_decimal_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) + install(TARGETS mcs_decimal_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) +endif() + +if (WITH_DATACONVERT_UT) + add_executable(dataconvert_tests dataconvert-tests.cpp) + target_link_libraries(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) + install(TARGETS dataconvert_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) +endif() + +if (WITH_SORTING_COMPARATORS_UT) + add_executable(comparators_tests comparators-tests.cpp) + target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) + install(TARGETS comparators_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) +endif() diff --git a/tests/arithmeticoperator-tests.cpp b/tests/arithmeticoperator-tests.cpp new file mode 100644 index 000000000..5f883eb9d --- /dev/null +++ b/tests/arithmeticoperator-tests.cpp @@ -0,0 +1,26 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include // googletest header file +#include "mcs_basic_types.h" + +//using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; + +TEST(SimpleCheck, check1) +{ + EXPECT_EQ(true, true); +} diff --git a/utils/windowfunction/comparators-tests.cpp b/tests/comparators-tests.cpp similarity index 89% rename from utils/windowfunction/comparators-tests.cpp rename to tests/comparators-tests.cpp index d2d5df316..4b754dacf 100644 --- a/utils/windowfunction/comparators-tests.cpp +++ b/tests/comparators-tests.cpp @@ -36,7 +36,8 @@ #include #include #include "bytestream.h" -#include "idborderby.h" +#include "utils/windowfunction/idborderby.h" +#include "mcs_decimal.h" #define DEBUG #define MEMORY_LIMIT 14983602176 @@ -123,32 +124,37 @@ class FilterDriver : public CppUnit::TestFixture CPPUNIT_TEST(INT_TEST); CPPUNIT_TEST(FLOAT_TEST); + CPPUNIT_TEST(WIDEDT_TEST); CPPUNIT_TEST_SUITE_END(); private: // The tests creates an RG with 1 column of the cscDt type // then initialize RGData. After that it adds two numeric values (v1 < v2)and two NULL. - // Then creates comparator structores and run a number of tests. v1 < v2 - void testComparatorWithDT(execplan::CalpontSystemCatalog::ColDataType cscDt, + // Then creates comparator structures and run a number of tests. v1 < v2 + void testComparatorWithDT(execplan::CalpontSystemCatalog::ColDataType cscDt, uint32_t width, - bool generateRandValues) + bool generateRandValues, + uint8_t precision) { std::cout << std::endl << "------------------------------------------------------------" << std::endl; uint32_t oid =3001; - std::vector offsets, roids, tkeys, cscale, cprecision; + std::vector offsets, roids, tkeys, cscale, cprecision; + std::vector charSetNumVec; std::vector types; offsets.push_back(2); offsets.push_back(2+width); roids.push_back(oid); tkeys.push_back(1); types.push_back(cscDt); cscale.push_back(0); - cprecision.push_back(20); + cprecision.push_back(precision); + charSetNumVec.push_back(8); rowgroup::RowGroup inRG(1, //column count offsets, //oldOffset roids, // column oids tkeys, //keys types, // types + charSetNumVec, // charset numbers cscale, //scale cprecision, // precision 20, // sTableThreshold @@ -295,7 +301,7 @@ private: r.setUintField<4>(joblist::INTNULL, 0); break; } - default: + case 8 : { r.setUintField<8>(42, 0); r.nextRow(rowSize); @@ -304,6 +310,21 @@ private: r.setUintField<8>(joblist::BIGINTNULL, 0); break; } + case 16 : + { + uint128_t dec = 42; + r.setBinaryField(&dec, 0); + r.nextRow(rowSize); + dec++; + r.setBinaryField(&dec, 0); + r.nextRow(rowSize); + r.setBinaryField( + const_cast(&datatypes::Decimal128Null), + 0); + break; + } + default : + std::cout << " This is the end. My only friend the end... " << std::endl; } break; } @@ -387,27 +408,33 @@ private: { //bool generateValues = true; bool fixedValues = false; - testComparatorWithDT(execplan::CalpontSystemCatalog::UTINYINT, 1, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::USMALLINT, 2, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::UMEDINT, 4, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::UBIGINT, 8, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::DATETIME, 8, fixedValues); + testComparatorWithDT(execplan::CalpontSystemCatalog::UTINYINT, 1, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::USMALLINT, 2, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::UMEDINT, 4, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::UBIGINT, 8, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::DATETIME, 8, fixedValues, 20); - testComparatorWithDT(execplan::CalpontSystemCatalog::TINYINT, 1, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::SMALLINT, 2, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::MEDINT, 4, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::DATE, 4, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::BIGINT, 8, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::DECIMAL, 8, fixedValues); + testComparatorWithDT(execplan::CalpontSystemCatalog::TINYINT, 1, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::SMALLINT, 2, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::MEDINT, 4, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::DATE, 4, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::BIGINT, 8, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::DECIMAL, 8, fixedValues, 20); - testComparatorWithDT(execplan::CalpontSystemCatalog::FLOAT, 4, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::DOUBLE, 8, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::LONGDOUBLE, 8, fixedValues); + testComparatorWithDT(execplan::CalpontSystemCatalog::FLOAT, 4, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::DOUBLE, 8, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::LONGDOUBLE, 8, fixedValues, 20); } void FLOAT_TEST() { } + + void WIDEDT_TEST() + { + bool fixedValues = false; + testComparatorWithDT(execplan::CalpontSystemCatalog::DECIMAL, 16, fixedValues, 38); + } }; CPPUNIT_TEST_SUITE_REGISTRATION(FilterDriver); diff --git a/tests/dataconvert-tests.cpp b/tests/dataconvert-tests.cpp new file mode 100644 index 000000000..a6c7179b5 --- /dev/null +++ b/tests/dataconvert-tests.cpp @@ -0,0 +1,546 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include +using namespace std; + +#include "gtest/gtest.h" + +#include "dataconvert.h" +using namespace dataconvert; +#include "joblisttypes.h" +#include "columnwidth.h" + +TEST(DataConvertTest, Strtoll128) +{ + char *ep = NULL; + bool saturate = false; + string str; + int128_t val, valMax; + bitset<64> b1, b2, b3, b4; + + // test empty + str = ""; + val = strtoll128(str.c_str(), saturate, &ep); + EXPECT_EQ(val, 0); + EXPECT_EQ(*ep, '\0'); + EXPECT_FALSE(saturate); + + // test simple values + str = "123"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + EXPECT_EQ(val, 123); + EXPECT_EQ(*ep, '\0'); + EXPECT_FALSE(saturate); + str = " 123"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + EXPECT_EQ(val, 123); + EXPECT_EQ(*ep, '\0'); + EXPECT_FALSE(saturate); + str = " 123.45"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + EXPECT_EQ(val, 123); + EXPECT_NE(*ep, '\0'); + EXPECT_FALSE(saturate); + str = "-123"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + EXPECT_EQ(val, -123); + EXPECT_EQ(*ep, '\0'); + EXPECT_FALSE(saturate); + str = " -123"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + EXPECT_EQ(val, -123); + EXPECT_EQ(*ep, '\0'); + EXPECT_FALSE(saturate); + str = " -123.45"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + EXPECT_EQ(val, -123); + EXPECT_NE(*ep, '\0'); + EXPECT_FALSE(saturate); + + // test max/min values + // test max + str = "170141183460469231731687303715884105727"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + b1 = *(reinterpret_cast(&val)); + b2 = *(reinterpret_cast(&val) + 1); + valMax = ((((((((int128_t)170141183 * 1000000000) + 460469231) * 1000000000) + 731687303) * 1000000000 ) + 715884105) * 1000) + 727; + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_EQ(*ep, '\0'); + EXPECT_FALSE(saturate); + // test min + str = "-170141183460469231731687303715884105728"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + b1 = *(reinterpret_cast(&val)); + b2 = *(reinterpret_cast(&val) + 1); + valMax = ((((((((int128_t)170141183 * 1000000000) + 460469231) * 1000000000) + 731687303) * 1000000000 ) + 715884105) * 1000) + 727; + valMax = -valMax - 1; + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_EQ(*ep, '\0'); + EXPECT_FALSE(saturate); + + // test saturation + // test saturation to max + str = "170141183460469231731687303715884105728"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + b1 = *(reinterpret_cast(&val)); + b2 = *(reinterpret_cast(&val) + 1); + valMax = ((((((((int128_t)170141183 * 1000000000) + 460469231) * 1000000000) + 731687303) * 1000000000 ) + 715884105) * 1000) + 727; + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_EQ(*ep, '\0'); + EXPECT_TRUE(saturate); + // test saturation to min + str = "-170141183460469231731687303715884105729"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + b1 = *(reinterpret_cast(&val)); + b2 = *(reinterpret_cast(&val) + 1); + valMax = ((((((((int128_t)170141183 * 1000000000) + 460469231) * 1000000000) + 731687303) * 1000000000 ) + 715884105) * 1000) + 727; + valMax = -valMax - 1; + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_EQ(*ep, '\0'); + EXPECT_TRUE(saturate); +} + +TEST(DataConvertTest, NumberIntValue) +{ + datatypes::SystemCatalog::TypeAttributesStd ct; + + int128_t res, valMax; + string data; + bool noRoundup = false; + bool pushWarning; + bitset<64> b1, b2, b3, b4; + + // tests for signed decimal + // behaviour of number_int_value for unsigned decimal + // is similar to the signed case. + datatypes::SystemCatalog::ColDataType typecode = datatypes::SystemCatalog::DECIMAL; + // test with decimal(38,0) + ct.precision = 38; + ct.scale = 0; + // test simple values + //data = ""; + data = "0"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 0); + EXPECT_FALSE(pushWarning); + data = "1234"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 1234); + EXPECT_FALSE(pushWarning); + data = "12.0"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 12); + EXPECT_FALSE(pushWarning); + data = "12.34"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 12); + EXPECT_TRUE(pushWarning); + data = "-1234"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, -1234); + EXPECT_FALSE(pushWarning); + data = "-12.34"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, -12); + EXPECT_TRUE(pushWarning); + // test max + data = "99999999999999999999999999999999999999"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + // test min + data = "-99999999999999999999999999999999999999"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + valMax = -valMax; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + // test rounding + data = "12.56"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 13); + EXPECT_TRUE(pushWarning); + data = "-12.56"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, -13); + EXPECT_TRUE(pushWarning); + // test saturation + data = "999999999999999999999999999999999999999"; // data has 39 9's + // valMax has 38 9's + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + data = "-999999999999999999999999999999999999999"; // data has 39 9's + // valMax has 38 9's + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + valMax = -valMax; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + // test scientific notation + data = "1.23e37"; + valMax = ((((((((int128_t)123000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 100) + 0; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + data = "1.23e38"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + + // test with decimal(38,10) + ct.scale = 10; + data = "0"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 0); + EXPECT_FALSE(pushWarning); + data = "1234"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 12340000000000); + EXPECT_FALSE(pushWarning); + data = "12.0"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 120000000000); + EXPECT_FALSE(pushWarning); + data = "12.34"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 123400000000); + EXPECT_FALSE(pushWarning); + data = "-1234"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, -12340000000000); + EXPECT_FALSE(pushWarning); + data = "-12.34"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, -123400000000); + EXPECT_FALSE(pushWarning); + // test max + data = "9999999999999999999999999999.9999999999"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + // test min + data = "-9999999999999999999999999999.9999999999"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + valMax = -valMax; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + // test rounding + data = "12.11111111119"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 121111111112); + EXPECT_TRUE(pushWarning); + data = "-12.11111111119"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, -121111111112); + EXPECT_TRUE(pushWarning); + // test saturation + data = "99999999999999999999999999999"; // data has 29 9's + // valMax has 38 9's + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + data = "-99999999999999999999999999999"; // data has 29 9's + // valMax has 38 9's + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + valMax = -valMax; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + // test scientific notation + data = "1.23e9"; + valMax = ((((int128_t)123000000 * 1000000000) + 0) * 100); + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + data = "1.23e28"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + // test with decimal(38,38) + ct.scale = 38; + data = "0"; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 0); + EXPECT_FALSE(pushWarning); + data = "1.234"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + data = "0.123"; + valMax = ((((((((int128_t)123000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 100) + 0; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + data = "-1.234"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + valMax = -valMax; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + data = "-0.123"; + valMax = ((((((((int128_t)123000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 100) + 0; + valMax = -valMax; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + // test max + data = "0.99999999999999999999999999999999999999"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + // test min + data = "-0.99999999999999999999999999999999999999"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + valMax = -valMax; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + // test rounding + data = "0.199999999999999999999999999999999999999"; + valMax = ((((((((int128_t)200000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 100) + 0; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + data = "-0.199999999999999999999999999999999999999"; + valMax = ((((((((int128_t)200000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 100) + 0; + valMax = -valMax; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + // test saturation + data = "99999999999999999999999999999"; // data has 29 9's + // valMax has 38 9's + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + data = "-99999999999999999999999999999"; // data has 29 9's + // valMax has 38 9's + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + valMax = -valMax; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + // test scientific notation + data = "123e-4"; + valMax = ((((((((int128_t)123000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 10) + 0; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + data = "123e-2"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + pushWarning = false; + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); +} +TEST(DataConvertTest, ConvertColumnData) +{ +} diff --git a/tests/mcs_decimal-tests.cpp b/tests/mcs_decimal-tests.cpp new file mode 100644 index 000000000..5d5b29189 --- /dev/null +++ b/tests/mcs_decimal-tests.cpp @@ -0,0 +1,1126 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include + +#include "gtest/gtest.h" + +#include "treenode.h" +#include "mcs_decimal.h" +#include "dataconvert.h" + +TEST(Decimal, compareCheck) +{ + // L values = R value, L scale < R scale + execplan::IDB_Decimal l, r; + l.scale = 20; + l.precision = 38; + l.s128Value = 42; + + r.scale = 21; + l.precision = 38; + r.s128Value = 420; + EXPECT_EQ(0, datatypes::Decimal::compare(l, r)); + // L values = R value, L scale > R scale + l.scale = 21; + l.precision = 38; + l.s128Value = 420; + + r.scale = 20; + l.precision = 38; + r.s128Value = 42; + EXPECT_EQ(0, datatypes::Decimal::compare(l, r)); + // L values > R value, L scale < R scale + l.scale = 20; + l.precision = 38; + l.s128Value = 999999; + + r.scale = 21; + l.precision = 38; + r.s128Value = 420; + EXPECT_EQ(1, datatypes::Decimal::compare(l, r)); + // L values > R value, L scale > R scale + l.scale = 21; + l.precision = 38; + l.s128Value = 99999999; + + r.scale = 20; + l.precision = 38; + r.s128Value = 420; + EXPECT_EQ(1, datatypes::Decimal::compare(l, r)); + // L values < R value, L scale < R scale + l.scale = 20; + l.precision = 38; + l.s128Value = 99; + + r.scale = 21; + l.precision = 38; + r.s128Value = 42000; + EXPECT_EQ(-1, datatypes::Decimal::compare(l, r)); + // L values < R value, L scale > R scale + l.scale = 21; + l.precision = 38; + l.s128Value = 99; + + r.scale = 20; + l.precision = 38; + r.s128Value = 420; + EXPECT_EQ(-1, datatypes::Decimal::compare(l, r)); +} + +TEST(Decimal, additionNoOverflowCheck) +{ + // Addition w/o overflow check + execplan::IDB_Decimal l, r, result; + // same precision, same scale, both positive values + l.scale = 38; + l.precision = 38; + l.s128Value = 42; + + r.scale = 38; + r.precision = 38; + r.s128Value = 420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(462, result.s128Value); + + // same precision, same scale, both negative values + l.s128Value = -42; + r.s128Value = -420; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(-462, result.s128Value); + + // same precision, same scale, +- values + l.s128Value = 42; + r.s128Value = -420; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(-378, result.s128Value); + + // same precision, same scale, both 0 + l.s128Value = 0; + r.s128Value = 0; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(0, result.s128Value); + + // different scale + // same precision, L scale > R scale, both positive values + l.scale = 38; + l.precision = 38; + l.s128Value = 42; + + r.scale = 15; + r.precision = 38; + r.s128Value = 420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ("0.00000000000042000000000000000000000042", result.toString()); + + // same precision, L scale > R scale, both negative values + l.s128Value = -42; + r.s128Value = -420; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ("-0.00000000000042000000000000000000000042", result.toString()); + + // same precision, L scale > R scale, +- values + l.s128Value = 42; + r.s128Value = -420; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ("-0.00000000000041999999999999999999999958", result.toString()); + + // same precision, L scale > R scale, both 0 + l.s128Value = 0; + r.s128Value = 0; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(0, result.s128Value); + + // same precision, L scale < R scale, both positive values + l.scale = 15; + l.precision = 38; + l.s128Value = 42; + + r.scale = 38; + r.precision = 38; + r.s128Value = 420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ("0.00000000000004200000000000000000000420", result.toString()); + + // same precision, L scale < R scale, both negative values + l.s128Value = -42; + r.s128Value = -420; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ("-0.00000000000004200000000000000000000420", result.toString()); + + // same precision, L scale < R scale, +- values + l.s128Value = 42; + r.s128Value = -420; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ("0.00000000000004199999999999999999999580", result.toString()); + + // same precision, L scale < R scale, both 0 + l.s128Value = 0; + r.s128Value = 0; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(0, result.s128Value); +} + +TEST(Decimal, divisionNoOverflowCheck) +{ + // DIVISION + // same precision, same scale, both positive values + execplan::IDB_Decimal l, r, result; + + l.scale = 38; + l.precision = 38; + l.s128Value = 43; + + r.scale = 38; + r.precision = 38; + r.s128Value = 420; + + result.scale = 10; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("9.7674418605", result.toString()); + + // same precision, same scale, both negative values + l.s128Value = -43; + r.s128Value = -420; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("9.7674418605", result.toString()); + + // same precision, same scale, +- values + l.s128Value = 2200000; + r.s128Value = -1900; + result.s128Value = 0; + + datatypes::Decimal::division(l, r, result); + EXPECT_EQ("-1157.8947368421", result.toString()); + + // same precision, same scale, l = 0 + l.s128Value = 0; + r.s128Value = 42424242; + result.s128Value = 0; + + datatypes::Decimal::division(l, r, result); + EXPECT_EQ(0, result.s128Value); + + // diff scale + // same precision, L scale > R scale, both positive values + l.scale = 38; + l.s128Value = 19; + + r.scale = 15; + r.s128Value = 22; + + result.scale = 10; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("115789473684210526315789.4736842105", result.toString()); + + // same precision, L scale > R scale, both negative values + l.s128Value = -22; + r.s128Value = -19; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("86363636363636363636363.6363636364", result.toString()); + + // same precision, L scale > R scale, +- values + l.s128Value = 19; + r.s128Value = -22; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("-115789473684210526315789.4736842105", result.toString()); + + // same precision, L scale > R scale, R = 0 + l.s128Value = 424242; + r.s128Value = 0; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ(0, result.s128Value); + + // same precision, L scale > R scale, both MAX positive values + utils::int128Max(l.s128Value); + utils::int128Max(r.s128Value); + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("100000000000000000000000.0000000000", result.toString()); + + // same precision, L scale > R scale, both MIN negative values + utils::int128Min(l.s128Value); + utils::int128Min(r.s128Value); + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("100000000000000000000000.0000000000", result.toString()); + + // same precision, L scale < R scale, both positive values + l.scale = 15; + l.precision = 38; + l.s128Value = 42; + + r.scale = 38; + r.precision = 38; + r.s128Value = 42; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("0.00000000000000000000001000000000000000", result.toString()); + + // same precision, L scale < R scale, both negative values + l.s128Value = -22; + r.s128Value = -19; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("0.00000000000000000000000863636363636364", result.toString()); + + // same precision, L scale < R scale, +- values + l.s128Value = 22; + r.s128Value = -19; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("-0.00000000000000000000000863636363636364", result.toString()); + + // same precision, L scale < R scale, R = 0 + l.s128Value = 42; + r.s128Value = 0; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ(0, result.s128Value); + + // same precision, L scale < R scale, both MAX positive values + // WIP Investigate the next two + utils::int128Max(l.s128Value); + utils::int128Max(r.s128Value); + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("0.00000000000000000000001000000000000000", result.toString()); + + // same precision, L scale < R scale, both MIN negative values + utils::int128Min(l.s128Value); + utils::int128Min(r.s128Value); + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("0.00000000000000000000001000000000000000", result.toString()); + + // same precision, L scale < R scale, result.scale < (r.scale-l.scale) + // both positive values + l.scale = 37; + l.precision = 38; + l.s128Value = 43; + + r.scale = 38; + r.precision = 38; + r.s128Value = 420; + + result.scale = 0; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("1", result.toString()); + + // same precision, L scale < R scale, result.scale < (r.scale-l.scale) + // both negative values + l.s128Value = -22; + r.s128Value = -1900; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("9", result.toString()); + + // same precision, L scale < R scale, result.scale < (r.scale-l.scale) + // +- values + l.s128Value = 22; + r.s128Value = -1900; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("-9", result.toString()); + + // same precision, L scale < R scale, result.scale < (r.scale-l.scale) + // R = 0 + l.s128Value = 42; + r.s128Value = 0; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ(0, result.s128Value); + + // same precision, L scale < R scale, result.scale < (r.scale-l.scale) + // both MAX positive values + utils::int128Max(l.s128Value); + utils::int128Max(r.s128Value); + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("0", result.toString()); + + // same precision, L scale < R scale, result.scale < (r.scale-l.scale) + // both MIN negative values + utils::int128Min(l.s128Value); + utils::int128Min(r.s128Value); + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ("0", result.toString()); +} + +void doDiv(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, + execplan::IDB_Decimal& result) +{ + datatypes::Decimal::division(l, r, result); +} + +TEST(Decimal, divisionWithOverflowCheck) +{ + // Divide min int128 by -1 + execplan::IDB_Decimal l, r, result; + l.scale = 0; + l.precision = 38; + l.s128Value = datatypes::Decimal::minInt128; + + r.scale = 0; + r.precision = 38; + r.s128Value = -1; + + result.scale = 0; + result.precision = 38; + result.s128Value = 42; + EXPECT_THROW(doDiv(l, r, result), logging::OperationOverflowExcept); + + // Divide two ints one of which overflows after the scaling. + // TODO We currently do not test overflow due to scaling in + // case of division. Re-enable this test when we check for overflow + /*l.s128Value = datatypes::Decimal::maxInt128; + r.scale = 1; + r.s128Value = 42; + + result.scale = 1; + result.s128Value = 42; + EXPECT_THROW(doDiv(l, r, result), logging::OperationOverflowExcept);*/ + + // Normal execution w/o overflow + l.scale = 0; + l.s128Value = datatypes::Decimal::maxInt128 - 1; + + r.scale = 0; + r.s128Value = 0xFFFFFFFFFFFFFFFF; + + result.scale = 0; + result.s128Value = 0; + + EXPECT_NO_THROW(doDiv(l, r, result)); + EXPECT_EQ("9223372036854775809", result.toString()); +} + +void doAdd(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, + execplan::IDB_Decimal& result) +{ + datatypes::Decimal::addition(l, r, result); +} + +TEST(Decimal, additionWithOverflowCheck) +{ + // Add two max ints + execplan::IDB_Decimal l, r, result; + l.scale = 0; + l.precision = 38; + l.s128Value = datatypes::Decimal::maxInt128 - 1; + + r.scale = 0; + r.precision = 38; + r.s128Value = datatypes::Decimal::maxInt128 - 1; + + result.scale = 0; + result.precision = 38; + result.s128Value = 42; + EXPECT_THROW(doAdd(l, r, result), logging::OperationOverflowExcept); + + // Add two ints one of which overflows after the scaling. + l.s128Value = datatypes::Decimal::maxInt128 - 1; + r.scale = 1; + r.s128Value = 0xFFFFFFFFFFFFFFFF; + + result.scale = 1; + result.precision = 38; + result.s128Value = 0; + + EXPECT_THROW(doAdd(l, r, result), logging::OperationOverflowExcept); + + // Normal execution w/o overflow + l.scale = 0; + l.s128Value = datatypes::Decimal::minInt128; + + r.scale = 0; + r.s128Value = 0xFFFFFFFFFFFFFFFF; + + result.scale = 0; + result.s128Value = 0; + + EXPECT_NO_THROW(doAdd(l, r, result)); + EXPECT_EQ("-170141183460469231713240559642174554113", result.toString()); +} + +TEST(Decimal, subtractionNoOverflowCheck) +{ + // Subtractio w/o overflow check + execplan::IDB_Decimal l, r, result; + // same precision, same scale, both positive values + l.scale = 38; + l.precision = 38; + l.s128Value = 42; + + r.scale = 38; + r.precision = 38; + r.s128Value = 420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ("-0.00000000000000000000000000000000000378", result.toString()); + + // same precision, same scale, both negative values + l.s128Value = -42; + r.s128Value = -420; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ("0.00000000000000000000000000000000000378", result.toString()); + + // same precision, same scale, +- values + l.s128Value = 42; + r.s128Value = -420; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ("0.00000000000000000000000000000000000462", result.toString()); + + // same precision, same scale, both 0 + l.s128Value = 0; + r.s128Value = 0; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ(0, result.s128Value); + + // different scale + // same precision, L scale > R scale, both positive values + l.scale = 38; + l.precision = 38; + l.s128Value = 42; + + r.scale = 15; + r.precision = 38; + r.s128Value = 420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ("-0.00000000000041999999999999999999999958", result.toString()); + + // same precision, L scale > R scale, both negative values + l.s128Value = -42; + r.s128Value = -420; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ("0.00000000000041999999999999999999999958", result.toString()); + + // same precision, L scale > R scale, +- values + l.s128Value = 42; + r.s128Value = -420; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ("0.00000000000042000000000000000000000042", result.toString()); + + // same precision, L scale > R scale, both 0 + l.s128Value = 0; + r.s128Value = 0; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ(0, result.s128Value); + + // same precision, L scale < R scale, both positive values + l.scale = 15; + l.precision = 38; + l.s128Value = 42; + + r.scale = 38; + r.precision = 38; + r.s128Value = 420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ("0.00000000000004199999999999999999999580", result.toString()); + + // same precision, L scale < R scale, both negative values + l.s128Value = -42; + r.s128Value = -420; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ("-0.00000000000004199999999999999999999580", result.toString()); + + // same precision, L scale < R scale, +- values + l.s128Value = 42; + r.s128Value = -420; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ("0.00000000000004200000000000000000000420", result.toString()); + + // same precision, L scale < R scale, both 0 + l.s128Value = 0; + r.s128Value = 0; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ(0, result.s128Value); +} + +void doSubtract(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, + execplan::IDB_Decimal& result) +{ + datatypes::Decimal::subtraction(l, r, result); +} + +TEST(Decimal, subtractionWithOverflowCheck) +{ + // Subtract a max int from a min int + execplan::IDB_Decimal l, r, result; + l.scale = 0; + l.precision = 38; + l.s128Value = datatypes::Decimal::minInt128 + 1; + + r.scale = 0; + r.precision = 38; + r.s128Value = datatypes::Decimal::maxInt128 - 1; + + result.scale = 0; + result.precision = 38; + result.s128Value = 42; + EXPECT_THROW(doSubtract(l, r, result), logging::OperationOverflowExcept); + + // Subtract two ints one of which overflows after the scaling. + l.s128Value = datatypes::Decimal::minInt128 + 1; + r.scale = 1; + r.s128Value = 0xFFFFFFFFFFFFFFFF; + + result.scale = 1; + result.precision = 38; + result.s128Value = 0; + + EXPECT_THROW(doSubtract(l, r, result), logging::OperationOverflowExcept); + + // Normal execution w/o overflow + l.scale = 0; + l.s128Value = datatypes::Decimal::maxInt128; + + r.scale = 0; + r.s128Value = 0xFFFFFFFFFFFFFFFF; + + result.scale = 0; + result.s128Value = 0; + + EXPECT_NO_THROW(doSubtract(l, r, result)); + EXPECT_EQ("170141183460469231713240559642174554112", result.toString()); +} + +TEST(Decimal, multiplicationNoOverflowCheck) +{ + // Multiplication + // same precision, l.scale + r.scale = result.scale, both positive values + execplan::IDB_Decimal l, r, result; + + l.scale = 19; + l.precision = 38; + l.s128Value = 4611686018427387904; + + r.scale = 19; + r.precision = 38; + r.s128Value = UINT64_MAX; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::multiplication(r, l, result); + EXPECT_EQ("0.85070591730234615861231965839514664960", result.toString()); + + // same precision, l.scale + r.scale = result.scale, both negative values + l.s128Value = -4611686018427387904; + r.s128Value = UINT64_MAX; + r.s128Value = -r.s128Value; + result.s128Value = 0; + + datatypes::Decimal::multiplication(r, l, result); + EXPECT_EQ("0.85070591730234615861231965839514664960", result.toString()); + + // same precision, l.scale + r.scale = result.scale, +- values + l.s128Value = -4611686018427387904; + r.s128Value = UINT64_MAX; + result.s128Value = 0; + + datatypes::Decimal::multiplication(l, r, result); + EXPECT_EQ("-0.85070591730234615861231965839514664960", result.toString()); + + // same precision, l.scale + r.scale = result.scale, l = 0 + l.s128Value = 0; + r.s128Value = UINT64_MAX; + result.s128Value = 0; + + datatypes::Decimal::multiplication(l, r, result); + EXPECT_EQ(0, result.s128Value); + + // same precision, l.scale + r.scale < result.scale, both positive values + l.scale = 18; + l.precision = 38; + l.s128Value = 72057594037927936; + + r.scale = 18; + r.precision = 38; + r.s128Value = 9223372036854775808ULL; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::multiplication(r, l, result); + EXPECT_EQ("0.66461399789245793645190353014017228800", result.toString()); + + // same precision, l.scale + r.scale < result.scale, both negative values + l.s128Value = -72057594037927936; + r.s128Value = 9223372036854775808ULL; + r.s128Value = -r.s128Value; + result.s128Value = 0; + + datatypes::Decimal::multiplication(r, l, result); + EXPECT_EQ("0.66461399789245793645190353014017228800", result.toString()); + + // same precision, l.scale + r.scale < result.scale, +- values + l.s128Value = -72057594037927936; + r.s128Value = 9223372036854775808ULL; + result.s128Value = 0; + + datatypes::Decimal::multiplication(l, r, result); + EXPECT_EQ("-0.66461399789245793645190353014017228800", result.toString()); + + // same precision, l.scale + r.scale < result.scale, l = 0 + l.s128Value = 0; + r.s128Value = 9223372036854775808ULL; + result.s128Value = 0; + + datatypes::Decimal::multiplication(l, r, result); + EXPECT_EQ(0, result.s128Value); + + // same precision, l.scale + r.scale > result.scale, both positive values + l.scale = 38; + l.precision = 38; + l.s128Value = (((int128_t)1234567890123456789ULL * 10000000000000000000ULL) + + 1234567890123456789); + + r.scale = 38; + r.precision = 38; + r.s128Value = (((int128_t)1234567890123456789ULL * 10000000000000000000ULL) + + 1234567890123456789ULL); + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::multiplication(r, l, result); + EXPECT_EQ("0.01524157875323883675019051998750190521", result.toString()); + + // same precision, l.scale + r.scale > result.scale, both negative values + l.s128Value = -l.s128Value; + r.s128Value = -r.s128Value; + result.s128Value = 0; + + datatypes::Decimal::multiplication(r, l, result); + EXPECT_EQ("0.01524157875323883675019051998750190521", result.toString()); + + // same precision, l.scale + r.scale > result.scale, +- values + r.s128Value = -r.s128Value; + result.s128Value = 0; + + datatypes::Decimal::multiplication(l, r, result); + EXPECT_EQ("-0.01524157875323883675019051998750190521", result.toString()); + + // same precision, l.scale + r.scale > result.scale, l = 0 + r.s128Value = 0; + result.s128Value = 0; + + datatypes::Decimal::multiplication(l, r, result); + EXPECT_EQ(0, result.s128Value); +} + +void doMultiply(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, + execplan::IDB_Decimal& result) +{ + datatypes::Decimal::multiplication(l, r, result); +} + +TEST(Decimal, multiplicationWithOverflowCheck) +{ + execplan::IDB_Decimal l, r, result; + // result.scale >= l.scale + r.scale + l.scale = 0; + l.precision = 38; + l.s128Value = UINT64_MAX; + + r.scale = 0; + r.precision = 38; + r.s128Value = UINT64_MAX; + + result.scale = 0; + result.precision = 38; + result.s128Value = 42; + EXPECT_THROW(doMultiply(l, r, result), logging::OperationOverflowExcept); + + // result.scale < l.scale + r.scale + l.scale = 36; + l.precision = 38; + l.s128Value = (((int128_t)1234567890123456789ULL * 10000000000000000000ULL) + + 1234567890123456789); + + r.scale = 36; + r.precision = 38; + r.s128Value = (((int128_t)1234567890123456789ULL * 10000000000000000000ULL) + + 1234567890123456789); + + result.scale = 38; + result.precision = 38; + result.s128Value = 42; + EXPECT_THROW(doMultiply(l, r, result), logging::OperationOverflowExcept); + + // Normal execution w/o overflow + l.scale = 0; + l.s128Value = 4611686018427387904; + + r.scale = 0; + r.s128Value = 4611686018427387904; + + result.scale = 0; + result.s128Value = 0; + + EXPECT_NO_THROW(doMultiply(l, r, result)); + EXPECT_EQ("21267647932558653966460912964485513216", result.toString()); +} + +TEST(Decimal, DecimalToStringCheckScale0) +{ + string input, expected; + int128_t res; + int precision = 38; + int scale = 0; + res = 0; + datatypes::VDecimal dec(0, scale, precision, res); + + // test simple values + expected = "0"; + EXPECT_EQ(dec.toString(), expected); + res = 2; + expected = "2"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -2; + expected = "-2"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = 123; + expected = "123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -123; + expected = "-123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + // test max/min decimal (i.e. 38 9's) + res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + expected = "99999999999999999999999999999999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-99999999999999999999999999999999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + // test trailing zeros + res = 123000; + expected = "123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); +} +TEST(Decimal, DecimalToStringCheckScale10) +{ + string input, expected; + int128_t res; + int precision = 38; + int scale = 10; + res = 0; + datatypes::VDecimal dec(0, scale, precision, res); + + // test simple values + expected = "0.0000000000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + res = 2; + expected = "0.0000000002"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + res = -2; + expected = "-0.0000000002"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + res = 123; + expected = "0.0000000123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -123; + expected = "-0.0000000123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = 12345678901; + expected = "1.2345678901"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -12345678901; + expected = "-1.2345678901"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + // test max/min decimal (i.e. 38 9's) + res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + expected = "9999999999999999999999999999.9999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-9999999999999999999999999999.9999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + // test trailing zeros + res = 123000; + expected = "0.0000123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-0.0000123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + // test leading zeros + res = 10000000009; + expected = "1.0000000009"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-1.0000000009"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); +} + +TEST(Decimal, DecimalToStringCheckScale38) +{ + string input, expected; + int128_t res; + int precision = 38; + int scale = 38; + res = 0; + datatypes::VDecimal dec(0, scale, precision, res); + + // test simple values + res = 0; + expected = "0.00000000000000000000000000000000000000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = 2; + expected = "0.00000000000000000000000000000000000002"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -2; + expected = "-0.00000000000000000000000000000000000002"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = 123; + expected = "0.00000000000000000000000000000000000123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -123; + expected = "-0.00000000000000000000000000000000000123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = ((((((int128_t)1234567890 * 10000000000) + 1234567890) * 10000000000) + 1234567890) * 100000000 ) + 12345678; + expected = "0.12345678901234567890123456789012345678"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-0.12345678901234567890123456789012345678"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + // test max/min decimal (i.e. 38 9's) + res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + expected = "0.99999999999999999999999999999999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-0.99999999999999999999999999999999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + // test trailing zeros + res = 123000; + expected = "0.00000000000000000000000000000000123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-0.00000000000000000000000000000000123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); +} + +TEST(Decimal, DecimalToStringCheckScale37) +{ + string input, expected; + int128_t res; + int precision = 38; + int scale = 37; + res = 0; + datatypes::VDecimal dec(0, scale, precision, res); + + // test simple values + res = 0; + expected = "0.0000000000000000000000000000000000000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = 2; + expected = "0.0000000000000000000000000000000000002"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -2; + expected = "-0.0000000000000000000000000000000000002"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = 123; + expected = "0.0000000000000000000000000000000000123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -123; + expected = "-0.0000000000000000000000000000000000123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = ((((((int128_t)1234567890 * 10000000000) + 1234567890) * 10000000000) + 1234567890) * 100000000 ) + 12345678; + expected = "1.2345678901234567890123456789012345678"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-1.2345678901234567890123456789012345678"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + // test max/min decimal (i.e. 38 9's) + res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + expected = "9.9999999999999999999999999999999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-9.9999999999999999999999999999999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + // test trailing zeros + res = 123000; + expected = "0.0000000000000000000000000000000123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-0.0000000000000000000000000000000123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); +} + + diff --git a/tests/rowgroup-tests.cpp b/tests/rowgroup-tests.cpp new file mode 100644 index 000000000..cc031606b --- /dev/null +++ b/tests/rowgroup-tests.cpp @@ -0,0 +1,316 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include // googletest header file +#include + +#include "rowgroup.h" +#include "columnwidth.h" +#include "joblisttypes.h" +#include "dataconvert.h" + +#define WIDE_DEC_PRECISION 38U +#define INITIAL_ROW_OFFSET 2 + +using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; + +class RowDecimalTest : public ::testing::Test +{ +protected: + void SetUp() override + { + uint32_t precision = WIDE_DEC_PRECISION; + uint32_t oid = 3001; + + std::vector types; + std::vector precisionVec; + std::vector roids, tkeys, cscale; + std::vector widthVec; + std::vector charSetNumVec; + types.push_back(execplan::CalpontSystemCatalog::DECIMAL); + types.push_back(execplan::CalpontSystemCatalog::UDECIMAL); + + for (size_t i = 0; i <= 3; i++) + types.push_back(execplan::CalpontSystemCatalog::DECIMAL); + + precisionVec.push_back(precision); + precisionVec.push_back(precision); + precisionVec.push_back(18); + precisionVec.push_back(9); + precisionVec.push_back(4); + precisionVec.push_back(2); + + uint32_t offset = INITIAL_ROW_OFFSET; + offsets.push_back(offset); + + for (size_t i = 0; i < types.size(); i++) + { + uint8_t width = utils::widthByPrecision(precisionVec[i]); + widthVec.push_back(width); + offset += width; + offsets.push_back(offset); + roids.push_back(oid + i); + tkeys.push_back(i + 1); + cscale.push_back(0); + charSetNumVec.push_back(8); + } + + rowgroup::RowGroup inRG(roids.size(), // column count + offsets, // oldOffset + roids, // column oids + tkeys, // keys + types, // types + charSetNumVec, // charset numbers + cscale, // scale + precisionVec, // precision + 20, // sTableThreshold + false // useStringTable + ); + + rg = rgOut = inRG; + rgD.reinit(rg); + rgDOut.reinit(rgOut); + rg.setData(&rgD); + rgOut.setData(&rgDOut); + + rg.initRow(&r); + rg.initRow(&rOutMappingCheck); + rowSize = r.getSize(); + rg.getRow(0, &r); + + int128_t nullValue = 0; + int128_t bigValue = 0; + + datatypes::Decimal::setWideDecimalNullValue(nullValue); + bigValue = -static_cast(0xFFFFFFFF)*0xFFFFFFFFFFFFFFFF; + + sValueVector.push_back(nullValue); + sValueVector.push_back(-42); + sValueVector.push_back(bigValue); + sValueVector.push_back(0); + sValueVector.push_back(nullValue-1); + + uValueVector.push_back(nullValue); + uValueVector.push_back(42); + uValueVector.push_back(bigValue); + uValueVector.push_back(0); + uValueVector.push_back(nullValue-1); + + s8ValueVector.push_back(joblist::TINYINTNULL); + s8ValueVector.push_back(-0x79); + s8ValueVector.push_back(0); + s8ValueVector.push_back(0x81); + s8ValueVector.push_back(joblist::TINYINTNULL-1); + + s16ValueVector.push_back(joblist::SMALLINTNULL); + s16ValueVector.push_back(-0x79); + s16ValueVector.push_back(0); + s16ValueVector.push_back(0x81); + s16ValueVector.push_back(joblist::SMALLINTNULL-1); + + s32ValueVector.push_back(joblist::INTNULL); + s32ValueVector.push_back(-0x79); + s32ValueVector.push_back(0); + s32ValueVector.push_back(0x81); + s32ValueVector.push_back(joblist::INTNULL-1); + + s64ValueVector.push_back(joblist::BIGINTNULL); + s64ValueVector.push_back(-0x79); + s64ValueVector.push_back(0); + s64ValueVector.push_back(0x81); + s64ValueVector.push_back(joblist::BIGINTNULL-1); + + for (size_t i = 0; i < sValueVector.size(); i++) + { + r.setBinaryField_offset(&sValueVector[i], + sizeof(sValueVector[0]), offsets[0]); + r.setBinaryField_offset(&uValueVector[i], + sizeof(uValueVector[0]), offsets[1]); + r.setIntField(s64ValueVector[i], 2); + r.setIntField(s32ValueVector[i], 3); + r.setIntField(s16ValueVector[i], 4); + r.setIntField(s8ValueVector[i], 5); + r.nextRow(rowSize); + } + + rowCount = sValueVector.size(); + } + +// void TearDown() override {} + + rowgroup::Row r, rOut, sameRow, nonequiRow; + rowgroup::Row rOutMappingCheck; + rowgroup::RowGroup rg, rgOut; + rowgroup::RGData rgD, rgDOut; + uint32_t rowSize; + size_t rowCount; + std::vector sValueVector; + std::vector uValueVector; + std::vector s8ValueVector; + std::vector s16ValueVector; + std::vector s32ValueVector; + std::vector s64ValueVector; + std::vector offsets; +}; + +TEST_F(RowDecimalTest, NonNullValueCheck) +{ + rg.getRow(1, &r); + + for (size_t i = 1; i <= sValueVector.size(); i++) + { + EXPECT_FALSE(r.isNullValue(0)); + EXPECT_FALSE(r.isNullValue(1)); + EXPECT_FALSE(r.isNullValue(2)); + EXPECT_FALSE(r.isNullValue(3)); + EXPECT_FALSE(r.isNullValue(4)); + EXPECT_FALSE(r.isNullValue(5)); + r.nextRow(rowSize); + } +} + +TEST_F(RowDecimalTest, InitToNullAndIsNullValueCheck) +{ + rg.getRow(0, &r); + r.initToNull(); + + EXPECT_TRUE(r.isNullValue(0)); + EXPECT_TRUE(r.isNullValue(1)); + EXPECT_TRUE(r.isNullValue(2)); + EXPECT_TRUE(r.isNullValue(3)); + EXPECT_TRUE(r.isNullValue(4)); + EXPECT_TRUE(r.isNullValue(5)); +} + +TEST_F(RowDecimalTest, GetBinaryFieldCheck) +{ + rg.getRow(0, &r); + uint128_t* u128Value; + int128_t* s128Value; + + for (size_t i = 0; i < sValueVector.size(); i++) + { + s128Value = r.getBinaryField(0); + EXPECT_EQ(sValueVector[i], *s128Value); + u128Value = r.getBinaryField(1); + EXPECT_EQ(uValueVector[i], *u128Value); + //EXPECT_EQ(s64ValueVector[i], r.getIntField(2)); + //EXPECT_EQ(s32ValueVector[i],r.getIntField(3)); + //EXPECT_EQ(r.getIntField(4),s16ValueVector[i]); + //EXPECT_EQ(r.getIntField(5),s8ValueVector[i]); + r.nextRow(rowSize); + } +} + +TEST_F(RowDecimalTest, ToStringCheck) +{ + std::vector exemplarVector; + + exemplarVector.push_back(std::string("0: NULL NULL NULL NULL NULL NULL ")); + exemplarVector.push_back(std::string("0: -42 42 -121 -121 -121 -121 ")); + exemplarVector.push_back(std::string("0: -79228162495817593515539431425 -79228162495817593515539431425 0 0 0 0 ")); + exemplarVector.push_back(std::string("0: 0 0 129 129 129 -127 ")); + exemplarVector.push_back(std::string("0: 170141183460469231731687303715884105727 170141183460469231731687303715884105727 9223372036854775807 2147483647 32767 127 ")); + + rg.getRow(0, &r); + r.initToNull(); + + for (auto& el: exemplarVector) + { + EXPECT_EQ(el, r.toString()); + r.nextRow(rowSize); + } +} + +TEST_F(RowDecimalTest, ToCSVCheck) +{ +} + +TEST_F(RowDecimalTest, ApplyMappingCheck) +{ + int mapping[] = {0, 1, -1, -1, -1, -1}; + rg.getRow(1, &r); + rg.getRow(2, &rOutMappingCheck); + int128_t* s128Value = rOutMappingCheck.getBinaryField(0); + uint128_t* u128Value = rOutMappingCheck.getBinaryField(1); + EXPECT_NE(sValueVector[1], *s128Value); + EXPECT_NE(uValueVector[1], *u128Value); + applyMapping(mapping, r, &rOutMappingCheck); + s128Value = rOutMappingCheck.getBinaryField(0); + u128Value = rOutMappingCheck.getBinaryField(1); + EXPECT_EQ(sValueVector[1], *s128Value); + EXPECT_EQ(uValueVector[1], *u128Value); +} + +TEST_F(RowDecimalTest, CopyBinaryFieldCheck) +{ + int128_t constVal = 1; + int128_t *col1Out, *col1In; + uint128_t *col2Out, *col2In; + rgOut.getRow(0, &rOut); + + for (size_t i = 0; i < sValueVector.size(); i++) + { + rOut.setBinaryField_offset(&constVal, 16, offsets[0]); + rOut.setBinaryField_offset((uint128_t*)&constVal, 16, offsets[1]); + rOut.nextRow(rowSize); + } + + rgOut.initRow(&rOut); + rgOut.getRow(0, &rOut); + rg.getRow(0, &r); + + for (size_t i = 0; i < sValueVector.size(); i++) + { + col1In = r.getBinaryField(0); + col1Out = rOut.getBinaryField(0); + col2In = r.getBinaryField(1); + col2Out = rOut.getBinaryField(1); + EXPECT_NE(*col1In, *col1Out); + EXPECT_NE(*col2In, *col2Out); + r.copyBinaryField(rOut, 0, 0); + r.copyBinaryField(rOut, 1, 1); + col1Out = rOut.getBinaryField(0); + col2Out = rOut.getBinaryField(1); + EXPECT_EQ(*col1In, *col1Out); + EXPECT_EQ(*col2In, *col2Out); + r.nextRow(rowSize); + rOut.nextRow(rowSize); + } +} + +TEST_F(RowDecimalTest, RowEqualsCheck) +{ + rg.getRow(0, &r); + rg.getRow(0, &sameRow); + rg.getRow(1, &nonequiRow); + + for (size_t i = 0; i < sValueVector.size(); i++) + { + EXPECT_TRUE(r.equals(sameRow)); + if (i < sValueVector.size()-1) + { + EXPECT_FALSE(r.equals(nonequiRow)); + } + r.nextRow(rowSize); + sameRow.nextRow(rowSize); + if (i < sValueVector.size()-1) + { + nonequiRow.nextRow(rowSize); + } + } +} diff --git a/tools/editem/editem.cpp b/tools/editem/editem.cpp index 15d4d2a5f..e106db90d 100644 --- a/tools/editem/editem.cpp +++ b/tools/editem/editem.cpp @@ -44,6 +44,8 @@ using namespace config; #include "dataconvert.h" using namespace dataconvert; +#include "mcs_decimal.h" + #include "liboamcpp.h" #undef REALLY_DANGEROUS @@ -164,40 +166,76 @@ const string charcolToString(int64_t v) //------------------------------------------------------------------------------ // Formats an integer to it's date, datetime, or char equivalent //------------------------------------------------------------------------------ -const string fmt(int64_t v) +template +const string fmt(T v) { ostringstream oss; if (tflg) { - oss << DataConvert::dateToString(v); + oss << DataConvert::dateToString((int64_t) v); } else if (sflg) { - oss << DataConvert::datetimeToString(v); + oss << DataConvert::datetimeToString((int64_t) v); } else if (aflg) { - oss << charcolToString(v); + oss << charcolToString((int64_t) v); } else if (mflg) { - oss << v; + if (typeid(T) != typeid(int128_t)) + { + oss << (int64_t) v; + } + else + { + oss << datatypes::TSInt128(v); + } } else if (uflg) { - if (static_cast(v) > numeric_limits::max() - 2) - oss << "notset"; + if (typeid(T) != typeid(int128_t)) + { + if (static_cast(v) > numeric_limits::max() - 2) + oss << "notset"; + else + oss << static_cast(v); + } else - oss << static_cast(v); + { + if (v <= utils::minInt128 + 1) + { + oss << "notset"; + } + else + { + oss << datatypes::TSInt128(v); + } + } } else { - if (v == numeric_limits::max() || - v <= (numeric_limits::min() + 2)) - oss << "notset"; + if (typeid(T) != typeid(int128_t)) + { + if (v == numeric_limits::max() || + v <= (numeric_limits::min() + 1)) + oss << "notset"; + else + oss << (int64_t) v; + } else - oss << v; + { + if (v == utils::maxInt128 || (v <= utils::minInt128 + 1)) + { + oss << "notset"; + } + else + { + oss << datatypes::TSInt128(v); + } + } } return oss.str(); @@ -234,6 +272,8 @@ int dumpone(OID_t oid, unsigned int sortOrder) std::vector::iterator end; int64_t max; int64_t min; + int128_t bigMax; + int128_t bigMin; int32_t seqNum; bool header; bool needtrailer = false; @@ -262,8 +302,6 @@ int dumpone(OID_t oid, unsigned int sortOrder) while (iter != end) { uint32_t lbidRangeSize = iter->range.size * 1024; - max = iter->partition.cprange.hi_val; - min = iter->partition.cprange.lo_val; seqNum = iter->partition.cprange.sequenceNum; int state = iter->partition.cprange.isValid; @@ -287,10 +325,26 @@ int dumpone(OID_t oid, unsigned int sortOrder) if (vflg) cout << oid << ' '; - cout << iter->range.start << " - " << - (iter->range.start + lbidRangeSize - 1) << - " (" << lbidRangeSize << ") min: " << fmt(min) << - ", max: " << fmt(max) << ", seqNum: " << seqNum << ", state: "; + if (iter->colWid != datatypes::MAXDECIMALWIDTH) + { + max = iter->partition.cprange.hiVal; + min = iter->partition.cprange.loVal; + + cout << iter->range.start << " - " << + (iter->range.start + lbidRangeSize - 1) << + " (" << lbidRangeSize << ") min: " << fmt(min) << + ", max: " << fmt(max) << ", seqNum: " << seqNum << ", state: "; + } + else + { + bigMax = iter->partition.cprange.bigHiVal; + bigMin = iter->partition.cprange.bigLoVal; + + cout << iter->range.start << " - " << + (iter->range.start + lbidRangeSize - 1) << + " (" << lbidRangeSize << ") min: " << fmt(bigMin) << + ", max: " << fmt(bigMax) << ", seqNum: " << seqNum << ", state: "; + } switch (state) { @@ -408,11 +462,31 @@ int clearAllCPData() if (err == 0 && ranges.size() > 0) { + // Get the extents for a given OID to determine it's column width + std::vector entries; + CHECK(emp->getExtents(oid, entries, false, false, true)); + + if (entries.empty()) + continue; + + bool isBinaryColumn = (entries[0].colWid == datatypes::MAXDECIMALWIDTH); + BRM::CPInfo cpInfo; BRM::CPInfoList_t vCpInfo; - cpInfo.max = numeric_limits::min(); - cpInfo.min = numeric_limits::max(); + + if (!isBinaryColumn) + { + cpInfo.max = numeric_limits::min(); + cpInfo.min = numeric_limits::max(); + } + else + { + utils::int128Min(cpInfo.bigMax); + utils::int128Max(cpInfo.bigMin); + } + cpInfo.seqNum = -1; + cpInfo.isBinaryColumn = isBinaryColumn; for (uint32_t i = 0; i < ranges.size(); i++) { @@ -433,17 +507,38 @@ int clearAllCPData() //------------------------------------------------------------------------------ int clearmm(OID_t oid) { - BRM::LBIDRange_v ranges; CHECK(emp->lookup(oid, ranges)); BRM::LBIDRange_v::size_type rcount = ranges.size(); + // Get the extents for a given OID to determine it's column width + std::vector entries; + CHECK(emp->getExtents(oid, entries, false, false, true)); + if (entries.empty()) + { + cerr << "There are no entries in the Extent Map for OID: " << oid << endl; + return 1; + } + + bool isBinaryColumn = (entries[0].colWid == datatypes::MAXDECIMALWIDTH); + // @bug 2280. Changed to use the batch interface to clear the CP info to make the clear option faster. BRM::CPInfo cpInfo; BRM::CPInfoList_t vCpInfo; - cpInfo.max = numeric_limits::min(); - cpInfo.min = numeric_limits::max(); + + if (!isBinaryColumn) + { + cpInfo.max = numeric_limits::min(); + cpInfo.min = numeric_limits::max(); + } + else + { + utils::int128Min(cpInfo.bigMax); + utils::int128Max(cpInfo.bigMin); + } + cpInfo.seqNum = -1; + cpInfo.isBinaryColumn = isBinaryColumn; for (unsigned i = 0; i < rcount; i++) { diff --git a/utils/cloudio/CMakeLists.txt b/utils/cloudio/CMakeLists.txt index 1e00662e8..f94924b99 100755 --- a/utils/cloudio/CMakeLists.txt +++ b/utils/cloudio/CMakeLists.txt @@ -1,6 +1,6 @@ include_directories(${ENGINE_COMMON_INCLUDES} ${ENGINE_SRC_DIR}/storage-manager/include) -set(cloudio_LIB_SRCS SMComm.cpp SMDataFile.cpp SMFileFactory.cpp SMFileSystem.cpp SocketPool.cpp cloud_plugin.cpp) +set(cloudio_LIB_SRCS SMComm.cpp SMDataFile.cpp SMFileFactory.cpp SMFileSystem.cpp SocketPool.cpp cloud_plugin.cpp ../../datatypes/mcs_datatype.cpp) add_library(cloudio SHARED ${cloudio_LIB_SRCS}) diff --git a/utils/common/CMakeLists.txt b/utils/common/CMakeLists.txt index c6a0fde01..e3dd4607e 100644 --- a/utils/common/CMakeLists.txt +++ b/utils/common/CMakeLists.txt @@ -10,7 +10,8 @@ set(common_LIB_SRCS MonitorProcMem.cpp nullvaluemanip.cpp threadnaming.cpp - utils_utf8.cpp) + utils_utf8.cpp + emptyvaluemanip.cpp) add_library(common SHARED ${common_LIB_SRCS}) diff --git a/utils/common/any.hpp b/utils/common/any.hpp index 78c65ffb2..64b6e1bd4 100755 --- a/utils/common/any.hpp +++ b/utils/common/any.hpp @@ -9,9 +9,10 @@ * http://www.boost.org/LICENSE_1_0.txt */ -#include +#include #include #include +#include "mcs_basic_types.h" namespace static_any { @@ -123,7 +124,13 @@ namespace anyimpl typedef void type; }; - /// Specializations for small types. + /// Specializations for big types. +#define BIG_POLICY(TYPE) template<> struct \ + choose_policy { typedef big_any_policy type; }; + + BIG_POLICY(int128_t); + + /// Specializations for small types. #define SMALL_POLICY(TYPE) template<> struct \ choose_policy { typedef small_any_policy type; }; diff --git a/utils/common/checks.h b/utils/common/checks.h index 36b87312e..fb4a94e94 100644 --- a/utils/common/checks.h +++ b/utils/common/checks.h @@ -18,25 +18,33 @@ #define UTILS_COMMON_CHECKS_H #include +#include "mcs_int128.h" namespace utils { template -typename std::enable_if::value, bool>::type is_nonnegative(T) -{ return true; }; +typename std::enable_if::value || + datatypes::is_uint128_t::value, + bool>::type +is_nonnegative(T) { return true; }; template -typename std::enable_if::value, bool>::type is_nonnegative(T v) -{ return v >= 0; }; - +typename std::enable_if::value || + datatypes::is_int128_t::value, + bool>::type +is_nonnegative(T v) { return v >= 0; }; template -typename std::enable_if::value, bool>::type is_negative(T) -{ return false; }; +typename std::enable_if::value || + datatypes::is_uint128_t::value, + bool>::type +is_negative(T) { return false; }; template -typename std::enable_if::value, bool>::type is_negative(T v) -{ return v < 0; }; +typename std::enable_if::value || + datatypes::is_int128_t::value, + bool>::type +is_negative(T v) { return v < 0; }; } // namespace utils diff --git a/utils/common/columnwidth.h b/utils/common/columnwidth.h new file mode 100644 index 000000000..9d02a7312 --- /dev/null +++ b/utils/common/columnwidth.h @@ -0,0 +1,92 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#ifndef UTILS_COLWIDTH_H +#define UTILS_COLWIDTH_H + +#include "branchpred.h" + +namespace utils +{ + const uint8_t MAXLEGACYWIDTH = 8ULL; + const uint8_t MAXCOLUMNWIDTH = 16ULL; + + inline bool isWide(uint8_t width) + { + return width > MAXLEGACYWIDTH; + } + + inline bool isNarrow(uint8_t width) + { + return width <= MAXLEGACYWIDTH; + } + + /** @brief Map a DECIMAL precision to data width in bytes */ + inline uint8_t widthByPrecision(unsigned p) + { + if (LIKELY(p > 18 && p < 39)) + return 16; + + switch (p) + { + case 1: + case 2: + return 1; + + case 3: + case 4: + return 2; + + case 5: + case 6: + case 7: + case 8: + case 9: + return 4; + + case 10: + case 11: + case 12: + case 13: + case 14: + case 15: + case 16: + case 17: + case 18: + return 8; + + default: + return 16; + } + } + + inline uint8_t precisionByWidth(unsigned w) + { + switch(w) + { + case 16: + return 38; + // In case we will support decimals that spans 32 bytes. + default: + return 65; + } + } + + +} + +#endif // UTILS_COLWIDTH_H diff --git a/utils/common/emptyvaluemanip.cpp b/utils/common/emptyvaluemanip.cpp new file mode 100644 index 000000000..3300bc690 --- /dev/null +++ b/utils/common/emptyvaluemanip.cpp @@ -0,0 +1,114 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include "emptyvaluemanip.h" + +namespace utils +{ + +void getEmptyRowValue(const execplan::CalpontSystemCatalog::ColDataType colDataType, + const int width, uint8_t* emptyVal) +{ + switch (colDataType) + { + case execplan::CalpontSystemCatalog::TINYINT: + *(uint8_t*)emptyVal = joblist::TINYINTEMPTYROW; + break; + + case execplan::CalpontSystemCatalog::SMALLINT: + *(uint16_t*)emptyVal = joblist::SMALLINTEMPTYROW; + break; + + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + *(uint32_t*)emptyVal = joblist::INTEMPTYROW; + break; + + case execplan::CalpontSystemCatalog::BIGINT: + *(uint64_t*)emptyVal = joblist::BIGINTEMPTYROW; + break; + + case execplan::CalpontSystemCatalog::UTINYINT: + *(uint8_t*)emptyVal = joblist::UTINYINTEMPTYROW; + break; + + case execplan::CalpontSystemCatalog::USMALLINT: + *(uint16_t*)emptyVal = joblist::USMALLINTEMPTYROW; + break; + + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + *(uint32_t*)emptyVal = joblist::UINTEMPTYROW; + break; + + case execplan::CalpontSystemCatalog::UBIGINT: + *(uint64_t*)emptyVal = joblist::UBIGINTEMPTYROW; + break; + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + *(uint32_t*)emptyVal = joblist::FLOATEMPTYROW; + break; + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + *(uint64_t*)emptyVal = joblist::DOUBLEEMPTYROW; + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + if (width <= 1) + *(uint8_t*)emptyVal = joblist::TINYINTEMPTYROW; + else if (width <= 2) + *(uint16_t*)emptyVal = joblist::SMALLINTEMPTYROW; + else if (width <= 4) + *(uint32_t*)emptyVal = joblist::INTEMPTYROW; + else if (width <= 8) + *(uint64_t*)emptyVal = joblist::BIGINTEMPTYROW; + else + datatypes::Decimal::setWideDecimalEmptyValue(*(reinterpret_cast(emptyVal))); + break; + + //case CalpontSystemCatalog::BINARY: + // emptyVal = joblist::BINARYEMPTYROW; + // break; + + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::VARBINARY: + case execplan::CalpontSystemCatalog::BLOB: + case execplan::CalpontSystemCatalog::TEXT: + default: + *(uint8_t*)emptyVal = joblist::CHAR1EMPTYROW; + int offset = (colDataType == execplan::CalpontSystemCatalog::VARCHAR) ? -1 : 0; + + if (width == (2 + offset)) + *(uint16_t*)emptyVal = joblist::CHAR2EMPTYROW; + else if (width >= (3 + offset) && width <= (4 + offset)) + *(uint32_t*)emptyVal = joblist::CHAR4EMPTYROW; + else if (width >= (5 + offset)) + *(uint64_t*)emptyVal = joblist::CHAR8EMPTYROW; + + break; + } +} + +} // namespace utils diff --git a/utils/common/emptyvaluemanip.h b/utils/common/emptyvaluemanip.h new file mode 100644 index 000000000..2cf6222de --- /dev/null +++ b/utils/common/emptyvaluemanip.h @@ -0,0 +1,31 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#ifndef EMPTY_VALUE_MANIP_H +#define EMPTY_VALUE_MANIP_H + +#include "calpontsystemcatalog.h" + +namespace utils +{ + +void getEmptyRowValue(const execplan::CalpontSystemCatalog::ColDataType colDataType, + const int width, uint8_t* emptyVal); + +} // namespace utils + +#endif // EMPTY_VALUE_MANIP_H diff --git a/utils/common/hasher.h b/utils/common/hasher.h index 4978689c4..1633986a2 100644 --- a/utils/common/hasher.h +++ b/utils/common/hasher.h @@ -29,6 +29,7 @@ #include #include +#include "mcs_basic_types.h" namespace utils { @@ -346,6 +347,25 @@ public: } }; +// TODO a copy of these classes also exists in primitiveprocessor.h; consolidate +class Hash128 +{ +public: + inline size_t operator()(const int128_t i) const + { + return *reinterpret_cast(&i); + } +}; + +class Equal128 +{ +public: + inline bool operator()(const int128_t f1, const int128_t f2) const + { + return f1 == f2; + } +}; + //------------------------------------------------------------------------------ /** @brief class TupleHasher * diff --git a/utils/common/mcs_basic_types.h b/utils/common/mcs_basic_types.h new file mode 100644 index 000000000..131fcdc68 --- /dev/null +++ b/utils/common/mcs_basic_types.h @@ -0,0 +1,28 @@ +/* + Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. +*/ +#ifndef MCS_BASIC_TYPES_H_INCLUDED +#define MCS_BASIC_TYPES_H_INCLUDED + + +using int128_t = __int128; +using uint128_t = unsigned __int128; + + +#endif // MCS_BASIC_TYPES_H_INCLUDED +// vim:ts=2 sw=2: diff --git a/utils/common/nullvaluemanip.cpp b/utils/common/nullvaluemanip.cpp index 3b4255dea..adb8e108d 100644 --- a/utils/common/nullvaluemanip.cpp +++ b/utils/common/nullvaluemanip.cpp @@ -125,7 +125,10 @@ uint64_t getNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidth) case CalpontSystemCatalog::UBIGINT: return joblist::UBIGINTNULL; - + + case CalpontSystemCatalog::BINARY: + return joblist::BINARYNULLVALUELOW; + case CalpontSystemCatalog::VARBINARY: default: ostringstream os; @@ -198,7 +201,6 @@ int64_t getSignedNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidt default: throw logic_error("getSignedNullValue() Can't return the NULL string"); } - break; } @@ -216,8 +218,14 @@ int64_t getSignedNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidt case 4 : return (int64_t) ((int32_t) joblist::INTNULL); - default: + case 8: return joblist::BIGINTNULL; + + default: + ostringstream os; + os << "getSignedNullValue(): got bad column width (" << t << + "). Width=" << colWidth << endl; + throw logic_error(os.str()); } break; @@ -239,6 +247,9 @@ int64_t getSignedNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidt case CalpontSystemCatalog::LONGDOUBLE: return (int64_t)joblist::LONGDOUBLENULL; + case CalpontSystemCatalog::BINARY: + return (int64_t)joblist::BINARYNULLVALUELOW; + case CalpontSystemCatalog::VARBINARY: default: ostringstream os; diff --git a/utils/common/widedecimalutils.h b/utils/common/widedecimalutils.h new file mode 100644 index 000000000..7e931d7fe --- /dev/null +++ b/utils/common/widedecimalutils.h @@ -0,0 +1,54 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#ifndef WIDE_DECIMAL_UTILS_H +#define WIDE_DECIMAL_UTILS_H + +#include +#include "mcs_basic_types.h" + +namespace utils +{ + const uint64_t BINARYNULLVALUELOW = 0ULL; + const uint64_t BINARYNULLVALUEHIGH = 0x8000000000000000ULL; + const uint64_t BINARYEMPTYVALUELOW = 1ULL; + const uint64_t BINARYEMPTYVALUEHIGH = 0x8000000000000000ULL; + + const int128_t minInt128 = int128_t(0x8000000000000000LL) << 64; + const int128_t maxInt128 = (int128_t(0x7FFFFFFFFFFFFFFFLL) << 64) + 0xFFFFFFFFFFFFFFFFLL; + + inline void int128Max(int128_t& val) + { + uint64_t* ptr = reinterpret_cast(&val); + ptr[0] = 0xFFFFFFFFFFFFFFFF; + ptr[1] = 0x7FFFFFFFFFFFFFFF; + } + + inline void int128Min(int128_t& val) + { + val = int128_t(0x8000000000000000LL) << 64; + } + + inline void uint128Max(uint128_t& val) + { + uint64_t* ptr = reinterpret_cast(&val); + ptr[0] = 0xFFFFFFFFFFFFFFFF; + ptr[1] = 0xFFFFFFFFFFFFFFFF; + } +} + +#endif // WIDE_DECIMAL_UTILS_H diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index d920f5ab3..11c40c221 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -28,15 +28,14 @@ #include #include #include +#include using namespace std; #include #include using namespace boost::algorithm; #include -#include "calpontsystemcatalog.h" #include "calpontselectexecutionplan.h" #include "columnresult.h" -using namespace execplan; #include "joblisttypes.h" @@ -50,6 +49,7 @@ typedef uint32_t ulong; using namespace logging; + namespace { @@ -100,10 +100,18 @@ bool number_value ( const string& data ) return true; } -int64_t number_int_value(const string& data, - const CalpontSystemCatalog::ColType& ct, - bool& pushwarning, - bool noRoundup) +} // namespace anon + +namespace dataconvert +{ + +template +void number_int_value(const string& data, + cscDataType typeCode, + const datatypes::SystemCatalog::TypeAttributesStd& ct, + bool& pushwarning, + bool noRoundup, + T& intVal) { // copy of the original input string valStr(data); @@ -129,11 +137,13 @@ int64_t number_int_value(const string& data, if (boost::iequals(valStr, "true")) { - return 1; + intVal = 1; + return; } if (boost::iequals(valStr, "false")) { - return 0; + intVal = 0; + return; } // convert to fixed-point notation if input is in scientific notation @@ -149,9 +159,9 @@ int64_t number_int_value(const string& data, // get the exponent string exp = valStr.substr(epos + 1); bool overflow = false; - int64_t exponent = dataconvert::string_to_ll(exp, overflow); + T exponent = dataconvert::string_to_ll(exp, overflow); - // if the exponent can not be held in 64-bit, not supported or saturated. + // if the exponent can not be held in 64 or 128 bits, not supported or saturated. if (overflow) throw QueryDataExcept("value is invalid.", formatErr); @@ -262,7 +272,7 @@ int64_t number_int_value(const string& data, if (dp != string::npos) { //Check if need round up - int frac1 = dataconvert::string_to_ll(valStr.substr(dp + 1, 1), pushwarning); + int frac1 = dataconvert::string_to_ll(valStr.substr(dp + 1, 1), pushwarning); if ((!noRoundup) && frac1 >= 5) roundup = 1; @@ -278,18 +288,18 @@ int64_t number_int_value(const string& data, } } - int64_t intVal = dataconvert::string_to_ll(intStr, pushwarning); + intVal = dataconvert::string_to_ll(intStr, pushwarning); //@Bug 3350 negative value round up. intVal += intVal >= 0 ? roundup : -roundup; bool dummy = false; - int64_t frnVal = (frnStr.length() > 0) ? dataconvert::string_to_ll(frnStr, dummy) : 0; + T frnVal = (frnStr.length() > 0) ? dataconvert::string_to_ll(frnStr, dummy) : 0; if (frnVal != 0) pushwarning = true; - switch (ct.colDataType) + switch (typeCode) { - case CalpontSystemCatalog::TINYINT: + case datatypes::SystemCatalog::TINYINT: if (intVal < MIN_TINYINT) { intVal = MIN_TINYINT; @@ -303,7 +313,7 @@ int64_t number_int_value(const string& data, break; - case CalpontSystemCatalog::SMALLINT: + case datatypes::SystemCatalog::SMALLINT: if (intVal < MIN_SMALLINT) { intVal = MIN_SMALLINT; @@ -317,7 +327,7 @@ int64_t number_int_value(const string& data, break; - case CalpontSystemCatalog::MEDINT: + case datatypes::SystemCatalog::MEDINT: if (intVal < MIN_MEDINT) { intVal = MIN_MEDINT; @@ -331,7 +341,7 @@ int64_t number_int_value(const string& data, break; - case CalpontSystemCatalog::INT: + case datatypes::SystemCatalog::INT: if (intVal < MIN_INT) { intVal = MIN_INT; @@ -345,7 +355,7 @@ int64_t number_int_value(const string& data, break; - case CalpontSystemCatalog::BIGINT: + case datatypes::SystemCatalog::BIGINT: if (intVal < MIN_BIGINT) { intVal = MIN_BIGINT; @@ -354,31 +364,23 @@ int64_t number_int_value(const string& data, break; - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (ct.colWidth == 1) + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::UDECIMAL: + if (LIKELY(ct.colWidth == 16)) { - if (intVal < MIN_TINYINT) + int128_t tmp; + utils::int128Min(tmp); + if (intVal < tmp + 2) // + 2 for NULL and EMPTY values { - intVal = MIN_TINYINT; - pushwarning = true; - } - else if (intVal > MAX_TINYINT) - { - intVal = MAX_TINYINT; + intVal = tmp + 2; pushwarning = true; } } - else if (ct.colWidth == 2) + else if (ct.colWidth == 8) { - if (intVal < MIN_SMALLINT) + if (intVal < MIN_BIGINT) { - intVal = MIN_SMALLINT; - pushwarning = true; - } - else if (intVal > MAX_SMALLINT) - { - intVal = MAX_SMALLINT; + intVal = MIN_BIGINT; pushwarning = true; } } @@ -395,11 +397,29 @@ int64_t number_int_value(const string& data, pushwarning = true; } } - else if (ct.colWidth == 8) + else if (ct.colWidth == 2) { - if (intVal < MIN_BIGINT) + if (intVal < MIN_SMALLINT) { - intVal = MIN_BIGINT; + intVal = MIN_SMALLINT; + pushwarning = true; + } + else if (intVal > MAX_SMALLINT) + { + intVal = MAX_SMALLINT; + pushwarning = true; + } + } + else if (ct.colWidth == 1) + { + if (intVal < MIN_TINYINT) + { + intVal = MIN_TINYINT; + pushwarning = true; + } + else if (intVal > MAX_TINYINT) + { + intVal = MAX_TINYINT; pushwarning = true; } } @@ -411,12 +431,24 @@ int64_t number_int_value(const string& data, } // @ bug 3285 make sure the value is in precision range for decimal data type - if ( (ct.colDataType == CalpontSystemCatalog::DECIMAL) || - (ct.colDataType == CalpontSystemCatalog::UDECIMAL) || + if ( (typeCode == datatypes::SystemCatalog::DECIMAL) || + (typeCode == datatypes::SystemCatalog::UDECIMAL) || (ct.scale > 0)) { - int64_t rangeUp = columnstore_precision[ct.precision]; - int64_t rangeLow = -rangeUp; + T rangeUp, rangeLow; + + if (ct.precision < 19) + { + rangeUp = (T) columnstore_precision[ct.precision]; + } + else + { + bool dummy = false; + char *ep = NULL; + rangeUp = (T) dataconvert::strtoll128(columnstore_big_precision[ct.precision - 19].c_str(), dummy, &ep); + } + + rangeLow = -rangeUp; if (intVal > rangeUp) { @@ -429,12 +461,28 @@ int64_t number_int_value(const string& data, pushwarning = true; } } - - return intVal; } +// Explicit template instantiation +template +void number_int_value(const std::string& data, + cscDataType typeCode, + const datatypes::SystemCatalog::TypeAttributesStd& ct, + bool& pushwarning, + bool noRoundup, + int64_t& intVal); + +template +void number_int_value(const std::string& data, + cscDataType typeCode, + const datatypes::SystemCatalog::TypeAttributesStd& ct, + bool& pushwarning, + bool noRoundup, + int128_t& intVal); + uint64_t number_uint_value(const string& data, - const CalpontSystemCatalog::ColType& ct, + cscDataType typeCode, + const datatypes::SystemCatalog::TypeAttributesStd& ct, bool& pushwarning, bool noRoundup) { @@ -473,7 +521,7 @@ uint64_t number_uint_value(const string& data, // get the exponent string exp = valStr.substr(epos + 1); bool overflow = false; - int64_t exponent = dataconvert::string_to_ll(exp, overflow); + int64_t exponent = dataconvert::string_to_ll(exp, overflow); // if the exponent can not be held in 64-bit, not supported or saturated. if (overflow) @@ -539,9 +587,9 @@ uint64_t number_uint_value(const string& data, if (frnVal != 0) pushwarning = true; - switch (ct.colDataType) + switch (typeCode) { - case CalpontSystemCatalog::UTINYINT: + case datatypes::SystemCatalog::UTINYINT: if (uintVal > MAX_UTINYINT) { uintVal = MAX_UTINYINT; @@ -550,7 +598,7 @@ uint64_t number_uint_value(const string& data, break; - case CalpontSystemCatalog::USMALLINT: + case datatypes::SystemCatalog::USMALLINT: if (uintVal > MAX_USMALLINT) { uintVal = MAX_USMALLINT; @@ -559,7 +607,7 @@ uint64_t number_uint_value(const string& data, break; - case CalpontSystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UMEDINT: if (uintVal > MAX_UMEDINT) { uintVal = MAX_UMEDINT; @@ -568,7 +616,7 @@ uint64_t number_uint_value(const string& data, break; - case CalpontSystemCatalog::UINT: + case datatypes::SystemCatalog::UINT: if (uintVal > MAX_UINT) { uintVal = MAX_UINT; @@ -577,7 +625,7 @@ uint64_t number_uint_value(const string& data, break; - case CalpontSystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UBIGINT: if (uintVal > MAX_UBIGINT) { uintVal = MAX_UBIGINT; @@ -593,11 +641,6 @@ uint64_t number_uint_value(const string& data, return uintVal; } -} // namespace anon - -namespace dataconvert -{ - /** * This function reads a decimal value from a string. It will stop processing * in 3 cases: @@ -1162,610 +1205,424 @@ bool stringToTimestampStruct(const string& data, TimeStamp& timeStamp, const str } boost::any -DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, - const std::string& dataOrig, bool& pushWarning, const std::string& timeZone, bool nulFlag, bool noRoundup, bool isUpdate) +DataConvert::StringToBit(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const datatypes::ConvertFromStringParam &prm, + const std::string& dataOrig, + bool& pushWarning) +{ + std::string data(dataOrig); + unsigned int x = data.find("("); + + if (x <= data.length()) + { + data.replace ( x, 1, " "); + } + + x = data.find(")"); + + if (x <= data.length()) + { + data.replace (x, 1, " "); + } + + int64_t tmp = 0; + + number_int_value (data, datatypes::SystemCatalog::BIT, colType, pushWarning, prm.noRoundup(), tmp); + + if (tmp) + { + bool bitvalue; + + if (from_string(bitvalue, data, std::dec )) + { + boost::any value = bitvalue; + return value; + } + else + { + throw QueryDataExcept("range, valid value or conversion error on BIT type.", formatErr); + } + } + return boost::any(); +} + + +boost::any +DataConvert::StringToSDecimal(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const datatypes::ConvertFromStringParam &prm, + const std::string& data, bool& pushWarning) +{ + const cscDataType typeCode= datatypes::SystemCatalog::DECIMAL; + if (LIKELY(colType.colWidth == 16)) + { + int128_t val128; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val128); + boost::any value = (int128_t) val128; + return value; + } + else if (colType.colWidth == 8) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (long long) val64; + return value; + } + else if (colType.colWidth == 4) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (int) val64; + return value; + } + else if (colType.colWidth == 2) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (short) val64; + return value; + } + else if (colType.colWidth == 1) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (char) val64; + return value; + } + //else if (colType.colWidth == 32) + // value = data; + return boost::any(); +} + + +boost::any +DataConvert::StringToUDecimal(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const datatypes::ConvertFromStringParam &prm, + const std::string& data, bool& pushWarning) +{ + const cscDataType typeCode= datatypes::SystemCatalog::UDECIMAL; + + // UDECIMAL numbers may not be negative + if (LIKELY(colType.colWidth == 16)) + { + int128_t val128; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val128); + + if (val128 < 0 && + !datatypes::Decimal::isWideDecimalNullValue(val128) && + !datatypes::Decimal::isWideDecimalEmptyValue(val128)) + { + val128 = 0; + pushWarning = true; + } + + boost::any value = val128; + return value; + } + else if (colType.colWidth == 8) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + long long ival = static_cast(val64); + + if (ival < 0 && + ival != static_cast(joblist::BIGINTEMPTYROW) && + ival != static_cast(joblist::BIGINTNULL)) + { + ival = 0; + pushWarning = true; + } + + boost::any value = ival; + return value; + } + else if (colType.colWidth == 4) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + int ival = static_cast(val64); + + if (ival < 0 && + ival != static_cast(joblist::INTEMPTYROW) && + ival != static_cast(joblist::INTNULL)) + { + ival = 0; + pushWarning = true; + } + + boost::any value = ival; + return value; + } + else if (colType.colWidth == 2) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + short ival = (short) val64; + + if (ival < 0 && + ival != static_cast(joblist::SMALLINTEMPTYROW) && + ival != static_cast(joblist::SMALLINTNULL)) + { + ival = 0; + pushWarning = true; + } + + boost::any value = ival; + return value; + } + else if (colType.colWidth == 1) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + char ival = (char) val64; + + if (ival < 0 && + ival != static_cast(joblist::TINYINTEMPTYROW) && + ival != static_cast(joblist::TINYINTNULL)) + { + ival = 0; + pushWarning = true; + } + + boost::any value = ival; + return value; + } + return boost::any(); +} + + +boost::any +DataConvert::StringToFloat(cscDataType typeCode, + const std::string& dataOrig, + bool& pushWarning) { boost::any value; - std::string data( dataOrig ); - pushWarning = false; - CalpontSystemCatalog::ColDataType type = colType.colDataType; + std::string data(dataOrig); - //if ( !data.empty() ) - if (!nulFlag) + string::size_type x = data.find('('); + + if (x < string::npos) + data.erase(x, 1); + + x = data.find(')'); + + if (x < string::npos) + data.erase(x, 1); + + if ( number_value ( data ) ) { - switch (type) + float floatvalue; + errno = 0; +#ifdef _MSC_VER + double dval = strtod(data.c_str(), 0); + + if (dval > MAX_FLOAT) { - case CalpontSystemCatalog::BIT: - { - unsigned int x = data.find("("); - - if (x <= data.length()) - { - data.replace ( x, 1, " "); - } - - x = data.find(")"); - - if (x <= data.length()) - { - data.replace (x, 1, " "); - } - - if (number_int_value (data, colType, pushWarning, noRoundup)) - { - bool bitvalue; - - if (from_string(bitvalue, data, std::dec )) - { - value = bitvalue; - } - else - { - throw QueryDataExcept("range, valid value or conversion error on BIT type.", formatErr); - } - } - } - break; - - case CalpontSystemCatalog::TINYINT: - value = (char) number_int_value(data, colType, pushWarning, noRoundup); - break; - - case CalpontSystemCatalog::SMALLINT: - value = (short) number_int_value(data, colType, pushWarning, noRoundup); - break; - - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - value = (int) number_int_value(data, colType, pushWarning, noRoundup); - break; - - case CalpontSystemCatalog::BIGINT: - value = (long long) number_int_value(data, colType, pushWarning, noRoundup); - break; - - case CalpontSystemCatalog::DECIMAL: - if (colType.colWidth == 1) - value = (char) number_int_value(data, colType, pushWarning, noRoundup); - else if (colType.colWidth == 2) - value = (short) number_int_value(data, colType, pushWarning, noRoundup); - else if (colType.colWidth == 4) - value = (int) number_int_value(data, colType, pushWarning, noRoundup); - else if (colType.colWidth == 8) - value = (long long) number_int_value(data, colType, pushWarning, noRoundup); - - break; - - case CalpontSystemCatalog::UDECIMAL: - - // UDECIMAL numbers may not be negative - if (colType.colWidth == 1) - { - char ival = (char) number_int_value(data, colType, pushWarning, noRoundup); - - if (ival < 0 && - ival != static_cast(joblist::TINYINTEMPTYROW) && - ival != static_cast(joblist::TINYINTNULL)) - { - ival = 0; - pushWarning = true; - } - - value = ival; - } - else if (colType.colWidth == 2) - { - short ival = (short) number_int_value(data, colType, pushWarning, noRoundup); - - if (ival < 0 && - ival != static_cast(joblist::SMALLINTEMPTYROW) && - ival != static_cast(joblist::SMALLINTNULL)) - { - ival = 0; - pushWarning = true; - } - - value = ival; - } - else if (colType.colWidth == 4) - { - int ival = static_cast(number_int_value(data, colType, pushWarning, noRoundup)); - - if (ival < 0 && - ival != static_cast(joblist::INTEMPTYROW) && - ival != static_cast(joblist::INTNULL)) - { - ival = 0; - pushWarning = true; - } - - value = ival; - } - else if (colType.colWidth == 8) - { - long long ival = static_cast(number_int_value(data, colType, pushWarning, noRoundup)); - - if (ival < 0 && - ival != static_cast(joblist::BIGINTEMPTYROW) && - ival != static_cast(joblist::BIGINTNULL)) - { - ival = 0; - pushWarning = true; - } - - value = ival; - } - - break; - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - string::size_type x = data.find('('); - - if (x < string::npos) - data.erase(x, 1); - - x = data.find(')'); - - if (x < string::npos) - data.erase(x, 1); - - if ( number_value ( data ) ) - { - float floatvalue; - errno = 0; -#ifdef _MSC_VER - double dval = strtod(data.c_str(), 0); - - if (dval > MAX_FLOAT) - { - pushWarning = true; - floatvalue = MAX_FLOAT; - } - else if (dval < MIN_FLOAT) - { - pushWarning = true; - floatvalue = MIN_FLOAT; - } - else - { - floatvalue = (float)dval; - } - + pushWarning = true; + floatvalue = MAX_FLOAT; + } + else if (dval < MIN_FLOAT) + { + pushWarning = true; + floatvalue = MIN_FLOAT; + } + else + { + floatvalue = (float)dval; + } #else - floatvalue = strtof(data.c_str(), 0); + floatvalue = strtof(data.c_str(), 0); #endif - if (errno == ERANGE) - { - pushWarning = true; + if (errno == ERANGE) + { + pushWarning = true; #ifdef _MSC_VER - if ( abs(floatvalue) == HUGE_VAL ) + if ( abs(floatvalue) == HUGE_VAL ) #else - if ( abs(floatvalue) == HUGE_VALF ) + if ( abs(floatvalue) == HUGE_VALF ) #endif - { - if ( floatvalue > 0 ) - floatvalue = MAX_FLOAT; - else - floatvalue = MIN_FLOAT; - } - else - floatvalue = 0; - } - - if (floatvalue < 0.0 && type == CalpontSystemCatalog::UFLOAT && - floatvalue != joblist::FLOATEMPTYROW && floatvalue != joblist::FLOATNULL) - { - value = 0.0; - pushWarning = true; - } - - value = floatvalue; - } - else - throw QueryDataExcept("range, valid value or conversion error on FLOAT type.", formatErr); - } - break; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: { - string::size_type x = data.find('('); + if ( floatvalue > 0 ) + floatvalue = MAX_FLOAT; + else + floatvalue = MIN_FLOAT; + } + else + floatvalue = 0; + } - if (x < string::npos) - data.erase(x, 1); + if (floatvalue < 0.0 && + typeCode == datatypes::SystemCatalog::UFLOAT && + floatvalue != joblist::FLOATEMPTYROW && + floatvalue != joblist::FLOATNULL) + { + value = 0.0; // QQ: should it assign floatvalue? + pushWarning = true; + } - x = data.find(')'); + value = floatvalue; + } + else + throw QueryDataExcept("range, valid value or conversion error on FLOAT type.", formatErr); + return value; +} - if (x < string::npos) - data.erase(x, 1); - if ( number_value ( data ) ) - { - double doublevalue; - errno = 0; - doublevalue = strtod(data.c_str(), 0); - if (errno == ERANGE) - { - pushWarning = true; +boost::any +DataConvert::StringToDouble(cscDataType typeCode, + const std::string& dataOrig, + bool& pushWarning) +{ + boost::any value; + std::string data(dataOrig); + + string::size_type x = data.find('('); + + if (x < string::npos) + data.erase(x, 1); + + x = data.find(')'); + + if (x < string::npos) + data.erase(x, 1); + + if ( number_value ( data ) ) + { + double doublevalue; + errno = 0; + doublevalue = strtod(data.c_str(), 0); + + if (errno == ERANGE) + { + pushWarning = true; #ifdef _MSC_VER - if ( abs(doublevalue) == HUGE_VAL ) + if ( abs(doublevalue) == HUGE_VAL ) #else - if ( abs(doublevalue) == HUGE_VALL ) + if ( abs(doublevalue) == HUGE_VALL ) #endif - { - if ( doublevalue > 0 ) - value = MAX_DOUBLE; - else - value = MIN_DOUBLE; - } - else - value = 0; - } - else - value = doublevalue; - - if (doublevalue < 0.0 && type == CalpontSystemCatalog::UDOUBLE && - doublevalue != joblist::DOUBLEEMPTYROW && doublevalue != joblist::DOUBLENULL) - { - doublevalue = 0.0; - pushWarning = true; - } - } + { + if ( doublevalue > 0 ) + value = MAX_DOUBLE; else - { - throw QueryDataExcept("range, valid value or conversion error on DOUBLE type.", formatErr); - } + value = MIN_DOUBLE; } - break; + else + value = 0; + } + else + value = doublevalue; - case CalpontSystemCatalog::UTINYINT: - value = (uint8_t)number_uint_value(data, colType, pushWarning, noRoundup); - break; - - case CalpontSystemCatalog::USMALLINT: - value = (uint16_t)number_uint_value(data, colType, pushWarning, noRoundup); - break; - - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - value = (uint32_t)number_uint_value(data, colType, pushWarning, noRoundup); - break; - - case CalpontSystemCatalog::UBIGINT: - value = (uint64_t)number_uint_value(data, colType, pushWarning, noRoundup); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::TEXT: - { - //check data length - if ( data.length() > (unsigned int)colType.colWidth ) - { - data = data.substr(0, colType.colWidth); - pushWarning = true; - } - else - { - if ( (unsigned int)colType.colWidth > data.length()) - { - //Pad null character to the string - data.resize(colType.colWidth, 0); - } - } - - value = data; - } - break; - - case CalpontSystemCatalog::DATE: - { - Date aDay; - - if (stringToDateStruct(data, aDay)) - { - value = (*(reinterpret_cast (&aDay))); - } - else - { - value = (uint32_t) 0; - pushWarning = true; - } - } - break; - - case CalpontSystemCatalog::DATETIME: - { - DateTime aDatetime; - - if (stringToDatetimeStruct(data, aDatetime, 0)) - { - value = *(reinterpret_cast(&aDatetime)); - } - else - { - value = (uint64_t) 0; - pushWarning = true; - } - } - break; - - case CalpontSystemCatalog::TIME: - { - Time aTime; - - if (!stringToTimeStruct(data, aTime, colType.precision)) - { - pushWarning = true; - } - - value = (int64_t) * (reinterpret_cast(&aTime)); - } - break; - - case CalpontSystemCatalog::TIMESTAMP: - { - TimeStamp aTimestamp; - - if (!stringToTimestampStruct(data, aTimestamp, timeZone)) - { - pushWarning = true; - } - - value = (uint64_t) *(reinterpret_cast(&aTimestamp)); - } - break; - - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::CLOB: - value = data; - break; - - case CalpontSystemCatalog::VARBINARY: - value = data; - break; - - default: - throw QueryDataExcept("convertColumnData: unknown column data type.", dataTypeErr); - break; + if (doublevalue < 0.0 && + typeCode == datatypes::SystemCatalog::UDOUBLE && + doublevalue != joblist::DOUBLEEMPTYROW && + doublevalue != joblist::DOUBLENULL) + { + doublevalue = 0.0; // QQ: should it assign "value" ? + pushWarning = true; } } - else //null + else { - switch (type) - { - case CalpontSystemCatalog::BIT: - { - //TODO: How to communicate with write engine? - } - break; - - case CalpontSystemCatalog::TINYINT: - { - char tinyintvalue = joblist::TINYINTNULL; - value = tinyintvalue; - } - break; - - case CalpontSystemCatalog::SMALLINT: - { - short smallintvalue = joblist::SMALLINTNULL; - value = smallintvalue; - } - break; - - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - { - int intvalue = joblist::INTNULL; - value = intvalue; - } - break; - - case CalpontSystemCatalog::BIGINT: - { - long long bigint = joblist::BIGINTNULL; - value = bigint; - } - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - if (colType.colWidth == CalpontSystemCatalog::ONE_BYTE) - { - char tinyintvalue = joblist::TINYINTNULL; - value = tinyintvalue; - } - else if (colType.colWidth == CalpontSystemCatalog::TWO_BYTE) - { - short smallintvalue = joblist::SMALLINTNULL; - value = smallintvalue; - } - else if (colType.colWidth == CalpontSystemCatalog::FOUR_BYTE) - { - int intvalue = joblist::INTNULL; - value = intvalue; - } - else if (colType.colWidth == CalpontSystemCatalog::EIGHT_BYTE) - { - long long eightbyte = joblist::BIGINTNULL; - value = eightbyte; - } - else - { - WriteEngine::Token nullToken; - value = nullToken; - } - } - break; - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - uint32_t tmp = joblist::FLOATNULL; - float* floatvalue = (float*)&tmp; - value = *floatvalue; - } - break; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - uint64_t tmp = joblist::DOUBLENULL; - double* doublevalue = (double*)&tmp; - value = *doublevalue; - } - break; - - case CalpontSystemCatalog::DATE: - { - uint32_t d = joblist::DATENULL; - value = d; - } - break; - - case CalpontSystemCatalog::DATETIME: - { - uint64_t d = joblist::DATETIMENULL; - value = d; - } - break; - - case CalpontSystemCatalog::TIMESTAMP: - { - uint64_t d = joblist::TIMESTAMPNULL; - value = d; - } - break; - - case CalpontSystemCatalog::TIME: - { - uint64_t d = joblist::TIMENULL; - value = d; - } - break; - - case CalpontSystemCatalog::CHAR: - { - std::string charnull; - - if (colType.colWidth == 1) - { - //charnull = joblist::CHAR1NULL; - charnull = '\376'; - value = charnull; - } - else if (colType.colWidth == 2) - { - //charnull = joblist::CHAR2NULL; - charnull = "\377\376"; - value = charnull; - } - else if (( colType.colWidth < 5 ) && ( colType.colWidth > 2 )) - { - //charnull = joblist::CHAR4NULL; - charnull = "\377\377\377\376"; - value = charnull; - } - else if (( colType.colWidth < 9 ) && ( colType.colWidth > 4 )) - { - //charnull = joblist::CHAR8NULL; - charnull = "\377\377\377\377\377\377\377\376"; - value = charnull; - } - else - { - WriteEngine::Token nullToken; - value = nullToken; - } - } - break; - - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::TEXT: - { - std::string charnull; - - if (colType.colWidth == 1 ) - { - //charnull = joblist::CHAR2NULL; - charnull = "\377\376"; - value = charnull; - } - else if ((colType.colWidth < 4) && (colType.colWidth > 1)) - { - //charnull = joblist::CHAR4NULL; - charnull = "\377\377\377\376"; - value = charnull; - } - else if ((colType.colWidth < 8) && (colType.colWidth > 3)) - { - //charnull = joblist::CHAR8NULL; - charnull = "\377\377\377\377\377\377\377\376"; - value = charnull; - } - else if ( colType.colWidth > 7 ) - { - WriteEngine::Token nullToken; - value = nullToken; - } - } - break; - - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - { - WriteEngine::Token nullToken; - value = nullToken; - } - break; - - case CalpontSystemCatalog::UTINYINT: - { - uint8_t utinyintvalue = joblist::UTINYINTNULL; - value = utinyintvalue; - } - break; - - case CalpontSystemCatalog::USMALLINT: - { - uint16_t usmallintvalue = joblist::USMALLINTNULL; - value = usmallintvalue; - } - break; - - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - { - uint32_t uintvalue = joblist::UINTNULL; - value = uintvalue; - } - break; - - case CalpontSystemCatalog::UBIGINT: - { - uint64_t ubigint = joblist::UBIGINTNULL; - value = ubigint; - } - break; - - default: - throw QueryDataExcept("convertColumnData: unknown column data type.", dataTypeErr); - break; - - } + throw QueryDataExcept("range, valid value or conversion error on DOUBLE type.", formatErr); } - return value; } + +boost::any +DataConvert::StringToString(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const std::string& dataOrig, + bool& pushWarning) + +{ + std::string data(dataOrig); + //check data length + if ( data.length() > (unsigned int)colType.colWidth ) + { + data = data.substr(0, colType.colWidth); + pushWarning = true; + boost::any value = data; + return value; + } + if ( (unsigned int)colType.colWidth > data.length()) + { + //Pad null character to the string + data.resize(colType.colWidth, 0); + } + boost::any value = data; + return value; +} + + +boost::any +DataConvert::StringToDate(const std::string& data, bool& pushWarning) +{ + Date aDay; + + if (stringToDateStruct(data, aDay)) + { + boost::any value = (*(reinterpret_cast (&aDay))); + return value; + } + boost::any value = (uint32_t) 0; + pushWarning = true; + return value; +} + + +boost::any +DataConvert::StringToDatetime(const std::string& data, bool& pushWarning) +{ + DateTime aDatetime; + + if (stringToDatetimeStruct(data, aDatetime, 0)) // QQ: why 0? + { + boost::any value = *(reinterpret_cast(&aDatetime)); + return value; + } + boost::any value = (uint64_t) 0; + pushWarning = true; + return value; +} + + +boost::any +DataConvert::StringToTime(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const std::string& data, + bool& pushWarning) +{ + Time aTime; + + if (!stringToTimeStruct(data, aTime, colType.precision)) + { + pushWarning = true; + } + + boost::any value = (int64_t) * (reinterpret_cast(&aTime)); + return value; +} + + +boost::any +DataConvert::StringToTimestamp(const datatypes::ConvertFromStringParam &prm, + const std::string& data, + bool& pushWarning) +{ + TimeStamp aTimestamp; + + if (!stringToTimestampStruct(data, aTimestamp, prm.timeZone())) + { + pushWarning = true; + } + + boost::any value = (uint64_t) *(reinterpret_cast(&aTimestamp)); + return value; +} + + //------------------------------------------------------------------------------ // Convert date string to binary date. Used by BulkLoad. //------------------------------------------------------------------------------ @@ -2503,216 +2360,6 @@ std::string DataConvert::timeToString1( long long datetimevalue ) return buf; } -#if 0 -bool DataConvert::isNullData(ColumnResult* cr, int rownum, CalpontSystemCatalog::ColType colType) -{ - switch (colType.colDataType) - { - case CalpontSystemCatalog::TINYINT: - if (cr->GetData(rownum) == joblist::TINYINTNULL) - return true; - - return false; - - case CalpontSystemCatalog::SMALLINT: - if (cr->GetData(rownum) == joblist::SMALLINTNULL) - return true; - - return false; - - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - if (cr->GetData(rownum) == joblist::INTNULL) - return true; - - return false; - - case CalpontSystemCatalog::BIGINT: - if (cr->GetData(rownum) == static_cast(joblist::BIGINTNULL)) - return true; - - return false; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - if (colType.colWidth <= CalpontSystemCatalog::FOUR_BYTE) - { - if (cr->GetData(rownum) == joblist::SMALLINTNULL) - return true; - - return false; - } - else if (colType.colWidth <= 9) - { - if (cr->GetData(rownum) == joblist::INTNULL) - return true; - else return false; - } - else if (colType.colWidth <= 18) - { - if (cr->GetData(rownum) == static_cast(joblist::BIGINTNULL)) - return true; - - return false; - } - else - { - if (cr->GetStringData(rownum) == "\376\377\377\377\377\377\377\377") - return true; - - return false; - } - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - - //if (cr->GetStringData(rownum) == joblist::FLOATNULL) - if (cr->GetStringData(rownum).compare("null") == 0 ) - return true; - - return false; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - - //if (cr->GetStringData(rownum) == joblist::DOUBLENULL) - if (cr->GetStringData(rownum).compare("null") == 0 ) - return true; - - return false; - - case CalpontSystemCatalog::DATE: - if (cr->GetData(rownum) == joblist::DATENULL) - return true; - - return false; - - case CalpontSystemCatalog::DATETIME: - if (cr->GetData(rownum) == static_cast(joblist::DATETIMENULL)) - return true; - - return false; - - case CalpontSystemCatalog::CHAR: - { - std::string charnull; - - if ( cr->GetStringData(rownum) == "") - { - return true; - } - - if (colType.colWidth == 1) - { - if (cr->GetStringData(rownum) == "\376") - return true; - - return false; - } - else if (colType.colWidth == 2) - { - if (cr->GetStringData(rownum) == "\377\376") - return true; - - return false; - } - else if (( colType.colWidth < 5 ) && ( colType.colWidth > 2 )) - { - if (cr->GetStringData(rownum) == "\377\377\377\376") - return true; - - return false; - } - else if (( colType.colWidth < 9 ) && ( colType.colWidth > 4 )) - { - if (cr->GetStringData(rownum) == "\377\377\377\377\377\377\377\376") - return true; - - return false; - } - else - { - if (cr->GetStringData(rownum) == "\376\377\377\377\377\377\377\377") - return true; - - return false; - } - } - - case CalpontSystemCatalog::VARCHAR: - { - std::string charnull; - - if ( cr->GetStringData(rownum) == "") - { - return true; - } - - if (colType.colWidth == 1) - { - if (cr->GetStringData(rownum) == "\377\376") - return true; - - return false; - } - else if ((colType.colWidth < 4) && (colType.colWidth > 1)) - { - if (cr->GetStringData(rownum) == "\377\377\377\376") - return true; - - return false; - } - else if ((colType.colWidth < 8) && (colType.colWidth > 3)) - { - if (cr->GetStringData(rownum) == "\377\377\377\377\377\377\377\376") - return true; - - return false; - } - else - { - WriteEngine::Token nullToken; - - // bytes reversed - if (cr->GetStringData(rownum) == "\376\377\377\377\377\377\377\377") - return true; - - return false; - } - } - - case CalpontSystemCatalog::UTINYINT: - if (cr->GetData(rownum) == joblist::UTINYINTNULL) - return true; - - return false; - - case CalpontSystemCatalog::USMALLINT: - if (cr->GetData(rownum) == joblist::USMALLINTNULL) - return true; - - return false; - - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - if (cr->GetData(rownum) == joblist::UINTNULL) - return true; - - return false; - - case CalpontSystemCatalog::UBIGINT: - if (cr->GetData(rownum) == joblist::UBIGINTNULL) - return true; - - return false; - - default: - throw QueryDataExcept("convertColumnData: unknown column data type.", dataTypeErr); - } -} -#endif int64_t DataConvert::dateToInt(const string& date) { return stringToDate(date); @@ -3284,441 +2931,455 @@ int64_t DataConvert::stringToTime(const string& data) return *(reinterpret_cast(&atime)); } -CalpontSystemCatalog::ColType DataConvert::convertUnionColType(vector& types) + +void +DataConvert::joinColTypeForUnion(datatypes::SystemCatalog::TypeHolderStd &unionedType, + const datatypes::SystemCatalog::TypeHolderStd &type) { - idbassert(types.size()); - - CalpontSystemCatalog::ColType unionedType = types[0]; - - for (uint64_t i = 1; i < types.size(); i++) + // limited support for VARBINARY, no implicit conversion. + if (type.colDataType == datatypes::SystemCatalog::VARBINARY || + unionedType.colDataType == datatypes::SystemCatalog::VARBINARY) { - // limited support for VARBINARY, no implicit conversion. - if (types[i].colDataType == CalpontSystemCatalog::VARBINARY || - unionedType.colDataType == CalpontSystemCatalog::VARBINARY) + if (type.colDataType != unionedType.colDataType || + type.colWidth != unionedType.colWidth) + throw runtime_error("VARBINARY in UNION must be the same width."); + } + + switch (type.colDataType) + { + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UDECIMAL: { - if (types[i].colDataType != unionedType.colDataType || - types[i].colWidth != unionedType.colWidth) - throw runtime_error("VARBINARY in UNION must be the same width."); + switch (unionedType.colDataType) + { + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UDECIMAL: + if (type.colWidth > unionedType.colWidth) + { + unionedType.colDataType = type.colDataType; + unionedType.colWidth = type.colWidth; + } + + // If same size and result is signed but source is unsigned... + if (type.colWidth == unionedType.colWidth && !isUnsigned(unionedType.colDataType) && isUnsigned(type.colDataType)) + { + unionedType.colDataType = type.colDataType; + } + + if (type.colDataType == datatypes::SystemCatalog::DECIMAL || type.colDataType == datatypes::SystemCatalog::UDECIMAL) + { + unionedType.colDataType = datatypes::SystemCatalog::DECIMAL; + } + + if (type.precision > unionedType.precision) + unionedType.precision = type.precision; + + unionedType.scale = (type.scale > unionedType.scale) ? type.scale : unionedType.scale; + break; + + case datatypes::SystemCatalog::DATE: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.colWidth = 20; + break; + + case datatypes::SystemCatalog::TIME: + case datatypes::SystemCatalog::DATETIME: + case datatypes::SystemCatalog::TIMESTAMP: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.colWidth = 26; + break; + + case datatypes::SystemCatalog::CHAR: + if (unionedType.colWidth < 20) + unionedType.colWidth = 20; + + break; + + case datatypes::SystemCatalog::VARCHAR: + if (unionedType.colWidth < 21) + unionedType.colWidth = 21; + + break; + + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + case datatypes::SystemCatalog::LONGDOUBLE: + default: + break; + } + + break; } - switch (types[i].colDataType) + case datatypes::SystemCatalog::DATE: { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: + switch (unionedType.colDataType) { - switch (unionedType.colDataType) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: - if (types[i].colWidth > unionedType.colWidth) - { - unionedType.colDataType = types[i].colDataType; - unionedType.colWidth = types[i].colWidth; - } + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UDECIMAL: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + case datatypes::SystemCatalog::LONGDOUBLE: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.scale = 0; + unionedType.colWidth = 20; + break; - // If same size and result is signed but source is unsigned... - if (types[i].colWidth == unionedType.colWidth && !isUnsigned(unionedType.colDataType) && isUnsigned(types[i].colDataType)) - { - unionedType.colDataType = types[i].colDataType; - } + case datatypes::SystemCatalog::CHAR: + if (unionedType.colWidth < 10) + unionedType.colWidth = 10; - if (types[i].colDataType == CalpontSystemCatalog::DECIMAL || types[i].colDataType == CalpontSystemCatalog::UDECIMAL) - { - unionedType.colDataType = CalpontSystemCatalog::DECIMAL; - } + break; - unionedType.scale = (types[i].scale > unionedType.scale) ? types[i].scale : unionedType.scale; - break; + case datatypes::SystemCatalog::VARCHAR: + if (unionedType.colWidth < 11) + unionedType.colWidth = 11; - case CalpontSystemCatalog::DATE: - unionedType.colDataType = CalpontSystemCatalog::CHAR; + break; + + case datatypes::SystemCatalog::DATE: + case datatypes::SystemCatalog::DATETIME: + case datatypes::SystemCatalog::TIMESTAMP: + case datatypes::SystemCatalog::TIME: + default: + break; + } + + break; + } + + case datatypes::SystemCatalog::DATETIME: + { + switch (unionedType.colDataType) + { + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UDECIMAL: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + case datatypes::SystemCatalog::TIME: + case datatypes::SystemCatalog::LONGDOUBLE: + case datatypes::SystemCatalog::TIMESTAMP: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.scale = 0; + unionedType.colWidth = 26; + break; + + case datatypes::SystemCatalog::DATE: + unionedType.colDataType = datatypes::SystemCatalog::DATETIME; + unionedType.colWidth = type.colWidth; + break; + + case datatypes::SystemCatalog::CHAR: + if (unionedType.colWidth < 26) + unionedType.colWidth = 26; + + break; + + case datatypes::SystemCatalog::VARCHAR: + if (unionedType.colWidth < 27) + unionedType.colWidth = 27; + + break; + + case datatypes::SystemCatalog::DATETIME: + default: + break; + } + + break; + } + + case datatypes::SystemCatalog::TIMESTAMP: + { + switch (unionedType.colDataType) + { + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UDECIMAL: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + case datatypes::SystemCatalog::TIME: + case datatypes::SystemCatalog::DATETIME: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.scale = 0; + unionedType.colWidth = 26; + break; + + case datatypes::SystemCatalog::DATE: + unionedType.colDataType = datatypes::SystemCatalog::TIMESTAMP; + unionedType.colWidth = type.colWidth; + break; + + case datatypes::SystemCatalog::CHAR: + if (unionedType.colWidth < 26) + unionedType.colWidth = 26; + + break; + + case datatypes::SystemCatalog::VARCHAR: + if (unionedType.colWidth < 27) + unionedType.colWidth = 27; + + break; + + case datatypes::SystemCatalog::TIMESTAMP: + default: + break; + } + + break; + } + + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + { + switch (unionedType.colDataType) + { + case datatypes::SystemCatalog::DATE: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.scale = 0; + unionedType.colWidth = 20; + break; + + case datatypes::SystemCatalog::DATETIME: + case datatypes::SystemCatalog::TIMESTAMP: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.scale = 0; + unionedType.colWidth = 26; + break; + + case datatypes::SystemCatalog::CHAR: + if (unionedType.colWidth < 20) unionedType.colWidth = 20; - break; - case CalpontSystemCatalog::TIME: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - unionedType.colDataType = CalpontSystemCatalog::CHAR; - unionedType.colWidth = 26; - break; + break; - case CalpontSystemCatalog::CHAR: - if (unionedType.colWidth < 20) - unionedType.colWidth = 20; + case datatypes::SystemCatalog::VARCHAR: + if (unionedType.colWidth < 21) + unionedType.colWidth = 21; - break; + break; - case CalpontSystemCatalog::VARCHAR: - if (unionedType.colWidth < 21) - unionedType.colWidth = 21; + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + unionedType.colDataType = datatypes::SystemCatalog::DOUBLE; + unionedType.scale = 0; + unionedType.colWidth = sizeof(double); + break; - break; - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UFLOAT: - case CalpontSystemCatalog::UDOUBLE: - case CalpontSystemCatalog::LONGDOUBLE: - default: - break; - } - - break; - } - - case CalpontSystemCatalog::DATE: - { - switch (unionedType.colDataType) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: - case CalpontSystemCatalog::UFLOAT: - case CalpontSystemCatalog::UDOUBLE: - case CalpontSystemCatalog::LONGDOUBLE: - unionedType.colDataType = CalpontSystemCatalog::CHAR; - unionedType.scale = 0; - unionedType.colWidth = 20; - break; - - case CalpontSystemCatalog::CHAR: - if (unionedType.colWidth < 10) - unionedType.colWidth = 10; - - break; - - case CalpontSystemCatalog::VARCHAR: - if (unionedType.colWidth < 11) - unionedType.colWidth = 11; - - break; - - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: - default: - break; - } - - break; - } - - case CalpontSystemCatalog::DATETIME: - { - switch (unionedType.colDataType) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: - case CalpontSystemCatalog::UFLOAT: - case CalpontSystemCatalog::UDOUBLE: - case CalpontSystemCatalog::TIME: - case CalpontSystemCatalog::LONGDOUBLE: - case CalpontSystemCatalog::TIMESTAMP: - unionedType.colDataType = CalpontSystemCatalog::CHAR; - unionedType.scale = 0; - unionedType.colWidth = 26; - break; - - case CalpontSystemCatalog::DATE: - unionedType.colDataType = CalpontSystemCatalog::DATETIME; - unionedType.colWidth = types[i].colWidth; - break; - - case CalpontSystemCatalog::CHAR: - if (unionedType.colWidth < 26) - unionedType.colWidth = 26; - - break; - - case CalpontSystemCatalog::VARCHAR: - if (unionedType.colWidth < 27) - unionedType.colWidth = 27; - - break; - - case CalpontSystemCatalog::DATETIME: - default: - break; - } - - break; - } - - case CalpontSystemCatalog::TIMESTAMP: - { - switch (unionedType.colDataType) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: - case CalpontSystemCatalog::UFLOAT: - case CalpontSystemCatalog::UDOUBLE: - case CalpontSystemCatalog::TIME: - case CalpontSystemCatalog::DATETIME: - unionedType.colDataType = CalpontSystemCatalog::CHAR; - unionedType.scale = 0; - unionedType.colWidth = 26; - break; - - case CalpontSystemCatalog::DATE: - unionedType.colDataType = CalpontSystemCatalog::TIMESTAMP; - unionedType.colWidth = types[i].colWidth; - break; - - case CalpontSystemCatalog::CHAR: - if (unionedType.colWidth < 26) - unionedType.colWidth = 26; - - break; - - case CalpontSystemCatalog::VARCHAR: - if (unionedType.colWidth < 27) - unionedType.colWidth = 27; - - break; - - case CalpontSystemCatalog::TIMESTAMP: - default: - break; - } - - break; - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UFLOAT: - case CalpontSystemCatalog::UDOUBLE: - { - switch (unionedType.colDataType) - { - case CalpontSystemCatalog::DATE: - unionedType.colDataType = CalpontSystemCatalog::CHAR; - unionedType.scale = 0; - unionedType.colWidth = 20; - break; - - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - unionedType.colDataType = CalpontSystemCatalog::CHAR; - unionedType.scale = 0; - unionedType.colWidth = 26; - break; - - case CalpontSystemCatalog::CHAR: - if (unionedType.colWidth < 20) - unionedType.colWidth = 20; - - break; - - case CalpontSystemCatalog::VARCHAR: - if (unionedType.colWidth < 21) - unionedType.colWidth = 21; - - break; - - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: - case CalpontSystemCatalog::UFLOAT: - case CalpontSystemCatalog::UDOUBLE: - unionedType.colDataType = CalpontSystemCatalog::DOUBLE; + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::UDECIMAL: + if (unionedType.colWidth != datatypes::MAXDECIMALWIDTH) + { + unionedType.colDataType = datatypes::SystemCatalog::DOUBLE; unionedType.scale = 0; unionedType.colWidth = sizeof(double); - break; + } + break; - default: - break; - } - - break; + default: + break; } - case CalpontSystemCatalog::LONGDOUBLE: + break; + } + + case datatypes::SystemCatalog::LONGDOUBLE: + { + switch (unionedType.colDataType) { - switch (unionedType.colDataType) - { - case CalpontSystemCatalog::DATE: - unionedType.colDataType = CalpontSystemCatalog::CHAR; - unionedType.scale = 0; + case datatypes::SystemCatalog::DATE: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.scale = 0; + unionedType.colWidth = 20; + break; + + case datatypes::SystemCatalog::DATETIME: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.scale = 0; + unionedType.colWidth = 26; + break; + + case datatypes::SystemCatalog::CHAR: + if (unionedType.colWidth < 20) unionedType.colWidth = 20; - break; - case CalpontSystemCatalog::DATETIME: - unionedType.colDataType = CalpontSystemCatalog::CHAR; - unionedType.scale = 0; - unionedType.colWidth = 26; - break; + break; - case CalpontSystemCatalog::CHAR: - if (unionedType.colWidth < 20) - unionedType.colWidth = 20; + case datatypes::SystemCatalog::VARCHAR: + if (unionedType.colWidth < 21) + unionedType.colWidth = 21; - break; + break; - case CalpontSystemCatalog::VARCHAR: - if (unionedType.colWidth < 21) - unionedType.colWidth = 21; + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + case datatypes::SystemCatalog::LONGDOUBLE: + unionedType.colDataType = datatypes::SystemCatalog::LONGDOUBLE; + unionedType.scale = (type.scale > unionedType.scale) ? type.scale : unionedType.scale; + unionedType.colWidth = sizeof(long double); + unionedType.precision = -1; + break; - break; - - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: - case CalpontSystemCatalog::UFLOAT: - case CalpontSystemCatalog::UDOUBLE: - case CalpontSystemCatalog::LONGDOUBLE: - unionedType.colDataType = CalpontSystemCatalog::LONGDOUBLE; - unionedType.scale = (types[i].scale > unionedType.scale) ? types[i].scale : unionedType.scale; + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::UDECIMAL: + if (unionedType.colWidth != datatypes::MAXDECIMALWIDTH) + { + unionedType.colDataType = datatypes::SystemCatalog::LONGDOUBLE; + unionedType.scale = (type.scale > unionedType.scale) ? type.scale : unionedType.scale; unionedType.colWidth = sizeof(long double); unionedType.precision = -1; - break; + } + break; - default: - break; - } - - break; + default: + break; } - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: + break; + } + + case datatypes::SystemCatalog::CHAR: + case datatypes::SystemCatalog::VARCHAR: + { + switch (unionedType.colDataType) { - switch (unionedType.colDataType) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: - case CalpontSystemCatalog::UFLOAT: - case CalpontSystemCatalog::UDOUBLE: - case CalpontSystemCatalog::LONGDOUBLE: - unionedType.scale = 0; - unionedType.colWidth = (types[i].colWidth > 20) ? types[i].colWidth : 20; - break; + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UDECIMAL: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + case datatypes::SystemCatalog::LONGDOUBLE: + unionedType.scale = 0; + unionedType.colWidth = (type.colWidth > 20) ? type.colWidth : 20; + break; - case CalpontSystemCatalog::DATE: - unionedType.colWidth = (types[i].colWidth > 10) ? types[i].colWidth : 10; - break; + case datatypes::SystemCatalog::DATE: + unionedType.colWidth = (type.colWidth > 10) ? type.colWidth : 10; + break; - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - unionedType.colWidth = (types[i].colWidth > 26) ? types[i].colWidth : 26; - break; + case datatypes::SystemCatalog::DATETIME: + case datatypes::SystemCatalog::TIMESTAMP: + unionedType.colWidth = (type.colWidth > 26) ? type.colWidth : 26; + break; - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: + case datatypes::SystemCatalog::CHAR: + case datatypes::SystemCatalog::VARCHAR: - // VARCHAR will fit in CHAR of the same width - if (unionedType.colWidth < types[i].colWidth) - unionedType.colWidth = types[i].colWidth; + // VARCHAR will fit in CHAR of the same width + if (unionedType.colWidth < type.colWidth) + unionedType.colWidth = type.colWidth; - break; + break; - default: - break; - } - - // MariaDB bug 651. Setting to CHAR broke union in subquery - unionedType.colDataType = CalpontSystemCatalog::VARCHAR; - break; + default: + break; } - default: - { - break; - } - } // switch - } // for + // MariaDB bug 651. Setting to CHAR broke union in subquery + unionedType.colDataType = datatypes::SystemCatalog::VARCHAR; + break; + } - return unionedType; + default: + { + break; + } + } // switch } } // namespace dataconvert diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index c2d27b1b8..5cbd28929 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -47,9 +47,11 @@ #include #endif -#include "calpontsystemcatalog.h" +#include "mcs_datatype.h" #include "columnresult.h" #include "exceptclasses.h" +#include "common/branchpred.h" + // remove this block if the htonll is defined in library #ifdef __linux__ @@ -84,6 +86,7 @@ inline uint64_t uint64ToStr(uint64_t n) return htonll(n); } +using cscDataType = datatypes::SystemCatalog::ColDataType; #if defined(_MSC_VER) && defined(xxxDATACONVERT_DLLEXPORT) #define EXPORT __declspec(dllexport) @@ -114,6 +117,29 @@ const int64_t IDB_pow[19] = 1000000000000000000LL }; +const std::string columnstore_big_precision[20] = +{ + "9999999999999999999", + "99999999999999999999", + "999999999999999999999", + "9999999999999999999999", + "99999999999999999999999", + "999999999999999999999999", + "9999999999999999999999999", + "99999999999999999999999999", + "999999999999999999999999999", + "9999999999999999999999999999", + "99999999999999999999999999999", + "999999999999999999999999999999", + "9999999999999999999999999999999", + "99999999999999999999999999999999", + "999999999999999999999999999999999", + "9999999999999999999999999999999999", + "99999999999999999999999999999999999", + "999999999999999999999999999999999999", + "9999999999999999999999999999999999999", + "99999999999999999999999999999999999999" +}; const int32_t SECS_PER_MIN = 60; const int32_t MINS_PER_HOUR = 60; @@ -802,8 +828,8 @@ void TimeStamp::reset() second = 0xFFFFFFFFFFF; } -inline -int64_t string_to_ll( const std::string& data, bool& bSaturate ) +template +inline T string_to_ll( const std::string& data, bool& bSaturate ) { // This function doesn't take into consideration our special values // for NULL and EMPTY when setting the saturation point. Should it? @@ -850,23 +876,26 @@ uint64_t string_to_ull( const std::string& data, bool& bSaturate ) return value; } +template +void number_int_value(const std::string& data, + cscDataType typeCode, + const datatypes::SystemCatalog::TypeAttributesStd &ct, + bool& pushwarning, + bool noRoundup, + T& intVal); + +uint64_t number_uint_value(const string& data, + cscDataType typeCode, + const datatypes::SystemCatalog::TypeAttributesStd& ct, + bool& pushwarning, + bool noRoundup); + /** @brief DataConvert is a component for converting string data to Calpont format */ class DataConvert { public: - /** - * @brief convert a columns data, represnted as a string, to it's native - * format - * - * @param type the columns data type - * @param data the columns string representation of it's data - */ - EXPORT static boost::any convertColumnData( const execplan::CalpontSystemCatalog::ColType& colType, - const std::string& dataOrig, bool& bSaturate, const std::string& timeZone, - bool nulFlag = false, bool noRoundup = false, bool isUpdate = false); - /** * @brief convert a columns data from native format to a string * @@ -1008,9 +1037,6 @@ public: EXPORT static bool isColumnTimeValid( int64_t time ); EXPORT static bool isColumnTimeStampValid( int64_t timeStamp ); - EXPORT static bool isNullData(execplan::ColumnResult* cr, int rownum, execplan::CalpontSystemCatalog::ColType colType); - static inline std::string decimalToString(int64_t value, uint8_t scale, execplan::CalpontSystemCatalog::ColDataType colDataType); - static inline void decimalToString(int64_t value, uint8_t scale, char* buf, unsigned int buflen, execplan::CalpontSystemCatalog::ColDataType colDataType); static inline std::string constructRegexp(const std::string& str); static inline void trimWhitespace(int64_t& charData); static inline bool isEscapedChar(char c) @@ -1038,7 +1064,49 @@ public: EXPORT static int64_t timeToInt(const std::string& time); EXPORT static int64_t stringToTime (const std::string& data); // bug4388, union type conversion - EXPORT static execplan::CalpontSystemCatalog::ColType convertUnionColType(std::vector&); + EXPORT static void joinColTypeForUnion(datatypes::SystemCatalog::TypeHolderStd &unionedType, + const datatypes::SystemCatalog::TypeHolderStd &type); + + static boost::any StringToBit(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const datatypes::ConvertFromStringParam &prm, + const std::string& dataOrig, + bool& pushWarning); + + static boost::any StringToSDecimal(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const datatypes::ConvertFromStringParam &prm, + const std::string& data, + bool& pushWarning); + + static boost::any StringToUDecimal(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const datatypes::ConvertFromStringParam &prm, + const std::string& data, + bool& pushWarning); + + static boost::any StringToFloat(cscDataType typeCode, + const std::string& dataOrig, + bool& pushWarning); + + static boost::any StringToDouble(cscDataType typeCode, + const std::string& dataOrig, + bool& pushWarning); + + static boost::any StringToString(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const std::string& dataOrig, + bool& pushWarning); + + static boost::any StringToDate(const std::string& data, + bool& pushWarning); + + static boost::any StringToDatetime(const std::string& data, + bool& pushWarning); + + static boost::any StringToTime(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const std::string& data, + bool& pushWarning); + + static boost::any StringToTimestamp(const datatypes::ConvertFromStringParam &prm, + const std::string& data, + bool& pushWarning); }; inline void DataConvert::dateToString( int datevalue, char* buf, unsigned int buflen) @@ -1221,95 +1289,6 @@ inline void DataConvert::timeToString1( long long timevalue, char* buf, unsigned #endif } -inline std::string DataConvert::decimalToString(int64_t value, uint8_t scale, execplan::CalpontSystemCatalog::ColDataType colDataType) -{ - char buf[80]; - DataConvert::decimalToString(value, scale, buf, 80, colDataType); - return std::string(buf); -} - -inline void DataConvert::decimalToString(int64_t int_val, uint8_t scale, char* buf, unsigned int buflen, - execplan::CalpontSystemCatalog::ColDataType colDataType) -{ - // Need to convert a string with a binary unsigned number in it to a 64-bit signed int - - // MySQL seems to round off values unless we use the string store method. Groan. - // Taken from ha_mcs_impl.cpp - - //biggest Calpont supports is DECIMAL(18,x), or 18 total digits+dp+sign for column - // Need 19 digits maxium to hold a sum result of 18 digits decimal column. - if (isUnsigned(colDataType)) - { -#ifndef __LP64__ - snprintf(buf, buflen, "%llu", static_cast(int_val)); -#else - snprintf(buf, buflen, "%lu", static_cast(int_val)); -#endif - } - else - { -#ifndef __LP64__ - snprintf(buf, buflen, "%lld", int_val); -#else - snprintf(buf, buflen, "%ld", int_val); -#endif - } - - if (scale == 0) - return; - - //we want to move the last dt_scale chars right by one spot to insert the dp - //we want to move the trailing null as well, so it's really dt_scale+1 chars - size_t l1 = strlen(buf); - char* ptr = &buf[0]; - - if (int_val < 0) - { - ptr++; - idbassert(l1 >= 2); - l1--; - } - - //need to make sure we have enough leading zeros for this to work... - //at this point scale is always > 0 - size_t l2 = 1; - - if ((unsigned)scale > l1) - { - const char* zeros = "00000000000000000000"; //20 0's - size_t diff = 0; - - if (int_val != 0) - diff = scale - l1; //this will always be > 0 - else - diff = scale; - - memmove((ptr + diff), ptr, l1 + 1); //also move null - memcpy(ptr, zeros, diff); - - if (int_val != 0) - l1 = 0; - else - l1 = 1; - } - else if ((unsigned)scale == l1) - { - l1 = 0; - l2 = 2; - } - else - { - l1 -= scale; - } - - memmove((ptr + l1 + l2), (ptr + l1), scale + 1); //also move null - - if (l2 == 2) - *(ptr + l1++) = '0'; - - *(ptr + l1) = '.'; -} - inline void DataConvert::trimWhitespace(int64_t& charData) { // Trims whitespace characters off non-dict character data @@ -1406,6 +1385,107 @@ inline std::string DataConvert::constructRegexp(const std::string& str) return ret; } +inline int128_t add128(int128_t a, int128_t b) +{ + return a + b; +} + +inline int128_t subtract128(int128_t a, int128_t b) +{ + return a - b; +} + +inline bool lessThan128(int128_t a, int128_t b) +{ + return a < b; +} + +inline bool greaterThan128(int128_t a, int128_t b) +{ + return a > b; +} + +// Naive int128_t version of strtoll +inline int128_t strtoll128(const char* data, bool& saturate, char** ep) +{ + int128_t res = 0; + + if (*data == '\0') + { + if (ep) + *ep = (char*)data; + return res; + } + + // skip leading whitespace characters + while (*data != '\0' && + (*data == ' ' || *data == '\t' || *data == '\n')) + data++; + + int128_t (*op)(int128_t, int128_t); + op = add128; + bool (*compare)(int128_t, int128_t); + compare = lessThan128; + + // check the -ve sign + bool is_neg = false; + if (*data == '-') + { + is_neg = true; + op = subtract128; + compare = greaterThan128; + data++; + } + + int128_t tmp; + + for (; *data != '\0' && isdigit(*data); data++) + { + tmp = op(res*10, *data - '0'); + + if (UNLIKELY(compare(tmp, res))) + { + saturate = true; + + if (is_neg) + utils::int128Min(res); + else + utils::int128Max(res); + + while (*data != '\0' && isdigit(*data)) + data++; + + if (ep) + *ep = (char*)data; + + return res; + } + + res = tmp; + } + + if (ep) + *ep = (char*)data; + + return res; +} + +template<> +inline int128_t string_to_ll ( const std::string& data, bool& bSaturate ) +{ + // This function doesn't take into consideration our special values + // for NULL and EMPTY when setting the saturation point. Should it? + char* ep = NULL; + const char* str = data.c_str(); + int128_t value = strtoll128(str, bSaturate, &ep); + + // (no digits) || (more chars) + if ((ep == str) || (*ep != '\0')) + throw logging::QueryDataExcept("value is not numerical.", logging::formatErr); + + return value; +} + } // namespace dataconvert #undef EXPORT diff --git a/utils/funcexp/CMakeLists.txt b/utils/funcexp/CMakeLists.txt index 889b44d64..8d194f105 100644 --- a/utils/funcexp/CMakeLists.txt +++ b/utils/funcexp/CMakeLists.txt @@ -112,7 +112,7 @@ add_library(funcexp SHARED ${funcexp_LIB_SRCS}) add_dependencies(funcexp loggingcpp) -target_link_libraries(funcexp ${NETSNMP_LIBRARIES}) +target_link_libraries(funcexp ${NETSNMP_LIBRARIES} quadmath) install(TARGETS funcexp DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) diff --git a/utils/funcexp/func_abs.cpp b/utils/funcexp/func_abs.cpp index 7b04408ba..830a48a6d 100644 --- a/utils/funcexp/func_abs.cpp +++ b/utils/funcexp/func_abs.cpp @@ -66,7 +66,11 @@ IDB_Decimal Func_abs::getDecimalVal(Row& row, CalpontSystemCatalog::ColType&) { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - d.value = llabs(d.value); + + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + d.s128Value = (d.s128Value < 0) ? -d.s128Value : d.s128Value; + else + d.value = llabs(d.value); return d; } diff --git a/utils/funcexp/func_between.cpp b/utils/funcexp/func_between.cpp index ab6aef028..f4c0eabf6 100644 --- a/utils/funcexp/func_between.cpp +++ b/utils/funcexp/func_between.cpp @@ -237,6 +237,9 @@ inline bool getBool(rowgroup::Row& row, { IDB_Decimal val = pm[0]->data()->getDecimalVal(row, isNull); + if (isNull) + return false; + if (notBetween) { if (!numericGE(val, pm[1]->data()->getDecimalVal(row, isNull)) && !isNull) @@ -246,8 +249,7 @@ inline bool getBool(rowgroup::Row& row, return (!numericLE(val, pm[2]->data()->getDecimalVal(row, isNull)) && !isNull); } - return !isNull && - numericGE(val, pm[1]->data()->getDecimalVal(row, isNull)) && + return numericGE(val, pm[1]->data()->getDecimalVal(row, isNull)) && numericLE(val, pm[2]->data()->getDecimalVal(row, isNull)); } diff --git a/utils/funcexp/func_bitwise.cpp b/utils/funcexp/func_bitwise.cpp index 2154f125b..5eb48252e 100644 --- a/utils/funcexp/func_bitwise.cpp +++ b/utils/funcexp/func_bitwise.cpp @@ -56,8 +56,12 @@ bool getUIntValFromParm( const execplan::SPTP& parm, uint64_t& value, bool& isNull, - const string& timeZone) + const string& timeZone, + bool& isBigVal, + int128_t& bigval) { + isBigVal = false; + switch (parm->data()->resultType().colDataType) { case execplan::CalpontSystemCatalog::BIGINT: @@ -102,22 +106,53 @@ bool getUIntValFromParm( { IDB_Decimal d = parm->data()->getDecimalVal(row, isNull); - if (parm->data()->resultType().colDataType == execplan::CalpontSystemCatalog::UDECIMAL && - d.value < 0) + if (parm->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) { - d.value = 0; + isBigVal = true; + + if (parm->data()->resultType().colDataType == execplan::CalpontSystemCatalog::UDECIMAL && + d.value < 0) + { + bigval = 0; + break; + } + + int128_t scaleDivisor, scaleDivisor2; + + datatypes::getScaleDivisor(scaleDivisor, d.scale); + + scaleDivisor2 = (scaleDivisor <= 10) ? 1 : (scaleDivisor / 10); + + int128_t tmpval = d.s128Value / scaleDivisor; + int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; + + if (tmpval >= 0 && lefto > 4) + tmpval++; + + if (tmpval < 0 && lefto < -4) + tmpval--; + + bigval = tmpval; } - double dscale = d.scale; - int64_t tmpval = d.value / pow(10.0, dscale); - int lefto = (d.value - tmpval * pow(10.0, dscale)) / pow(10.0, dscale - 1); + else + { + if (parm->data()->resultType().colDataType == execplan::CalpontSystemCatalog::UDECIMAL && + d.value < 0) + { + d.value = 0; + } + double dscale = d.scale; + int64_t tmpval = d.value / pow(10.0, dscale); + int lefto = (d.value - tmpval * pow(10.0, dscale)) / pow(10.0, dscale - 1); - if ( tmpval >= 0 && lefto > 4 ) - tmpval++; + if (tmpval >= 0 && lefto > 4) + tmpval++; - if ( tmpval < 0 && lefto < -4 ) - tmpval--; + if (tmpval < 0 && lefto < -4) + tmpval--; - value = tmpval; + value = tmpval; + } } break; @@ -195,15 +230,39 @@ int64_t Func_bitand::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone()) || - !getUIntValFromParm(row, parm[1], val2, isNull, timeZone())) + int128_t bigval1 = 0; + int128_t bigval2 = 0; + bool isBigVal1; + bool isBigVal2; + + if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone(), isBigVal1, bigval1) || + !getUIntValFromParm(row, parm[1], val2, isNull, timeZone(), isBigVal2, bigval2)) { std::ostringstream oss; oss << "bitand: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); } - return val1 & val2; + if (LIKELY(!isBigVal1 && !isBigVal2)) + { + return val1 & val2; + } + + // Type promotion to int128_t + if (!isBigVal1) + bigval1 = val1; + + if (!isBigVal2) + bigval2 = val2; + + int128_t res = bigval1 & bigval2; + + if (res > static_cast(UINT64_MAX)) + res = UINT64_MAX; + else if (res < static_cast(INT64_MIN)) + res = INT64_MIN; + + return (int64_t) res; } @@ -231,15 +290,39 @@ int64_t Func_leftshift::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone()) || - !getUIntValFromParm(row, parm[1], val2, isNull, timeZone())) + int128_t bigval1 = 0; + int128_t bigval2 = 0; + bool isBigVal1; + bool isBigVal2; + + if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone(), isBigVal1, bigval1) || + !getUIntValFromParm(row, parm[1], val2, isNull, timeZone(), isBigVal2, bigval2)) { std::ostringstream oss; oss << "leftshift: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); } - return val1 << val2; + if (LIKELY(!isBigVal1 && !isBigVal2)) + { + return val1 << val2; + } + + // Type promotion to int128_t + if (!isBigVal1) + bigval1 = val1; + + if (!isBigVal2) + bigval2 = val2; + + int128_t res = bigval1 << bigval2; + + if (res > static_cast(UINT64_MAX)) + res = UINT64_MAX; + else if (res < static_cast(INT64_MIN)) + res = INT64_MIN; + + return (int64_t) res; } @@ -267,15 +350,39 @@ int64_t Func_rightshift::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone()) || - !getUIntValFromParm(row, parm[1], val2, isNull, timeZone())) + int128_t bigval1 = 0; + int128_t bigval2 = 0; + bool isBigVal1; + bool isBigVal2; + + if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone(), isBigVal1, bigval1) || + !getUIntValFromParm(row, parm[1], val2, isNull, timeZone(), isBigVal2, bigval2)) { std::ostringstream oss; oss << "rightshift: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); } - return val1 >> val2; + if (LIKELY(!isBigVal1 && !isBigVal2)) + { + return val1 >> val2; + } + + // Type promotion to int128_t + if (!isBigVal1) + bigval1 = val1; + + if (!isBigVal2) + bigval2 = val2; + + int128_t res = bigval1 >> bigval2; + + if (res > static_cast(UINT64_MAX)) + res = UINT64_MAX; + else if (res < static_cast(INT64_MIN)) + res = INT64_MIN; + + return (int64_t) res; } @@ -303,15 +410,39 @@ int64_t Func_bitor::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone()) || - !getUIntValFromParm(row, parm[1], val2, isNull, timeZone())) + int128_t bigval1 = 0; + int128_t bigval2 = 0; + bool isBigVal1; + bool isBigVal2; + + if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone(), isBigVal1, bigval1) || + !getUIntValFromParm(row, parm[1], val2, isNull, timeZone(), isBigVal2, bigval2)) { std::ostringstream oss; oss << "bitor: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); } - return val1 | val2; + if (LIKELY(!isBigVal1 && !isBigVal2)) + { + return val1 | val2; + } + + // Type promotion to int128_t + if (!isBigVal1) + bigval1 = val1; + + if (!isBigVal2) + bigval2 = val2; + + int128_t res = bigval1 | bigval2; + + if (res > static_cast(UINT64_MAX)) + res = UINT64_MAX; + else if (res < static_cast(INT64_MIN)) + res = INT64_MIN; + + return (int64_t) res; } uint64_t Func_bitor::getUintVal(rowgroup::Row& row, @@ -347,15 +478,39 @@ int64_t Func_bitxor::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone()) || - !getUIntValFromParm(row, parm[1], val2, isNull, timeZone())) + int128_t bigval1 = 0; + int128_t bigval2 = 0; + bool isBigVal1; + bool isBigVal2; + + if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone(), isBigVal1, bigval1) || + !getUIntValFromParm(row, parm[1], val2, isNull, timeZone(), isBigVal2, bigval2)) { std::ostringstream oss; oss << "bitxor: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); } - return val1 ^ val2; + if (LIKELY(!isBigVal1 && !isBigVal2)) + { + return val1 ^ val2; + } + + // Type promotion to int128_t + if (!isBigVal1) + bigval1 = val1; + + if (!isBigVal2) + bigval2 = val2; + + int128_t res = bigval1 ^ bigval2; + + if (res > static_cast(UINT64_MAX)) + res = UINT64_MAX; + else if (res < static_cast(INT64_MIN)) + res = INT64_MIN; + + return (int64_t) res; } @@ -369,6 +524,20 @@ CalpontSystemCatalog::ColType Func_bit_count::operationType( FunctionParm& fp, C return resultType; } +inline int64_t bitCount(uint64_t val) +{ + // Refer to Hacker's Delight Chapter 5 + // for the bit counting algo used here + val = val - ((val >> 1) & 0x5555555555555555); + val = (val & 0x3333333333333333) + ((val >> 2) & 0x3333333333333333); + val = (val + (val >> 4)) & 0x0F0F0F0F0F0F0F0F; + val = val + (val >> 8); + val = val + (val >> 16); + val = val + (val >> 32); + + return (int64_t)(val & 0x000000000000007F); +} + int64_t Func_bit_count::getIntVal(Row& row, FunctionParm& parm, bool& isNull, @@ -382,24 +551,25 @@ int64_t Func_bit_count::getIntVal(Row& row, uint64_t val = 0; - if (!getUIntValFromParm(row, parm[0], val, isNull, timeZone())) + int128_t bigval = 0; + bool isBigVal; + + if (!getUIntValFromParm(row, parm[0], val, isNull, timeZone(), isBigVal, bigval)) { std::ostringstream oss; oss << "bit_count: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); } - // Refer to Hacker's Delight Chapter 5 - // for the bit counting algo used here - val = val - ((val >> 1) & 0x5555555555555555); - val = (val & 0x3333333333333333) + ((val >> 2) & 0x3333333333333333); - val = (val + (val >> 4)) & 0x0F0F0F0F0F0F0F0F; - val = val + (val >> 8); - val = val + (val >> 16); - val = val + (val >> 32); - - return (int64_t)(val & 0x000000000000007F); - + if (LIKELY(!isBigVal)) + { + return bitCount(val); + } + else + { + return (bitCount(*reinterpret_cast(&bigval)) + + bitCount(*(reinterpret_cast(&bigval) + 1))); + } } diff --git a/utils/funcexp/func_case.cpp b/utils/funcexp/func_case.cpp index 327c02cc0..0204d9abf 100644 --- a/utils/funcexp/func_case.cpp +++ b/utils/funcexp/func_case.cpp @@ -212,7 +212,7 @@ inline uint64_t simple_case_cmp(Row& row, for (i = 1; i <= whereCount; i++) { - if (ev == parm[i]->data()->getDecimalVal(row, isNull) && !isNull) + if (ev == parm[i]->data()->getDecimalVal(row, isNull) && !isNull) { foundIt = true; break; @@ -329,7 +329,6 @@ inline uint64_t searched_case_cmp(Row& row, uint64_t whereCount = hasElse ? (parm.size() - 1) / 2 : parm.size() / 2; bool foundIt = false; - for (i = 0; i < whereCount; i++) { if (parm[i]->getBoolVal(row, isNull)) @@ -362,9 +361,7 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, bool simpleCase) { uint64_t simple = simpleCase ? 1 : 0; - bool hasElse = (((fp.size()-simple) % 2) != 0); // if 1, then ELSE exist - - + bool hasElse = (((fp.size()-simple) % 2) != 0); // if 1, then ELSE exist uint64_t parmCount = hasElse ? (fp.size() - 2) : (fp.size() - 1); uint64_t whereCount = hasElse ? (fp.size() - 2 + simple) / 2 : (fp.size() - 1) / 2 + simple; @@ -382,8 +379,8 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, { // for SimpleCase, we return the type of the case expression, // which will always be in position 0. - if (i == 0 && simpleCase) - { + if (i == 0 && simpleCase) + { if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) @@ -393,15 +390,16 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, allStringO = false; oct = op.operationType(); } - i += 1; - } - // operation or result type - operation = ((i > 0+simple) && (i <= whereCount)); + i += 1; + } + + // operation or result type + operation = ((i > 0+simple) && (i <= whereCount)); if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && - fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && - fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) + fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && + fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) { // this is not a string column PredicateOperator op; @@ -414,7 +412,7 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, allStringO = false; oct = op.operationType(); } - } + } // If any parm is of string type, the result type should be string. (same as if) else if (rct.colDataType != CalpontSystemCatalog::CHAR && diff --git a/utils/funcexp/func_cast.cpp b/utils/funcexp/func_cast.cpp index 3584b7102..ef36bb320 100644 --- a/utils/funcexp/func_cast.cpp +++ b/utils/funcexp/func_cast.cpp @@ -22,6 +22,8 @@ #include using namespace std; +#include + #include "functor_dtm.h" #include "functor_int.h" #include "functor_real.h" @@ -183,17 +185,40 @@ int64_t Func_cast_signed::getIntVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - double dscale = d.scale; - int64_t value = d.value / pow(10.0, dscale); - int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1); - if ( value >= 0 && lefto > 4 ) - value++; + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t scaleDivisor, scaleDivisor2; - if ( value < 0 && lefto < -4 ) - value--; + datatypes::getScaleDivisor(scaleDivisor, d.scale); - return value; + scaleDivisor2 = (scaleDivisor <= 10) ? 1 : (scaleDivisor / 10); + + int128_t tmpval = d.s128Value / scaleDivisor; + int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; + + if (tmpval >= 0 && lefto > 4) + tmpval++; + + if (tmpval < 0 && lefto < -4) + tmpval--; + + return datatypes::Decimal::getInt64FromWideDecimal(tmpval); + } + else + { + double dscale = d.scale; + int64_t value = d.value / pow(10.0, dscale); + int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1); + + if ( value >= 0 && lefto > 4 ) + value++; + + if ( value < 0 && lefto < -4 ) + value--; + + return value; + } } break; @@ -341,20 +366,50 @@ uint64_t Func_cast_unsigned::getUintVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - double dscale = d.scale; - if (d.value < 0) + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) { - return 0; - } + if (d.s128Value < 0) + { + return 0; + } - uint64_t value = d.value / pow(10.0, dscale); - int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1); + int128_t scaleDivisor, scaleDivisor2; + + datatypes::getScaleDivisor(scaleDivisor, d.scale); + + scaleDivisor2 = (scaleDivisor <= 10) ? 1 : (scaleDivisor / 10); + + uint128_t tmpval = d.s128Value / scaleDivisor; + int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; + + if (utils::is_nonnegative(tmpval) && lefto > 4) + tmpval++; + + if (tmpval > static_cast(UINT64_MAX)) + tmpval = UINT64_MAX; + + return static_cast(tmpval); + } + else + { + if (d.value < 0) + { + return 0; + } + + double dscale = d.scale; + + uint64_t value = d.value / pow(10.0, dscale); + int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1); if ( utils::is_nonnegative(value) && lefto > 4 ) + { value++; + } - return value; + return value; + } } break; @@ -489,12 +544,10 @@ string Func_cast_char::getStrVal(Row& row, { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - char buf[80]; - - dataconvert::DataConvert::decimalToString( d.value, d.scale, buf, 80, parm[0]->data()->resultType().colDataType); - - string sbuf = buf; - return sbuf.substr(0, length); + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + return d.toString(true).substr(0, length); + else + return d.toString().substr(0, length); } break; @@ -583,10 +636,16 @@ IDB_Decimal Func_cast_date::getDecimalVal(Row& row, { IDB_Decimal decimal; - decimal.value = Func_cast_date::getDatetimeIntVal(row, - parm, - isNull, - operationColType); + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + decimal.s128Value = Func_cast_date::getDatetimeIntVal(row, + parm, + isNull, + operationColType); + else + decimal.value = Func_cast_date::getDatetimeIntVal(row, + parm, + isNull, + operationColType); return decimal; } @@ -886,10 +945,16 @@ IDB_Decimal Func_cast_datetime::getDecimalVal(Row& row, { IDB_Decimal decimal; - decimal.value = Func_cast_datetime::getDatetimeIntVal(row, - parm, - isNull, - operationColType); + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + decimal.s128Value = Func_cast_datetime::getDatetimeIntVal(row, + parm, + isNull, + operationColType); + else + decimal.value = Func_cast_datetime::getDatetimeIntVal(row, + parm, + isNull, + operationColType); return decimal; } @@ -1139,6 +1204,17 @@ int64_t Func_cast_decimal::getIntVal(Row& row, isNull, operationColType); + if (decimal.precision > datatypes::INT64MAXPRECISION) + { + int128_t scaleDivisor; + + datatypes::getScaleDivisor(scaleDivisor, decimal.scale); + + int128_t tmpval = decimal.s128Value / scaleDivisor; + + return datatypes::Decimal::getInt64FromWideDecimal(tmpval); + } + return (int64_t) decimal.value / helpers::powerOf10_c[decimal.scale]; } @@ -1152,14 +1228,10 @@ string Func_cast_decimal::getStrVal(Row& row, parm, isNull, operationColType); - - char buf[80]; - - dataconvert::DataConvert::decimalToString( decimal.value, decimal.scale, buf, 80, operationColType.colDataType); - - string value = buf; - return value; - + if (operationColType.colWidth == datatypes::MAXDECIMALWIDTH) + return decimal.toString(true); + else + return decimal.toString(); } @@ -1173,12 +1245,10 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, int32_t decimals = parm[1]->data()->getIntVal(row, isNull); int64_t max_length = parm[2]->data()->getIntVal(row, isNull); - // As of 2.0, max length columnStore can support is 18 - // decimal(0,0) is valid, and no limit on integer number - if (max_length > 18 || max_length <= 0) - max_length = 18; + if (max_length > datatypes::INT128MAXPRECISION || max_length <= 0) + max_length = datatypes::INT128MAXPRECISION; - int64_t max_number_decimal = helpers::maxNumber_c[max_length]; + decimal.precision = max_length; switch (parm[0]->data()->resultType().colDataType) { @@ -1188,19 +1258,45 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, case execplan::CalpontSystemCatalog::TINYINT: case execplan::CalpontSystemCatalog::SMALLINT: { - decimal.value = parm[0]->data()->getIntVal(row, isNull); - decimal.scale = 0; - int64_t value = decimal.value * helpers::powerOf10_c[decimals]; + if (max_length > datatypes::INT64MAXPRECISION) + { + bool dummy = false; + char *ep = NULL; + int128_t max_number_decimal = dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); + decimal.s128Value = parm[0]->data()->getIntVal(row, isNull); + decimal.scale = 0; + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, decimals); + int128_t value = decimal.s128Value * scaleDivisor; - if ( value > max_number_decimal ) - { - decimal.value = max_number_decimal; - decimal.scale = decimals; + if ( value > max_number_decimal ) + { + decimal.s128Value = max_number_decimal; + decimal.scale = decimals; + } + else if ( value < -max_number_decimal ) + { + decimal.s128Value = -max_number_decimal; + decimal.scale = decimals; + } } - else if ( value < -max_number_decimal ) + else { - decimal.value = -max_number_decimal; - decimal.scale = decimals; + int64_t max_number_decimal = helpers::maxNumber_c[max_length]; + decimal.value = parm[0]->data()->getIntVal(row, isNull); + decimal.scale = 0; + int64_t value = decimal.value * helpers::powerOf10_c[decimals]; + + if ( value > max_number_decimal ) + { + decimal.value = max_number_decimal; + decimal.scale = decimals; + } + else if ( value < -max_number_decimal ) + { + decimal.value = -max_number_decimal; + decimal.scale = decimals; + } } } break; @@ -1211,21 +1307,51 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, case execplan::CalpontSystemCatalog::UTINYINT: case execplan::CalpontSystemCatalog::USMALLINT: { - uint64_t uval = parm[0]->data()->getUintVal(row, isNull); - - if (uval > (uint64_t)numeric_limits::max()) + if (max_length > datatypes::INT64MAXPRECISION) { - uval = numeric_limits::max(); + bool dummy = false; + char *ep = NULL; + int128_t max_number_decimal = dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); + + uint128_t uval = parm[0]->data()->getUintVal(row, isNull); + + if (uval > (uint128_t)datatypes::Decimal::maxInt128) + { + uval = datatypes::Decimal::maxInt128; + } + + decimal.s128Value = uval; + decimal.scale = 0; + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, decimals); + int128_t value = decimal.s128Value * scaleDivisor; + + if ( value > max_number_decimal ) + { + decimal.s128Value = max_number_decimal; + decimal.scale = decimals; + } } - - decimal.value = uval; - decimal.scale = 0; - int64_t value = decimal.value * helpers::powerOf10_c[decimals]; - - if ( value > max_number_decimal ) + else { - decimal.value = max_number_decimal; - decimal.scale = decimals; + int64_t max_number_decimal = helpers::maxNumber_c[max_length]; + + uint64_t uval = parm[0]->data()->getUintVal(row, isNull); + + if (uval > (uint64_t)numeric_limits::max()) + { + uval = numeric_limits::max(); + } + + decimal.value = uval; + decimal.scale = 0; + int64_t value = decimal.value * helpers::powerOf10_c[decimals]; + + if ( value > max_number_decimal ) + { + decimal.value = max_number_decimal; + decimal.scale = decimals; + } } } break; @@ -1235,67 +1361,172 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::UFLOAT: { - double value = parm[0]->data()->getDoubleVal(row, isNull); + if (max_length > datatypes::INT64MAXPRECISION) + { + bool dummy = false; + char *ep = NULL; + int128_t max_number_decimal = dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); - if (value > 0) - decimal.value = (int64_t) (value * helpers::powerOf10_c[decimals] + 0.5); - else if (value < 0) - decimal.value = (int64_t) (value * helpers::powerOf10_c[decimals] - 0.5); + __float128 value = parm[0]->data()->getDoubleVal(row, isNull); + + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, decimals); + + if (value > 0) + decimal.s128Value = (int128_t) (value * scaleDivisor + 0.5); + else if (value < 0) + decimal.s128Value = (int128_t) (value * scaleDivisor - 0.5); + else + decimal.s128Value = 0; + + decimal.scale = decimals; + + if ( value > max_number_decimal ) + decimal.s128Value = max_number_decimal; + else if ( value < -max_number_decimal ) + decimal.s128Value = -max_number_decimal; + } else - decimal.value = 0; + { + int64_t max_number_decimal = helpers::maxNumber_c[max_length]; - decimal.scale = decimals; + double value = parm[0]->data()->getDoubleVal(row, isNull); - if ( value > max_number_decimal ) - decimal.value = max_number_decimal; - else if ( value < -max_number_decimal ) - decimal.value = -max_number_decimal; + if (value > 0) + decimal.value = (int64_t) (value * helpers::powerOf10_c[decimals] + 0.5); + else if (value < 0) + decimal.value = (int64_t) (value * helpers::powerOf10_c[decimals] - 0.5); + else + decimal.value = 0; + + decimal.scale = decimals; + + if ( value > max_number_decimal ) + decimal.value = max_number_decimal; + else if ( value < -max_number_decimal ) + decimal.value = -max_number_decimal; + } } break; case execplan::CalpontSystemCatalog::LONGDOUBLE: { - long double value = parm[0]->data()->getLongDoubleVal(row, isNull); + if (max_length > datatypes::INT64MAXPRECISION) + { + bool dummy = false; + char *ep = NULL; + int128_t max_number_decimal = dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); - if (value > 0) - decimal.value = (int64_t) (value * helpers::powerOf10_c[decimals] + 0.5); - else if (value < 0) - decimal.value = (int64_t) (value * helpers::powerOf10_c[decimals] - 0.5); + __float128 value = parm[0]->data()->getLongDoubleVal(row, isNull); + + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, decimals); + + if (value > 0) + decimal.s128Value = (int128_t) (value * scaleDivisor + 0.5); + else if (value < 0) + decimal.s128Value = (int128_t) (value * scaleDivisor - 0.5); + else + decimal.s128Value = 0; + + decimal.scale = decimals; + + if ( value > max_number_decimal ) + decimal.s128Value = max_number_decimal; + else if ( value < -max_number_decimal ) + decimal.s128Value = -max_number_decimal; + } else - decimal.value = 0; + { + int64_t max_number_decimal = helpers::maxNumber_c[max_length]; - decimal.scale = decimals; + long double value = parm[0]->data()->getLongDoubleVal(row, isNull); - if ( value > max_number_decimal ) - decimal.value = max_number_decimal; - else if ( value < -max_number_decimal ) - decimal.value = -max_number_decimal; + if (value > 0) + decimal.value = (int64_t) (value * helpers::powerOf10_c[decimals] + 0.5); + else if (value < 0) + decimal.value = (int64_t) (value * helpers::powerOf10_c[decimals] - 0.5); + else + decimal.value = 0; + + decimal.scale = decimals; + + if ( value > max_number_decimal ) + decimal.value = max_number_decimal; + else if ( value < -max_number_decimal ) + decimal.value = -max_number_decimal; + } } break; case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - decimal = parm[0]->data()->getDecimalVal(row, isNull); + if (max_length > datatypes::INT64MAXPRECISION) + { + bool dummy = false; + char *ep = NULL; + int128_t max_number_decimal = dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); + decimal = parm[0]->data()->getDecimalVal(row, isNull); - if (decimals > decimal.scale) - decimal.value *= helpers::powerOf10_c[decimals - decimal.scale]; + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, abs(decimals - decimal.scale)); + + if (decimal.precision <= datatypes::INT64MAXPRECISION) + decimal.s128Value = decimal.value; + + decimal.precision = max_length; + + if (scaleDivisor > 1) + { + if (decimals > decimal.scale) + decimal.s128Value *= scaleDivisor; + else + decimal.s128Value = (int128_t)(decimal.s128Value > 0 ? + (__float128)decimal.s128Value / scaleDivisor + 0.5 : + (__float128)decimal.s128Value / scaleDivisor - 0.5); + } + + decimal.scale = decimals; + + if ( decimal.s128Value > max_number_decimal ) + decimal.s128Value = max_number_decimal; + else if ( decimal.s128Value < -max_number_decimal ) + decimal.s128Value = -max_number_decimal; + } else - decimal.value = (int64_t)(decimal.value > 0 ? - (double)decimal.value / helpers::powerOf10_c[decimal.scale - decimals] + 0.5 : - (double)decimal.value / helpers::powerOf10_c[decimal.scale - decimals] - 0.5); + { + int64_t max_number_decimal = helpers::maxNumber_c[max_length]; - decimal.scale = decimals; + decimal = parm[0]->data()->getDecimalVal(row, isNull); + if (decimal.precision > datatypes::INT64MAXPRECISION) + { + if ( decimal.s128Value > (int128_t) max_number_decimal ) + decimal.value = max_number_decimal; + else if ( decimal.s128Value < (int128_t) -max_number_decimal ) + decimal.value = -max_number_decimal; + else + decimal.value = decimal.s128Value; + } + decimal.precision = max_length; - //int64_t value = decimal.value; + if (decimals > decimal.scale) + decimal.value *= helpers::powerOf10_c[decimals - decimal.scale]; + else + decimal.value = (int64_t)(decimal.value > 0 ? + (double)decimal.value / helpers::powerOf10_c[decimal.scale - decimals] + 0.5 : + (double)decimal.value / helpers::powerOf10_c[decimal.scale - decimals] - 0.5); - if ( decimal.value > max_number_decimal ) - decimal.value = max_number_decimal; - else if ( decimal.value < -max_number_decimal ) - decimal.value = -max_number_decimal; + decimal.scale = decimals; + + if ( decimal.value > max_number_decimal ) + decimal.value = max_number_decimal; + else if ( decimal.value < -max_number_decimal ) + decimal.value = -max_number_decimal; + } } break; @@ -1313,9 +1544,6 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, int negate = 1; bool bFoundSign = false; bool bRound = false; - double floatValue; - int64_t value = 0; - int64_t frac = 0; if (strValue.empty()) { @@ -1331,27 +1559,63 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, { if (*s == 'e' || *s == 'E') { - floatValue = strtod(str, 0); + if (max_length > datatypes::INT64MAXPRECISION) + { + bool dummy = false; + char *ep = NULL; + int128_t max_number_decimal = dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); - // If the float value is too large, the saturated result may end up with - // the wrong sign, so we just check first. - if ((int64_t)floatValue > max_number_decimal) - decimal.value = max_number_decimal; - else if ((int64_t)floatValue < -max_number_decimal) - decimal.value = -max_number_decimal; - else if (floatValue > 0) - decimal.value = (int64_t) (floatValue * helpers::powerOf10_c[decimals] + 0.5); - else if (floatValue < 0) - decimal.value = (int64_t) (floatValue * helpers::powerOf10_c[decimals] - 0.5); + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, decimals); + + __float128 floatValue = strtoflt128 (str, 0); + + // If the float value is too large, the saturated result may end up with + // the wrong sign, so we just check first. + if ((int128_t)floatValue > max_number_decimal) + decimal.s128Value = max_number_decimal; + else if ((int128_t)floatValue < -max_number_decimal) + decimal.s128Value = -max_number_decimal; + else if (floatValue > 0) + decimal.s128Value = (int128_t) (floatValue * scaleDivisor + 0.5); + else if (floatValue < 0) + decimal.s128Value = (int128_t) (floatValue * scaleDivisor - 0.5); + else + decimal.s128Value = 0; + + if (decimal.s128Value > max_number_decimal) + decimal.s128Value = max_number_decimal; + else if (decimal.s128Value < -max_number_decimal) + decimal.s128Value = -max_number_decimal; + + return decimal; + } else - decimal.value = 0; + { + int64_t max_number_decimal = helpers::maxNumber_c[max_length]; - if (decimal.value > max_number_decimal) - decimal.value = max_number_decimal; - else if (decimal.value < -max_number_decimal) - decimal.value = -max_number_decimal; + double floatValue = strtod(str, 0); - return decimal; + // If the float value is too large, the saturated result may end up with + // the wrong sign, so we just check first. + if ((int64_t)floatValue > max_number_decimal) + decimal.value = max_number_decimal; + else if ((int64_t)floatValue < -max_number_decimal) + decimal.value = -max_number_decimal; + else if (floatValue > 0) + decimal.value = (int64_t) (floatValue * helpers::powerOf10_c[decimals] + 0.5); + else if (floatValue < 0) + decimal.value = (int64_t) (floatValue * helpers::powerOf10_c[decimals] - 0.5); + else + decimal.value = 0; + + if (decimal.value > max_number_decimal) + decimal.value = max_number_decimal; + else if (decimal.value < -max_number_decimal) + decimal.value = -max_number_decimal; + + return decimal; + } } } @@ -1392,54 +1656,119 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, } } - errno = 0; - - if (firstInt) // Checking to see if we have a decimal point, but no previous digits. + if (max_length > datatypes::INT64MAXPRECISION) { - value = strtoll(firstInt, &endptr, 10); - } + bool dummy = false; + char *ep = NULL; + int128_t max_number_decimal = dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); - if (!errno && endptr) - { - // Scale the integer portion according to the DECIMAL description - value *= helpers::powerOf10_c[decimals]; + int128_t value = 0, frac = 0; - // Get the fractional part. - if (endptr && (*endptr == *convData->decimal_point || *endptr == '.')) + if (firstInt) // Checking to see if we have a decimal point, but no previous digits. { - s = endptr + 1; - - // Get the digits to the right of the decimal - // Only retrieve those that matter based on scale. - for (fracChars = 0; - *s && isdigit(*s) && fracChars < decimals; - ++fracChars, ++s) - { - // Save the frac characters to a side buffer. This way we can limit - // ourselves to the scale without modifying the original string. - fracBuf[fracChars] = *s; - } - - fracBuf[fracChars] = 0; - - // Check to see if we need to round - if (isdigit(*s) && *s > '4') - { - bRound = true; - } + value = dataconvert::strtoll128(firstInt, dummy, &endptr); } - frac = strtoll(fracBuf, &endptr, 10); - value += frac + (bRound ? 1 : 0); - value *= negate; + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, decimals); + + if (!dummy && endptr) + { + // Scale the integer portion according to the DECIMAL description + value *= scaleDivisor; + + // Get the fractional part. + if (endptr && (*endptr == *convData->decimal_point || *endptr == '.')) + { + s = endptr + 1; + + // Get the digits to the right of the decimal + // Only retrieve those that matter based on scale. + for (fracChars = 0; + *s && isdigit(*s) && fracChars < decimals; + ++fracChars, ++s) + { + // Save the frac characters to a side buffer. This way we can limit + // ourselves to the scale without modifying the original string. + fracBuf[fracChars] = *s; + } + + fracBuf[fracChars] = 0; + + // Check to see if we need to round + if (isdigit(*s) && *s > '4') + { + bRound = true; + } + } + + frac = dataconvert::strtoll128(fracBuf, dummy, &ep); + value += frac + (bRound ? 1 : 0); + value *= negate; + } + + decimal.s128Value = value; + + if (decimal.s128Value > max_number_decimal) + decimal.s128Value = max_number_decimal; + else if (decimal.s128Value < -max_number_decimal) + decimal.s128Value = -max_number_decimal; } + else + { + int64_t max_number_decimal = helpers::maxNumber_c[max_length]; - decimal.value = value; + int64_t value = 0, frac = 0; - if (decimal.value > max_number_decimal) - decimal.value = max_number_decimal; - else if (decimal.value < -max_number_decimal) - decimal.value = -max_number_decimal; + errno = 0; + + if (firstInt) // Checking to see if we have a decimal point, but no previous digits. + { + value = strtoll(firstInt, &endptr, 10); + } + + if (!errno && endptr) + { + // Scale the integer portion according to the DECIMAL description + value *= helpers::powerOf10_c[decimals]; + + // Get the fractional part. + if (endptr && (*endptr == *convData->decimal_point || *endptr == '.')) + { + s = endptr + 1; + + // Get the digits to the right of the decimal + // Only retrieve those that matter based on scale. + for (fracChars = 0; + *s && isdigit(*s) && fracChars < decimals; + ++fracChars, ++s) + { + // Save the frac characters to a side buffer. This way we can limit + // ourselves to the scale without modifying the original string. + fracBuf[fracChars] = *s; + } + + fracBuf[fracChars] = 0; + + // Check to see if we need to round + if (isdigit(*s) && *s > '4') + { + bRound = true; + } + } + + frac = strtoll(fracBuf, &endptr, 10); + value += frac + (bRound ? 1 : 0); + value *= negate; + } + + decimal.value = value; + + if (decimal.value > max_number_decimal) + decimal.value = max_number_decimal; + else if (decimal.value < -max_number_decimal) + decimal.value = -max_number_decimal; + } } break; @@ -1452,7 +1781,11 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, if (!isNull) { - decimal.value = x; + if (max_length > datatypes::INT64MAXPRECISION) + decimal.s128Value = x; + else + decimal.value = x; + decimal.scale = s; } } @@ -1471,7 +1804,11 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, if (!isNull) { - decimal.value = x; + if (max_length > datatypes::INT64MAXPRECISION) + decimal.s128Value = x; + else + decimal.value = x; + decimal.scale = s; } } @@ -1490,7 +1827,11 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, if (!isNull) { - decimal.value = x; + if (max_length > datatypes::INT64MAXPRECISION) + decimal.s128Value = x; + else + decimal.value = x; + decimal.scale = s; } } @@ -1509,7 +1850,11 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, if (!isNull) { - decimal.value = x; + if (max_length > datatypes::INT64MAXPRECISION) + decimal.s128Value = x; + else + decimal.value = x; + decimal.scale = s; } } @@ -1536,6 +1881,11 @@ double Func_cast_decimal::getDoubleVal(Row& row, isNull, operationColType); + if (decimal.precision > datatypes::INT64MAXPRECISION) + { + return datatypes::Decimal::getDoubleFromWideDecimal(decimal.s128Value, decimal.scale); + } + return (double) decimal.value / helpers::powerOf10_c[decimal.scale]; } @@ -1642,7 +1992,14 @@ double Func_cast_double::getDoubleVal(Row& row, { IDB_Decimal decimal = parm[0]->data()->getDecimalVal(row, isNull); - dblval = (double)(decimal.value / pow((double)10, decimal.scale)); + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + dblval = datatypes::Decimal::getDoubleFromWideDecimal(decimal.s128Value, decimal.scale); + } + else + { + dblval = (double)(decimal.value / pow((double)10, decimal.scale)); + } } break; diff --git a/utils/funcexp/func_ceil.cpp b/utils/funcexp/func_ceil.cpp index 967989c74..de374af85 100644 --- a/utils/funcexp/func_ceil.cpp +++ b/utils/funcexp/func_ceil.cpp @@ -65,8 +65,15 @@ int64_t Func_ceil::getIntVal(Row& row, case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::TINYINT: case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: + { + ret = parm[0]->data()->getIntVal(row, isNull); + } + break; + + // ceil(decimal(X,Y)) leads to this path if X, Y allows to + // downcast to INT otherwise Func_ceil::getDecimalVal() is called + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { if (op_ct.scale == 0) { @@ -74,33 +81,48 @@ int64_t Func_ceil::getIntVal(Row& row, break; } - IDB_Decimal decimal = parm[0]->data()->getDecimalVal(row, isNull); + IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); if (isNull) break; - ret = decimal.value; - // negative scale is not supported by CNX yet - if (decimal.scale > 0) + if (d.scale > 0) { - - if (decimal.scale >= 19) + if (d.scale > datatypes::INT128MAXPRECISION) { std::ostringstream oss; - oss << "ceil: datatype of " << colDataTypeToString(op_ct.colDataType) - << " with scale " << (int) decimal.scale << " is beyond supported scale"; + oss << "ceil: datatype of " << execplan::colDataTypeToString(op_ct.colDataType) + << " with scale " << (int) d.scale << " is beyond supported scale"; throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); } - // Adjust to an int based on the scale. - int64_t tmp = ret; - ret /= helpers::powerOf10_c[decimal.scale]; + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t tmp = d.s128Value; + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + d.s128Value /= scaleDivisor; - // Add 1 if this is a positive number and there were values to the right of the - // decimal point so that we return the largest integer value not less than X. - if ((tmp - (ret * helpers::powerOf10_c[decimal.scale]) > 0)) - ret += 1; + // Add 1 if this is a positive number and there were values to the right of the + // decimal point so that we return the largest integer value not less than X. + if ((tmp - (d.s128Value * scaleDivisor)) > 0) + d.s128Value += 1; + + ret = datatypes::Decimal::getInt64FromWideDecimal(d.s128Value); + } + else + { + int64_t tmp = d.value; + d.value /= helpers::powerOf10_c[d.scale]; + + // Add 1 if this is a positive number and there were values to the right of the + // decimal point so that we return the largest integer value not less than X. + if ((tmp - (d.value * helpers::powerOf10_c[d.scale])) > 0) + d.value += 1; + + ret = d.value; + } } } break; @@ -196,13 +218,68 @@ uint64_t Func_ceil::getUintVal(Row& row, case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::TINYINT: case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: { ret = (uint64_t)parm[0]->data()->getIntVal(row, isNull); } break; + // ceil(decimal(X,Y)) leads to this path if X, Y allows to + // downcast to INT otherwise Func_ceil::getDecimalVal() is called + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (op_ct.scale == 0) + { + ret = parm[0]->data()->getIntVal(row, isNull); + break; + } + + IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); + + if (isNull) + break; + + // negative scale is not supported by CNX yet + if (d.scale > 0) + { + if (d.scale > datatypes::INT128MAXPRECISION) + { + std::ostringstream oss; + oss << "ceil: datatype of " << execplan::colDataTypeToString(op_ct.colDataType) + << " with scale " << (int) d.scale << " is beyond supported scale"; + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t tmp = d.s128Value; + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + d.s128Value /= scaleDivisor; + + // Add 1 if this is a positive number and there were values to the right of the + // decimal point so that we return the largest integer value not less than X. + if ((tmp - (d.s128Value * scaleDivisor)) > 0) + d.s128Value += 1; + + ret = datatypes::Decimal::getUInt64FromWideDecimal(d.s128Value); + } + else + { + int64_t tmp = d.value; + d.value /= helpers::powerOf10_c[d.scale]; + + // Add 1 if this is a positive number and there were values to the right of the + // decimal point so that we return the largest integer value not less than X. + if ((tmp - (d.value * helpers::powerOf10_c[d.scale])) > 0) + d.value += 1; + + ret = (uint64_t) d.value; + } + } + } + break; + case CalpontSystemCatalog::UBIGINT: case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::UMEDINT: @@ -308,6 +385,20 @@ double Func_ceil::getDoubleVal(Row& row, { ret = (double)ceill(parm[0]->data()->getLongDoubleVal(row, isNull)); } + else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || + op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) + { + IDB_Decimal tmp = getDecimalVal(row, parm, isNull, op_ct); + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + ret = datatypes::Decimal::getDoubleFromWideDecimal(tmp.s128Value); + } + else + { + ret = (double) tmp.value; + } + } else { if (isUnsigned(op_ct.colDataType)) @@ -350,6 +441,20 @@ long double Func_ceil::getLongDoubleVal(Row& row, if (!isNull) ret = ceil(strtod(str.c_str(), 0)); } + else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || + op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) + { + IDB_Decimal tmp = getDecimalVal(row, parm, isNull, op_ct); + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + ret = datatypes::Decimal::getLongDoubleFromWideDecimal(tmp.s128Value); + } + else + { + ret = (long double) tmp.value; + } + } else { if (isUnsigned(op_ct.colDataType)) @@ -403,6 +508,20 @@ string Func_ceil::getStrVal(Row& row, *d = '\0'; } + else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || + op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) + { + IDB_Decimal d = getDecimalVal(row, parm, isNull, op_ct); + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + return d.toString(true); + } + else + { + return d.toString(); + } + } else if (isUnsigned(op_ct.colDataType)) { #ifndef __LP64__ @@ -424,5 +543,156 @@ string Func_ceil::getStrVal(Row& row, } +IDB_Decimal Func_ceil::getDecimalVal(Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& op_ct) +{ + IDB_Decimal ret; + + switch (op_ct.colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + { + ret.value = parm[0]->data()->getIntVal(row, isNull); + } + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + ret = parm[0]->data()->getDecimalVal(row, isNull); + + if (isNull) + break; + + // negative scale is not supported by CNX yet + if (ret.scale > 0) + { + if (ret.scale > datatypes::INT128MAXPRECISION) + { + std::ostringstream oss; + oss << "ceil: datatype of " << execplan::colDataTypeToString(op_ct.colDataType) + << " with scale " << (int) ret.scale << " is beyond supported scale"; + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t tmp = ret.s128Value; + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, ret.scale); + ret.s128Value /= scaleDivisor; + + // Add 1 if this is a positive number and there were values to the right of the + // decimal point so that we return the largest integer value not less than X. + if ((tmp - (ret.s128Value * scaleDivisor)) > 0) + ret.s128Value += 1; + } + else + { + int64_t tmp = ret.value; + ret.value /= helpers::powerOf10_c[ret.scale]; + + // Add 1 if this is a positive number and there were values to the right of the + // decimal point so that we return the largest integer value not less than X. + if ((tmp - (ret.value * helpers::powerOf10_c[ret.scale])) > 0) + ret.value += 1; + } + } + } + break; + + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + ret.value = (int64_t)parm[0]->data()->getUintVal(row, isNull); + } + break; + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + ret.value = (int64_t) ceil(parm[0]->data()->getDoubleVal(row, isNull)); + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + ret.value = (int64_t) ceill(parm[0]->data()->getLongDoubleVal(row, isNull)); + } + break; + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + const string& str = parm[0]->data()->getStrVal(row, isNull); + + if (!isNull) + ret.value = (int64_t) ceil(strtod(str.c_str(), 0)); + } + break; + + case CalpontSystemCatalog::DATE: + { + Date d (parm[0]->data()->getDateIntVal(row, isNull)); + + if (!isNull) + ret.value = d.convertToMySQLint(); + } + break; + + case CalpontSystemCatalog::DATETIME: + { + DateTime dt(parm[0]->data()->getDatetimeIntVal(row, isNull)); + + if (!isNull) + ret.value = dt.convertToMySQLint(); + } + break; + + case CalpontSystemCatalog::TIMESTAMP: + { + TimeStamp dt(parm[0]->data()->getTimestampIntVal(row, isNull)); + + if (!isNull) + ret.value = dt.convertToMySQLint(timeZone()); + } + break; + + case CalpontSystemCatalog::TIME: + { + Time dt(parm[0]->data()->getTimeIntVal(row, isNull)); + + if (!isNull) + ret.value = dt.convertToMySQLint(); + } + break; + + default: + { + std::ostringstream oss; + oss << "ceil: datatype of " << colDataTypeToString(op_ct.colDataType) + << " is not supported"; + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + ret.scale = 0; + + return ret; +} + + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_char.cpp b/utils/funcexp/func_char.cpp index c4a27730e..f76c36dea 100644 --- a/utils/funcexp/func_char.cpp +++ b/utils/funcexp/func_char.cpp @@ -95,7 +95,7 @@ string Func_char::getStrVal(Row& row, buf[0]= 0; char* pBuf = buf; CHARSET_INFO* cs = ct.getCharset(); - int32_t value; + int32_t value = 0; int32_t numBytes = 0; for (uint32_t i = 0; i < parm.size(); ++i) { @@ -135,14 +135,36 @@ string Func_char::getStrVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = rc->getDecimalVal(row, isNull); - double dscale = d.scale; - // get decimal and round up - value = d.value / pow(10.0, dscale); - int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1); - if ( lefto > 4 ) - value++; - + if (ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + if (d.s128Value < 0) + return ""; + + int128_t scaleDivisor, scaleDivisor2; + + datatypes::getScaleDivisor(scaleDivisor, d.scale); + + scaleDivisor2 = (scaleDivisor <= 10) ? 1 : (scaleDivisor / 10); + + int128_t tmpval = d.s128Value / scaleDivisor; + int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; + + if (lefto > 4) + tmpval++; + + value = datatypes::Decimal::getInt32FromWideDecimal(tmpval); + } + else + { + double dscale = d.scale; + // get decimal and round up + value = d.value / pow(10.0, dscale); + int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1); + + if ( lefto > 4 ) + value++; + } } break; @@ -165,6 +187,7 @@ string Func_char::getStrVal(Row& row, numBytes += getChar(value, pBuf); } + isNull = false; /* Check whether we got a well-formed string */ MY_STRCOPY_STATUS status; diff --git a/utils/funcexp/func_date_add.cpp b/utils/funcexp/func_date_add.cpp index 5004f5521..8c0e547fa 100644 --- a/utils/funcexp/func_date_add.cpp +++ b/utils/funcexp/func_date_add.cpp @@ -749,9 +749,12 @@ int64_t Func_date_add::getIntVal(rowgroup::Row& row, } case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { - if (parm[0]->data()->resultType().scale) + if (parm[0]->data()->resultType().scale == 0) val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + else + isNull = true; break; } diff --git a/utils/funcexp/func_date_format.cpp b/utils/funcexp/func_date_format.cpp index 105069c59..88a1ed99f 100644 --- a/utils/funcexp/func_date_format.cpp +++ b/utils/funcexp/func_date_format.cpp @@ -363,6 +363,7 @@ string Func_date_format::getStrVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -383,6 +384,11 @@ string Func_date_format::getStrVal(rowgroup::Row& row, dt.msecond = (uint32_t)((val & 0xfffff)); } } + else + { + isNull = true; + return ""; + } break; diff --git a/utils/funcexp/func_day.cpp b/utils/funcexp/func_day.cpp index c414fd5d4..9b7def395 100644 --- a/utils/funcexp/func_day.cpp +++ b/utils/funcexp/func_day.cpp @@ -118,6 +118,7 @@ int64_t Func_day::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -132,6 +133,10 @@ int64_t Func_day::getIntVal(rowgroup::Row& row, return (uint32_t)((val >> 38) & 0x3f); } } + else + { + isNull = true; + } break; diff --git a/utils/funcexp/func_dayname.cpp b/utils/funcexp/func_dayname.cpp index e46bfa46f..7a076435b 100644 --- a/utils/funcexp/func_dayname.cpp +++ b/utils/funcexp/func_dayname.cpp @@ -138,6 +138,7 @@ int64_t Func_dayname::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -154,6 +155,11 @@ int64_t Func_dayname::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); } } + else + { + isNull = true; + return -1; + } break; diff --git a/utils/funcexp/func_dayofweek.cpp b/utils/funcexp/func_dayofweek.cpp index 3a29f592d..d0b28d22e 100644 --- a/utils/funcexp/func_dayofweek.cpp +++ b/utils/funcexp/func_dayofweek.cpp @@ -136,6 +136,7 @@ int64_t Func_dayofweek::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -152,6 +153,11 @@ int64_t Func_dayofweek::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); } } + else + { + isNull = true; + return -1; + } break; diff --git a/utils/funcexp/func_dayofyear.cpp b/utils/funcexp/func_dayofyear.cpp index ee13730c1..cf9c37814 100644 --- a/utils/funcexp/func_dayofyear.cpp +++ b/utils/funcexp/func_dayofyear.cpp @@ -75,7 +75,7 @@ int64_t Func_dayofyear::getIntVal(rowgroup::Row& row, { dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; - dataconvert::MySQLTime m_time; + dataconvert::MySQLTime m_time; dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); year = m_time.year; month = m_time.month; @@ -135,6 +135,7 @@ int64_t Func_dayofyear::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); diff --git a/utils/funcexp/func_elt.cpp b/utils/funcexp/func_elt.cpp index ccdca2cae..c9c811d6c 100644 --- a/utils/funcexp/func_elt.cpp +++ b/utils/funcexp/func_elt.cpp @@ -71,17 +71,41 @@ string Func_elt::getStrVal(rowgroup::Row& row, } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - double dscale = d.scale; - number = d.value / pow(10.0, dscale); - int lefto = (d.value - number * pow(10.0, dscale)) / pow(10.0, dscale - 1); - if ( utils::is_nonnegative(number) && lefto > 4 ) - number++; + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t scaleDivisor, scaleDivisor2; - if ( utils::is_negative(number) && lefto < -4 ) - number--; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + + scaleDivisor2 = (scaleDivisor <= 10) ? 1 : (scaleDivisor / 10); + + int128_t tmpval = d.s128Value / scaleDivisor; + int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; + + if (utils::is_nonnegative(tmpval) && lefto > 4) + tmpval++; + + if (utils::is_negative(tmpval) && lefto < -4) + tmpval--; + + number = datatypes::Decimal::getInt64FromWideDecimal(tmpval); + } + else + { + double dscale = d.scale; + number = d.value / pow(10.0, dscale); + int lefto = (d.value - number * pow(10.0, dscale)) / pow(10.0, dscale - 1); + + if ( utils::is_nonnegative(number) && lefto > 4 ) + number++; + + if ( utils::is_negative(number) && lefto < -4 ) + number--; + } break; } diff --git a/utils/funcexp/func_floor.cpp b/utils/funcexp/func_floor.cpp index 1e0ee7f09..3252fa031 100644 --- a/utils/funcexp/func_floor.cpp +++ b/utils/funcexp/func_floor.cpp @@ -62,41 +62,8 @@ int64_t Func_floor::getIntVal(Row& row, case execplan::CalpontSystemCatalog::MEDINT: case execplan::CalpontSystemCatalog::TINYINT: case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: { - if (op_ct.scale == 0) - { - ret = parm[0]->data()->getIntVal(row, isNull); - break; - } - - IDB_Decimal decimal = parm[0]->data()->getDecimalVal(row, isNull); - - if (isNull) - break; - - ret = decimal.value; - - // negative scale is not supported by CNX yet - if (decimal.scale > 0) - { - - if (decimal.scale >= 19) - { - std::ostringstream oss; - oss << "floor: datatype of " << execplan::colDataTypeToString(op_ct.colDataType) - << " with scale " << (int) decimal.scale << " is beyond supported scale"; - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } - - int64_t tmp = ret; - ret /= helpers::powerOf10_c[decimal.scale]; - - // Largest integer value not greater than X. - if (tmp < 0 && tmp < ret) - ret -= 1; - } + ret = parm[0]->data()->getIntVal(row, isNull); } break; @@ -184,6 +151,24 @@ int64_t Func_floor::getIntVal(Row& row, } break; + // floor(decimal(X,Y)) leads to this path if X, Y allows to + // downcast to INT otherwise Func_floor::getDecimalVal() is called + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + IDB_Decimal tmp = getDecimalVal(row, parm, isNull, op_ct); + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + ret = datatypes::Decimal::getInt64FromWideDecimal(tmp.s128Value); + } + else + { + ret = tmp.value; + } + break; + } + default: { std::ostringstream oss; @@ -338,6 +323,20 @@ double Func_floor::getDoubleVal(Row& row, if (!isNull) ret = floor(strtod(str.c_str(), 0)); } + else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || + op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) + { + IDB_Decimal tmp = getDecimalVal(row, parm, isNull, op_ct); + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + ret = datatypes::Decimal::getDoubleFromWideDecimal(tmp.s128Value); + } + else + { + ret = (double) tmp.value; + } + } else { ret = (double) getIntVal(row, parm, isNull, op_ct); @@ -371,6 +370,20 @@ long double Func_floor::getLongDoubleVal(Row& row, if (!isNull) ret = floor(strtod(str.c_str(), 0)); } + else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || + op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) + { + IDB_Decimal tmp = getDecimalVal(row, parm, isNull, op_ct); + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + ret = datatypes::Decimal::getLongDoubleFromWideDecimal(tmp.s128Value); + } + else + { + ret = (long double) tmp.value; + } + } else { ret = (long double) getIntVal(row, parm, isNull, op_ct); @@ -416,6 +429,20 @@ string Func_floor::getStrVal(Row& row, *d = '\0'; } + else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || + op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) + { + IDB_Decimal d = getDecimalVal(row, parm, isNull, op_ct); + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + return d.toString(true); + } + else + { + return d.toString(); + } + } else if (isUnsigned(op_ct.colDataType)) { #ifndef __LP64__ @@ -436,6 +463,165 @@ string Func_floor::getStrVal(Row& row, return string(tmp); } +IDB_Decimal Func_floor::getDecimalVal(Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& op_ct) +{ + IDB_Decimal ret; + + switch (op_ct.colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + { + ret.value = parm[0]->data()->getIntVal(row, isNull); + } + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + ret = parm[0]->data()->getDecimalVal(row, isNull); + + if (isNull) + break; + + // negative scale is not supported by CNX yet + if (ret.scale > 0) + { + if (ret.scale > datatypes::INT128MAXPRECISION) + { + std::ostringstream oss; + oss << "floor: datatype of " << execplan::colDataTypeToString(op_ct.colDataType) + << " with scale " << (int) ret.scale << " is beyond supported scale"; + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t tmp = ret.s128Value; + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, ret.scale); + ret.s128Value /= scaleDivisor; + + // Largest integer value not greater than X. + if (tmp < 0 && tmp < ret.s128Value) + ret.s128Value -= 1; + } + else + { + int64_t tmp = ret.value; + ret.value /= helpers::powerOf10_c[ret.scale]; + + // Largest integer value not greater than X. + if (tmp < 0 && tmp < ret.value) + ret.value -= 1; + } + } + } + break; + + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + ret.value = (int64_t)parm[0]->data()->getUintVal(row, isNull); + } + break; + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + ret.value = (int64_t) floor(parm[0]->data()->getDoubleVal(row, isNull)); + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + ret.value = (int64_t) floorl(parm[0]->data()->getLongDoubleVal(row, isNull)); + } + break; + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + const string& str = parm[0]->data()->getStrVal(row, isNull); + + if (!isNull) + ret.value = (int64_t) floor(strtod(str.c_str(), 0)); + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + string str = DataConvert::dateToString1(parm[0]->data()->getDateIntVal(row, isNull)); + + if (!isNull) + ret.value = atoll(str.c_str()); + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + string str = + DataConvert::datetimeToString1(parm[0]->data()->getDatetimeIntVal(row, isNull)); + + // strip off micro seconds + str = str.substr(0, 14); + + if (!isNull) + ret.value = atoll(str.c_str()); + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + string str = + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); + + // strip off micro seconds + str = str.substr(0, 14); + + if (!isNull) + ret.value = atoll(str.c_str()); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + string str = + DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + + // strip off micro seconds + str = str.substr(0, 14); + + if (!isNull) + ret.value = atoll(str.c_str()); + } + break; + + default: + { + std::ostringstream oss; + oss << "floor: datatype of " << execplan::colDataTypeToString(op_ct.colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + ret.scale = 0; + + return ret; +} + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_from_unixtime.cpp b/utils/funcexp/func_from_unixtime.cpp index d1168d2d6..ab88e39e0 100644 --- a/utils/funcexp/func_from_unixtime.cpp +++ b/utils/funcexp/func_from_unixtime.cpp @@ -61,10 +61,22 @@ DateTime getDateTime(rowgroup::Row& row, break; } case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal dec = parm[0]->data()->getDecimalVal(row, isNull); - val = dec.value / IDB_pow[dec.scale]; - msec = dec.value % IDB_pow[dec.scale]; + + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, dec.scale); + val = datatypes::Decimal::getInt64FromWideDecimal(dec.s128Value / scaleDivisor); + msec = datatypes::Decimal::getUInt32FromWideDecimal(dec.s128Value % scaleDivisor); + } + else + { + val = dec.value / IDB_pow[dec.scale]; + msec = dec.value % IDB_pow[dec.scale]; + } break; } diff --git a/utils/funcexp/func_hex.cpp b/utils/funcexp/func_hex.cpp index 42e6b04d2..b45f6b52a 100644 --- a/utils/funcexp/func_hex.cpp +++ b/utils/funcexp/func_hex.cpp @@ -91,6 +91,7 @@ string Func_hex::getStrVal(rowgroup::Row& row, case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { /* Return hex of unsigned longlong value */ double val = parm[0]->data()->getDoubleVal(row, isNull); @@ -130,6 +131,15 @@ string Func_hex::getStrVal(rowgroup::Row& row, return string(hexPtr.get(), hexLen); } + case CalpontSystemCatalog::BINARY: + { + const string& arg = parm[0]->data()->getStrVal(row, isNull); + uint64_t hexLen = arg.size() * 2; + scoped_array hexPtr(new char[hexLen + 1]); // "+ 1" for the last \0 + octet2hex(hexPtr.get(), arg.data(), arg.size()); + return string(hexPtr.get(), hexLen); + } + default: { dec = (uint64_t)parm[0]->data()->getIntVal(row, isNull); diff --git a/utils/funcexp/func_hour.cpp b/utils/funcexp/func_hour.cpp index 918fe193d..af949a393 100644 --- a/utils/funcexp/func_hour.cpp +++ b/utils/funcexp/func_hour.cpp @@ -67,14 +67,19 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, } case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { - if (parm[0]->data()->resultType().scale) + if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); if (val == -1) isNull = true; } + else + { + isNull = true; + } break; } diff --git a/utils/funcexp/func_if.cpp b/utils/funcexp/func_if.cpp index 6d10d6829..168383b93 100644 --- a/utils/funcexp/func_if.cpp +++ b/utils/funcexp/func_if.cpp @@ -43,7 +43,7 @@ bool boolVal(SPTP& parm, Row& row, const string& timeZone) try { - ret = parm->getBoolVal(row, isNull); + ret = parm->getBoolVal(row, isNull) && !isNull; } catch (logging::NotImplementedExcept&) { @@ -67,7 +67,10 @@ bool boolVal(SPTP& parm, Row& row, const string& timeZone) break; case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - ret = (parm->data()->getDecimalVal(row, isNull).value != 0); + if (parm->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + ret = (parm->data()->getDecimalVal(row, isNull).s128Value != 0); + else + ret = (parm->data()->getDecimalVal(row, isNull).value != 0); break; case CalpontSystemCatalog::BIGINT: case CalpontSystemCatalog::SMALLINT: diff --git a/utils/funcexp/func_inet_aton.cpp b/utils/funcexp/func_inet_aton.cpp index 4bcdc3115..2f0e5295e 100644 --- a/utils/funcexp/func_inet_aton.cpp +++ b/utils/funcexp/func_inet_aton.cpp @@ -154,21 +154,34 @@ execplan::IDB_Decimal Func_inet_aton::getDecimalVal(rowgroup::Row& row, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { -// std::cout << "In Func_inet_aton::getDecimalVal" << std::endl; - - execplan::IDB_Decimal dValue ( joblist::NULL_INT64, 0, 0 ); + execplan::CalpontSystemCatalog::ColType colType = fp[0]->data()->resultType(); const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); - if (!isNull) + if (colType.precision <= datatypes::INT64MAXPRECISION) { - int64_t iValue = convertAton( sValue, isNull ); - if (!isNull) - return execplan::IDB_Decimal( iValue, 0, 0 ); - } + { + int64_t iValue = convertAton( sValue, isNull ); - return dValue; + if (!isNull) + return execplan::IDB_Decimal( iValue, colType.scale, colType.precision ); + } + + return execplan::IDB_Decimal( joblist::NULL_INT64, colType.scale, colType.precision ); + } + else + { + if (!isNull) + { + int64_t iValue = convertAton( sValue, isNull ); + + if (!isNull) + return execplan::IDB_Decimal( 0, colType.scale, colType.precision, (int128_t) iValue ); + } + + return execplan::IDB_Decimal( 0, colType.scale, colType.precision, datatypes::Decimal128Null ); + } } //------------------------------------------------------------------------------ diff --git a/utils/funcexp/func_inet_ntoa.cpp b/utils/funcexp/func_inet_ntoa.cpp index b9a31b08e..ea528ceb6 100644 --- a/utils/funcexp/func_inet_ntoa.cpp +++ b/utils/funcexp/func_inet_ntoa.cpp @@ -157,6 +157,8 @@ std::string Func_inet_ntoa::getStrVal(rowgroup::Row& row, // else just get integer value if ((fp[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::DECIMAL) || + (fp[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::UDECIMAL) || (fp[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::FLOAT) || (fp[0]->data()->resultType().colDataType == @@ -211,8 +213,6 @@ execplan::IDB_Decimal Func_inet_ntoa::getDecimalVal(rowgroup::Row& row, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { -// std::cout << "In Func_inet_ntoa::getDecimalVal" << std::endl; - // IDB_Decimal dValue = fp[0]->data()->getDecimalVal(row, isNull); execplan::IDB_Decimal dValue ( joblist::NULL_INT64, 0, 0 ); isNull = true; diff --git a/utils/funcexp/func_last_day.cpp b/utils/funcexp/func_last_day.cpp index 2099ea71a..b20e8502f 100644 --- a/utils/funcexp/func_last_day.cpp +++ b/utils/funcexp/func_last_day.cpp @@ -136,6 +136,7 @@ int64_t Func_last_day::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -152,6 +153,11 @@ int64_t Func_last_day::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); } } + else + { + isNull = true; + return -1; + } break; diff --git a/utils/funcexp/func_makedate.cpp b/utils/funcexp/func_makedate.cpp index 8c8c50abf..64c2d5e5d 100644 --- a/utils/funcexp/func_makedate.cpp +++ b/utils/funcexp/func_makedate.cpp @@ -66,17 +66,41 @@ uint64_t makedate(rowgroup::Row& row, } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - double dscale = d.scale; - year = d.value / pow(10.0, dscale); - int lefto = (d.value - year * pow(10.0, dscale)) / pow(10.0, dscale - 1); - if ( year >= 0 && lefto > 4 ) - year++; + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t scaleDivisor, scaleDivisor2; - if ( year < 0 && lefto < -4 ) - year--; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + + scaleDivisor2 = (scaleDivisor <= 10) ? 1 : (scaleDivisor / 10); + + int128_t tmpval = d.s128Value / scaleDivisor; + int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; + + if (tmpval >= 0 && lefto > 4) + tmpval++; + + if (tmpval < 0 && lefto < -4) + tmpval--; + + year = datatypes::Decimal::getInt64FromWideDecimal(tmpval); + } + else + { + double dscale = d.scale; + year = d.value / pow(10.0, dscale); + int lefto = (d.value - year * pow(10.0, dscale)) / pow(10.0, dscale - 1); + + if ( year >= 0 && lefto > 4 ) + year++; + + if ( year < 0 && lefto < -4 ) + year--; + } break; } @@ -126,25 +150,56 @@ uint64_t makedate(rowgroup::Row& row, } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[1]->data()->getDecimalVal(row, isNull); - double dscale = d.scale; - int64_t tmp = d.value / pow(10.0, dscale); - int lefto = (d.value - tmp * pow(10.0, dscale)) / pow(10.0, dscale - 1); - if ( tmp >= 0 && lefto > 4 ) - tmp++; - - if ( tmp < 0 && lefto < -4 ) - tmp--; - - if (tmp < 1) + if (parm[1]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) { - isNull = true; - return 0; - } + int128_t scaleDivisor, scaleDivisor2; - dayofyear = helpers::intToString(tmp); + datatypes::getScaleDivisor(scaleDivisor, d.scale); + + scaleDivisor2 = (scaleDivisor <= 10) ? 1 : (scaleDivisor / 10); + + int128_t tmpval = d.s128Value / scaleDivisor; + int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; + + if (tmpval >= 0 && lefto > 4) + tmpval++; + + if (tmpval < 0 && lefto < -4) + tmpval--; + + if (tmpval < 1) + { + isNull = true; + return 0; + } + + int64_t tmpval64 = datatypes::Decimal::getInt64FromWideDecimal(tmpval); + dayofyear = helpers::intToString(tmpval64); + } + else + { + double dscale = d.scale; + int64_t tmp = d.value / pow(10.0, dscale); + int lefto = (d.value - tmp * pow(10.0, dscale)) / pow(10.0, dscale - 1); + + if (tmp >= 0 && lefto > 4) + tmp++; + + if (tmp < 0 && lefto < -4) + tmp--; + + if (tmp < 1) + { + isNull = true; + return 0; + } + + dayofyear = helpers::intToString(tmp); + } break; } diff --git a/utils/funcexp/func_maketime.cpp b/utils/funcexp/func_maketime.cpp index b43fed4ab..6e315c7ec 100644 --- a/utils/funcexp/func_maketime.cpp +++ b/utils/funcexp/func_maketime.cpp @@ -72,17 +72,41 @@ string Func_maketime::getStrVal(rowgroup::Row& row, } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - double dscale = d.scale; - hour = d.value / pow(10.0, dscale); - int lefto = (d.value - hour * pow(10.0, dscale)) / pow(10.0, dscale - 1); - if ( hour >= 0 && lefto > 4 ) - hour++; + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t scaleDivisor, scaleDivisor2; - if ( hour < 0 && lefto < -4 ) - hour--; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + + scaleDivisor2 = (scaleDivisor <= 10) ? 1 : (scaleDivisor / 10); + + int128_t tmpval = d.s128Value / scaleDivisor; + int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; + + if (tmpval >= 0 && lefto > 4) + tmpval++; + + if (tmpval < 0 && lefto < -4) + tmpval--; + + hour = datatypes::Decimal::getInt64FromWideDecimal(tmpval); + } + else + { + double dscale = d.scale; + hour = d.value / pow(10.0, dscale); + int lefto = (d.value - hour * pow(10.0, dscale)) / pow(10.0, dscale - 1); + + if ( hour >= 0 && lefto > 4 ) + hour++; + + if ( hour < 0 && lefto < -4 ) + hour--; + } break; } @@ -112,17 +136,41 @@ string Func_maketime::getStrVal(rowgroup::Row& row, } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[1]->data()->getDecimalVal(row, isNull); - double dscale = d.scale; - min = d.value / pow(10.0, dscale); - int lefto = (d.value - min * pow(10.0, dscale)) / pow(10.0, dscale - 1); - if ( min >= 0 && lefto > 4 ) - min++; + if (parm[1]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t scaleDivisor, scaleDivisor2; - if ( min < 0 && lefto < -4 ) - min--; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + + scaleDivisor2 = (scaleDivisor <= 10) ? 1 : (scaleDivisor / 10); + + int128_t tmpval = d.s128Value / scaleDivisor; + int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; + + if (tmpval >= 0 && lefto > 4) + tmpval++; + + if (tmpval < 0 && lefto < -4) + tmpval--; + + min = datatypes::Decimal::getInt64FromWideDecimal(tmpval); + } + else + { + double dscale = d.scale; + min = d.value / pow(10.0, dscale); + int lefto = (d.value - min * pow(10.0, dscale)) / pow(10.0, dscale - 1); + + if ( min >= 0 && lefto > 4 ) + min++; + + if ( min < 0 && lefto < -4 ) + min--; + } break; } @@ -158,17 +206,41 @@ string Func_maketime::getStrVal(rowgroup::Row& row, } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[2]->data()->getDecimalVal(row, isNull); - double dscale = d.scale; - sec = d.value / pow(10.0, dscale); - int lefto = (d.value - sec * pow(10.0, dscale)) / pow(10.0, dscale - 1); - if ( sec >= 0 && lefto > 4 ) - sec++; + if (parm[2]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t scaleDivisor, scaleDivisor2; - if ( sec < 0 && lefto < -4 ) - sec--; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + + scaleDivisor2 = (scaleDivisor <= 10) ? 1 : (scaleDivisor / 10); + + int128_t tmpval = d.s128Value / scaleDivisor; + int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; + + if (tmpval >= 0 && lefto > 4) + tmpval++; + + if (tmpval < 0 && lefto < -4) + tmpval--; + + sec = datatypes::Decimal::getInt64FromWideDecimal(tmpval); + } + else + { + double dscale = d.scale; + sec = d.value / pow(10.0, dscale); + int lefto = (d.value - sec * pow(10.0, dscale)) / pow(10.0, dscale - 1); + + if ( sec >= 0 && lefto > 4 ) + sec++; + + if ( sec < 0 && lefto < -4 ) + sec--; + } break; } diff --git a/utils/funcexp/func_math.cpp b/utils/funcexp/func_math.cpp index 13c738217..a64e8fd75 100644 --- a/utils/funcexp/func_math.cpp +++ b/utils/funcexp/func_math.cpp @@ -1849,71 +1849,129 @@ string Func_format::getStrVal(Row& row, { IDB_Decimal decimal = parm[0]->data()->getDecimalVal(row, isNull); + // This is an unacceptable way of doing rounding //perform rouding if needed if ( scale < 0 ) scale = 0; - if ( scale < decimal.scale ) + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) { - int64_t d = 0; - int64_t p = 1; - - if (!isNull && parm.size() > 1) + if ( scale < decimal.scale ) { - d = scale; + int64_t d = 0; + int128_t p = 1; - if (!isNull) - helpers::decimalPlaceDec(d, p, decimal.scale); - } - - if (isNull) - break; - - int64_t x = decimal.value; - - if (d > 0) - { - x = x * p; - } - else if (d < 0) - { - int64_t h = p / 2; // 0.5 - - if ((x >= h) || (x <= -h)) + if (!isNull && parm.size() > 1) { - if (x >= 0) - x += h; - else - x -= h; + d = scale; - if (p != 0) - x = x / p; + if (!isNull) + helpers::decimalPlaceDec(d, p, decimal.scale); + } + + if (isNull) + break; + + int128_t x = decimal.s128Value; + + if (d > 0) + { + x = x * p; + } + else if (d < 0) + { + int128_t h = p / 2; // 0.5 + + if ((x >= h) || (x <= -h)) + { + if (x >= 0) + x += h; + else + x -= h; + + if (p != 0) + x = x / p; + else + x = 0; + } else + { x = 0; + } } - else + + // negative scale is not supported by CNX yet, set d to 0. + if (decimal.scale < 0) { - x = 0; + do + x *= 10; + + while (++decimal.scale < 0); } + + decimal.s128Value = x; } - - // negative scale is not supported by CNX yet, set d to 0. - if (decimal.scale < 0) - { - do - x *= 10; - - while (++decimal.scale < 0); - } - - decimal.value = x; + value = decimal.toString(true); } + else + { + if ( scale < decimal.scale ) + { + int64_t d = 0; + int64_t p = 1; - char buf[80]; + if (!isNull && parm.size() > 1) + { + d = scale; - dataconvert::DataConvert::decimalToString( decimal.value, decimal.scale, buf, 80, parm[0]->data()->resultType().colDataType); + if (!isNull) + helpers::decimalPlaceDec(d, p, decimal.scale); + } - value = buf; + if (isNull) + break; + + int64_t x = decimal.value; + + if (d > 0) + { + x = x * p; + } + else if (d < 0) + { + int64_t h = p / 2; // 0.5 + + if ((x >= h) || (x <= -h)) + { + if (x >= 0) + x += h; + else + x -= h; + + if (p != 0) + x = x / p; + else + x = 0; + } + else + { + x = 0; + } + } + + // negative scale is not supported by CNX yet, set d to 0. + if (decimal.scale < 0) + { + do + x *= 10; + + while (++decimal.scale < 0); + } + + decimal.value = x; + } + value = decimal.toString(); + } } break; @@ -1981,7 +2039,7 @@ string Func_format::getStrVal(Row& row, // pad extra with '0' if (*(value.data()) != '#') { - for ( int i = 0 ; i < pad ; i ++ ) + for ( int i = 0 ; i < pad ; i++ ) { value = value.append("0"); } @@ -1995,7 +2053,7 @@ string Func_format::getStrVal(Row& row, string::size_type pos = value.find ('-', 0); if (pos != string::npos) - end = 1;; + end = 1; while ((comma -= 3) > end) { diff --git a/utils/funcexp/func_microsecond.cpp b/utils/funcexp/func_microsecond.cpp index 12b2beb70..18b1cdabb 100644 --- a/utils/funcexp/func_microsecond.cpp +++ b/utils/funcexp/func_microsecond.cpp @@ -110,6 +110,7 @@ int64_t Func_microsecond::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -124,6 +125,11 @@ int64_t Func_microsecond::getIntVal(rowgroup::Row& row, microSecond = (uint32_t)((val & 0xfffff)); } } + else + { + isNull = true; + return -1; + } break; diff --git a/utils/funcexp/func_minute.cpp b/utils/funcexp/func_minute.cpp index fdad4e9cf..672008048 100644 --- a/utils/funcexp/func_minute.cpp +++ b/utils/funcexp/func_minute.cpp @@ -66,14 +66,19 @@ int64_t Func_minute::getIntVal(rowgroup::Row& row, } case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { - if (parm[0]->data()->resultType().scale) + if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); if (val == -1) isNull = true; } + else + { + isNull = true; + } break; } diff --git a/utils/funcexp/func_mod.cpp b/utils/funcexp/func_mod.cpp index 214fd2b6c..01432cea8 100644 --- a/utils/funcexp/func_mod.cpp +++ b/utils/funcexp/func_mod.cpp @@ -25,6 +25,8 @@ #include using namespace std; +#include + #include "functor_real.h" #include "funchelpers.h" #include "functioncolumn.h" @@ -54,7 +56,6 @@ IDB_Decimal Func_mod::getDecimalVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - IDB_Decimal retValue; retValue.value = 0; retValue.scale = 0; @@ -65,22 +66,85 @@ IDB_Decimal Func_mod::getDecimalVal(Row& row, return retValue; } - int64_t div = parm[1]->data()->getIntVal(row, isNull); - - if ( div == 0 ) + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH || + parm[1]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) { - isNull = true; - return retValue; + IDB_Decimal div = parm[1]->data()->getDecimalVal(row, isNull); + + int128_t divInt, dividendInt; + + if (parm[1]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + divInt = div.s128Value; + else + divInt = div.value; + + if (divInt == 0) + { + isNull = true; + return retValue; + } + + IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); + + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + dividendInt = d.s128Value; + else + dividendInt = d.value; + + // integer division + if (d.scale == 0 && div.scale == 0) + { + retValue.s128Value = dividendInt % divInt; + } + // special case integer division + else if (div.scale == 0) + { + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + int128_t value = dividendInt / scaleDivisor; + int128_t lefto = dividendInt % scaleDivisor; + int128_t mod = (value % divInt) * scaleDivisor + lefto; + retValue.s128Value = mod; + } + // float division + else + { + __float128 divF, dividendF; + + int128_t scaleDivisor; + + datatypes::getScaleDivisor(scaleDivisor, div.scale); + divF = (__float128) divInt / scaleDivisor; + + datatypes::getScaleDivisor(scaleDivisor, d.scale); + dividendF = (__float128) dividendInt / scaleDivisor; + + __float128 mod = fmodq(dividendF, divF) * scaleDivisor; + + retValue.s128Value = (int128_t) mod; + } + retValue.scale = d.scale; + retValue.precision = datatypes::INT128MAXPRECISION; } + else + { + int64_t div = parm[1]->data()->getIntVal(row, isNull); - IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - int64_t value = d.value / pow(10.0, d.scale); - int lefto = d.value % (int)pow(10.0, d.scale); + if ( div == 0 ) + { + isNull = true; + return retValue; + } - int64_t mod = (value % div) * pow(10.0, d.scale) + lefto; + IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); + int64_t value = d.value / pow(10.0, d.scale); + int lefto = d.value % (int)pow(10.0, d.scale); - retValue.value = mod; - retValue.scale = d.scale; + int64_t mod = (value % div) * pow(10.0, d.scale) + lefto; + + retValue.value = mod; + retValue.scale = d.scale; + } return retValue; } @@ -164,9 +228,28 @@ double Func_mod::getDoubleVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - int64_t value = d.value / pow(10.0, d.scale); - mod = value % div; + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + if (d.scale == 0) + { + mod = d.s128Value % div; + } + else + { + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + int128_t value = d.s128Value / scaleDivisor; + int128_t lefto = d.s128Value % scaleDivisor; + __float128 tmp = (__float128) (value % div) + (__float128) lefto / scaleDivisor; + mod = datatypes::getDoubleFromFloat128(tmp); + } + } + else + { + int64_t value = d.value / pow(10.0, d.scale); + mod = value % div; + } } break; @@ -268,9 +351,28 @@ long double Func_mod::getLongDoubleVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - int64_t value = d.value / pow(10.0, d.scale); - mod = value % div; + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + if (d.scale == 0) + { + mod = d.s128Value % div; + } + else + { + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + int128_t value = d.s128Value / scaleDivisor; + int128_t lefto = d.s128Value % scaleDivisor; + __float128 tmp = (__float128) (value % div) + (__float128) lefto / scaleDivisor; + mod = datatypes::getLongDoubleFromFloat128(tmp); + } + } + else + { + int64_t value = d.value / pow(10.0, d.scale); + mod = value % div; + } } break; @@ -375,9 +477,28 @@ int64_t Func_mod::getIntVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - int64_t value = d.value / pow(10.0, d.scale); - mod = value % div; + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + if (d.scale == 0) + { + mod = d.s128Value % div; + } + else + { + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + int128_t value = d.s128Value / scaleDivisor; + int128_t lefto = d.s128Value % scaleDivisor; + __float128 tmp = (__float128) (value % div) + (__float128) lefto / scaleDivisor; + mod = datatypes::Decimal::getInt64FromFloat128(tmp); + } + } + else + { + int64_t value = d.value / pow(10.0, d.scale); + mod = value % div; + } } break; @@ -473,9 +594,28 @@ uint64_t Func_mod::getUIntVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - int64_t value = d.value / pow(10.0, d.scale); - mod = value % div; + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + if (d.scale == 0) + { + mod = d.s128Value % div; + } + else + { + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + int128_t value = d.s128Value / scaleDivisor; + int128_t lefto = d.s128Value % scaleDivisor; + __float128 tmp = (__float128) (value % div) + (__float128) lefto / scaleDivisor; + mod = datatypes::Decimal::getUInt64FromFloat128(tmp); + } + } + else + { + int64_t value = d.value / pow(10.0, d.scale); + mod = value % div; + } } break; diff --git a/utils/funcexp/func_month.cpp b/utils/funcexp/func_month.cpp index ffb3f5fc1..f9cbfddd0 100644 --- a/utils/funcexp/func_month.cpp +++ b/utils/funcexp/func_month.cpp @@ -116,6 +116,7 @@ int64_t Func_month::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -130,6 +131,10 @@ int64_t Func_month::getIntVal(rowgroup::Row& row, return (unsigned)((val >> 44) & 0xf); } } + else + { + isNull = true; + } break; diff --git a/utils/funcexp/func_monthname.cpp b/utils/funcexp/func_monthname.cpp index 7c27c9581..7d41e3ba4 100644 --- a/utils/funcexp/func_monthname.cpp +++ b/utils/funcexp/func_monthname.cpp @@ -158,6 +158,7 @@ int64_t Func_monthname::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -197,7 +198,11 @@ execplan::IDB_Decimal Func_monthname::getDecimalVal(rowgroup::Row& row, execplan::CalpontSystemCatalog::ColType& op_ct) { IDB_Decimal d; - d.value = getIntVal(row, fp, isNull, op_ct); + + if (fp[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + d.s128Value = getIntVal(row, fp, isNull, op_ct); + else + d.value = getIntVal(row, fp, isNull, op_ct); d.scale = 0; return d; } diff --git a/utils/funcexp/func_nullif.cpp b/utils/funcexp/func_nullif.cpp index 90c546ea3..886c58f9c 100644 --- a/utils/funcexp/func_nullif.cpp +++ b/utils/funcexp/func_nullif.cpp @@ -430,6 +430,7 @@ int32_t Func_nullif::getDateIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -518,6 +519,7 @@ int64_t Func_nullif::getDatetimeIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -584,6 +586,7 @@ int64_t Func_nullif::getTimeIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -636,6 +639,7 @@ int64_t Func_nullif::getTimestampIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -689,6 +693,7 @@ double Func_nullif::getDoubleVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -779,6 +784,7 @@ long double Func_nullif::getLongDoubleVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: diff --git a/utils/funcexp/func_period_diff.cpp b/utils/funcexp/func_period_diff.cpp index 0f1b923c9..3f344fd8f 100644 --- a/utils/funcexp/func_period_diff.cpp +++ b/utils/funcexp/func_period_diff.cpp @@ -86,7 +86,24 @@ int64_t Func_period_diff::getIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - period1 = d.value / pow(10.0, d.scale); + + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + int128_t tmpval = d.s128Value / scaleDivisor; + + if (tmpval > static_cast(INT64_MAX)) + tmpval = INT64_MAX; + else if (tmpval < static_cast(INT64_MIN)) + tmpval = INT64_MIN; + + period1 = tmpval; + } + else + { + period1 = d.value / pow(10.0, d.scale); + } break; } @@ -135,7 +152,24 @@ int64_t Func_period_diff::getIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[1]->data()->getDecimalVal(row, isNull); - period2 = d.value / pow(10.0, d.scale); + + if (parm[1]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + int128_t tmpval = d.s128Value / scaleDivisor; + + if (tmpval > static_cast(INT64_MAX)) + tmpval = INT64_MAX; + else if (tmpval < static_cast(INT64_MIN)) + tmpval = INT64_MIN; + + period2 = tmpval; + } + else + { + period2 = d.value / pow(10.0, d.scale); + } break; } diff --git a/utils/funcexp/func_quarter.cpp b/utils/funcexp/func_quarter.cpp index c1dcda89e..35b9d93f4 100644 --- a/utils/funcexp/func_quarter.cpp +++ b/utils/funcexp/func_quarter.cpp @@ -115,6 +115,7 @@ int64_t Func_quarter::getIntVal(rowgroup::Row& row, } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { if (parm[0]->data()->resultType().scale == 0) { diff --git a/utils/funcexp/func_regexp.cpp b/utils/funcexp/func_regexp.cpp index 120a727e6..0fa61b0a0 100644 --- a/utils/funcexp/func_regexp.cpp +++ b/utils/funcexp/func_regexp.cpp @@ -116,11 +116,15 @@ inline bool getBool(rowgroup::Row& row, { IDB_Decimal d = pm[0]->data()->getDecimalVal(row, isNull); - char buf[80]; + if (pm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + expr = d.toString(true); + } + else + { + expr = d.toString(); + } - dataconvert::DataConvert::decimalToString(d.value, d.scale, buf, 80, pm[0]->data()->resultType().colDataType); - - expr = buf; break; } @@ -191,11 +195,14 @@ inline bool getBool(rowgroup::Row& row, { IDB_Decimal d = pm[1]->data()->getDecimalVal(row, isNull); - char buf[80]; - - dataconvert::DataConvert::decimalToString( d.value, d.scale, buf, 80, pm[1]->data()->resultType().colDataType); - - pattern = buf; + if (pm[1]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + pattern = d.toString(true); + } + else + { + pattern = d.toString(); + } break; } diff --git a/utils/funcexp/func_round.cpp b/utils/funcexp/func_round.cpp index 0064ae8df..b034cf3b3 100644 --- a/utils/funcexp/func_round.cpp +++ b/utils/funcexp/func_round.cpp @@ -116,18 +116,36 @@ int64_t Func_round::getIntVal(Row& row, { IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); - if (x.scale > 0) + if (!op_ct.isWideDecimalType()) { - while (x.scale-- > 0) - x.value /= 10; + if (x.scale > 0) + { + while (x.scale-- > 0) + x.value /= 10; + } + else + { + while (x.scale++ < 0) + x.value *= 10; + } + + return x.value; } else { - while (x.scale++ < 0) - x.value *= 10; - } + if (x.scale > 0) + { + while (x.scale-- > 0) + x.s128Value /= 10; + } + else + { + while (x.scale++ < 0) + x.s128Value *= 10; + } - return x.value; + return datatypes::Decimal::getInt64FromWideDecimal(x.s128Value); + } } @@ -187,7 +205,12 @@ double Func_round::getDoubleVal(Row& row, if (isNull) return 0.0; - double d = x.value; + double d; + + if (!op_ct.isWideDecimalType()) + d = x.value; + else + d = datatypes::Decimal::getDoubleFromWideDecimal(x.s128Value); if (x.scale > 0) { @@ -249,7 +272,12 @@ long double Func_round::getLongDoubleVal(Row& row, if (isNull) return 0.0; - double d = x.value; + double d; + + if (!op_ct.isWideDecimalType()) + d = x.value; + else + d = datatypes::Decimal::getDoubleFromWideDecimal(x.s128Value); if (x.scale > 0) { @@ -283,62 +311,130 @@ IDB_Decimal Func_round::getDecimalVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { int64_t d = 0; - //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. - volatile int64_t p = 1; decimal = parm[0]->data()->getDecimalVal(row, isNull); - if (!isNull && parm.size() > 1) // round(X, D) + if (!op_ct.isWideDecimalType()) { - int64_t nvp = p; - d = parm[1]->data()->getIntVal(row, isNull); + //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. + volatile int64_t p = 1; - if (!isNull) - helpers::decimalPlaceDec(d, nvp, decimal.scale); - - p = nvp; - } - - if (isNull) - break; - - int64_t x = decimal.value; - - if (d > 0) - { - x = x * p; - } - else if (d < 0) - { - int64_t h = p / 2; // 0.5 - - if ((x >= h) || (x <= -h)) + if (!isNull && parm.size() > 1) // round(X, D) { - if (x >= 0) - x += h; - else - x -= h; + int64_t nvp = p; + d = parm[1]->data()->getIntVal(row, isNull); - if (p != 0) - x = x / p; + if (!isNull) + helpers::decimalPlaceDec(d, nvp, decimal.scale); + + p = nvp; + } + + if (isNull) + break; + + int64_t x = decimal.value; + + if (d > 0) + { + x = x * p; + } + else if (d < 0) + { + int64_t h = p / 2; // 0.5 + + if ((x >= h) || (x <= -h)) + { + if (x >= 0) + x += h; + else + x -= h; + + if (p != 0) + x = x / p; + else + x = 0; + } else + { x = 0; + } } - else + + // negative scale is not supported by CNX yet, set d to 0. + if (decimal.scale < 0) { - x = 0; + do + x *= 10; + + while (++decimal.scale < 0); } - } - // negative scale is not supported by CNX yet, set d to 0. - if (decimal.scale < 0) + decimal.value = x; + } + else { - do - x *= 10; + //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. + volatile int128_t p = 1; - while (++decimal.scale < 0); + if (!isNull && parm.size() > 1) // round(X, D) + { + int128_t nvp = p; + d = parm[1]->data()->getIntVal(row, isNull); + + if (!isNull) + helpers::decimalPlaceDec(d, nvp, decimal.scale); + + p = nvp; + } + + if (isNull) + break; + + if (d < -datatypes::INT128MAXPRECISION) + { + decimal.s128Value = 0; + break; + } + + int128_t x = decimal.s128Value; + + if (d > 0) + { + x = x * p; + } + else if (d < 0) + { + int128_t h = p / 2; // 0.5 + + if ((x >= h) || (x <= -h)) + { + if (x >= 0) + x += h; + else + x -= h; + + if (p != 0) + x = x / p; + else + x = 0; + } + else + { + x = 0; + } + } + + // negative scale is not supported by CNX yet, set d to 0. + if (decimal.scale < 0) + { + do + x *= 10; + + while (++decimal.scale < 0); + } + + decimal.s128Value = x; } - - decimal.value = x; } break; @@ -624,7 +720,10 @@ string Func_round::getStrVal(Row& row, break; } - return dataconvert::DataConvert::decimalToString(x.value, x.scale, op_ct.colDataType); + if (!op_ct.isWideDecimalType()) + return x.toString(); + else + return x.toString(true); } diff --git a/utils/funcexp/func_sec_to_time.cpp b/utils/funcexp/func_sec_to_time.cpp index 3fc94b5ef..ff808a975 100644 --- a/utils/funcexp/func_sec_to_time.cpp +++ b/utils/funcexp/func_sec_to_time.cpp @@ -120,6 +120,7 @@ string Func_sec_to_time::getStrVal(rowgroup::Row& row, break; case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { const string& valStr = parm[0]->data()->getStrVal(row, isNull); @@ -257,12 +258,15 @@ execplan::IDB_Decimal Func_sec_to_time::getDecimalVal(rowgroup::Row& row, execplan::CalpontSystemCatalog::ColType& op_ct) { IDB_Decimal d; + int64_t val = parm[0]->data()->getIntVal(row, isNull); + int64_t tmpVal; + if (val > 3020399) - d.value = 8385959; + tmpVal = 8385959; else if (val < -3020399) - d.value = 4286581337LL; + tmpVal = 4286581337LL; else { string time = getStrVal(row, parm, isNull, op_ct); @@ -277,15 +281,17 @@ execplan::IDB_Decimal Func_sec_to_time::getDecimalVal(rowgroup::Row& row, char* ep = NULL; const char* str = time.c_str(); errno = 0; - d.value = strtoll(str, &ep, 10); + tmpVal = strtoll(str, &ep, 10); } + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + d.s128Value = tmpVal; + else + d.value = tmpVal; + d.scale = 0; return d; } - - - } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_second.cpp b/utils/funcexp/func_second.cpp index 475f7495e..ff09bf320 100644 --- a/utils/funcexp/func_second.cpp +++ b/utils/funcexp/func_second.cpp @@ -66,14 +66,19 @@ int64_t Func_second::getIntVal(rowgroup::Row& row, } case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { - if (parm[0]->data()->resultType().scale) + if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); if (val == -1) isNull = true; } + else + { + isNull = true; + } break; } diff --git a/utils/funcexp/func_str_to_date.cpp b/utils/funcexp/func_str_to_date.cpp index c4b0226f2..3a4c165ac 100644 --- a/utils/funcexp/func_str_to_date.cpp +++ b/utils/funcexp/func_str_to_date.cpp @@ -145,6 +145,7 @@ dataconvert::DateTime getDateTime (rowgroup::Row& row, } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { if (parm[0]->data()->resultType().scale == 0) { @@ -159,6 +160,11 @@ dataconvert::DateTime getDateTime (rowgroup::Row& row, return -1; } } + else + { + isNull = true; + return -1; + } break; } diff --git a/utils/funcexp/func_time.cpp b/utils/funcexp/func_time.cpp index 70202ac63..89d95b021 100644 --- a/utils/funcexp/func_time.cpp +++ b/utils/funcexp/func_time.cpp @@ -70,8 +70,9 @@ string Func_time::getStrVal(rowgroup::Row& row, } case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { - if (parm[0]->data()->resultType().scale) + if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); @@ -81,6 +82,10 @@ string Func_time::getStrVal(rowgroup::Row& row, //else // return *(reinterpret_cast(&val)); } + else + { + isNull = true; + } break; } diff --git a/utils/funcexp/func_time_format.cpp b/utils/funcexp/func_time_format.cpp index 5dd2f666a..c77e46934 100644 --- a/utils/funcexp/func_time_format.cpp +++ b/utils/funcexp/func_time_format.cpp @@ -128,6 +128,7 @@ string Func_time_format::getStrVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -145,6 +146,11 @@ string Func_time_format::getStrVal(rowgroup::Row& row, msec = (uint32_t)((val & 0xfffff)); } } + else + { + isNull = true; + return ""; + } break; diff --git a/utils/funcexp/func_time_to_sec.cpp b/utils/funcexp/func_time_to_sec.cpp index e19ce2f81..b8ab898b8 100644 --- a/utils/funcexp/func_time_to_sec.cpp +++ b/utils/funcexp/func_time_to_sec.cpp @@ -153,6 +153,7 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -169,6 +170,11 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, sec = (int32_t)((val >> 20) & 0x3f); } } + else + { + isNull = true; + return -1; + } break; diff --git a/utils/funcexp/func_timediff.cpp b/utils/funcexp/func_timediff.cpp index 3c629933f..184200528 100644 --- a/utils/funcexp/func_timediff.cpp +++ b/utils/funcexp/func_timediff.cpp @@ -199,6 +199,7 @@ string Func_timediff::getStrVal(rowgroup::Row& row, break; case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale != 0) { isNull = true; @@ -285,6 +286,7 @@ string Func_timediff::getStrVal(rowgroup::Row& row, break; case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: if (parm[1]->data()->resultType().scale != 0) { isNull = true; diff --git a/utils/funcexp/func_truncate.cpp b/utils/funcexp/func_truncate.cpp index 057a1c299..02455de61 100644 --- a/utils/funcexp/func_truncate.cpp +++ b/utils/funcexp/func_truncate.cpp @@ -119,18 +119,36 @@ int64_t Func_truncate::getIntVal(Row& row, { IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); - if (x.scale > 0) + if (!op_ct.isWideDecimalType()) { - while (x.scale-- > 0) - x.value /= 10; + if (x.scale > 0) + { + while (x.scale-- > 0) + x.value /= 10; + } + else + { + while (x.scale++ < 0) + x.value *= 10; + } + + return x.value; } else { - while (x.scale++ < 0) - x.value *= 10; - } + if (x.scale > 0) + { + while (x.scale-- > 0) + x.s128Value /= 10; + } + else + { + while (x.scale++ < 0) + x.s128Value *= 10; + } - return x.value; + return datatypes::Decimal::getInt64FromWideDecimal(x.s128Value); + } } @@ -210,7 +228,12 @@ double Func_truncate::getDoubleVal(Row& row, if (isNull) return 0.0; - double d = x.value; + double d; + + if (!op_ct.isWideDecimalType()) + d = x.value; + else + d = datatypes::Decimal::getDoubleFromWideDecimal(x.s128Value); if (x.scale > 0) { @@ -265,7 +288,12 @@ long double Func_truncate::getLongDoubleVal(Row& row, if (isNull) return 0.0; - double d = x.value; + double d; + + if (!op_ct.isWideDecimalType()) + d = x.value; + else + d = datatypes::Decimal::getDoubleFromWideDecimal(x.s128Value); if (x.scale > 0) { @@ -304,55 +332,116 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { int64_t d = 0; - //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. - volatile int64_t p = 1; decimal = parm[0]->data()->getDecimalVal(row, isNull); - if (!isNull) + if (!op_ct.isWideDecimalType()) { - int64_t nvp = p; - d = parm[1]->data()->getIntVal(row, isNull); + //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. + volatile int64_t p = 1; if (!isNull) - helpers::decimalPlaceDec(d, nvp, decimal.scale); - - p = nvp; - } - - if (isNull) - break; - - int64_t x = decimal.value; - - if (d > 0) - { - x = x * p; - } - else if (d < 0) - { - if ((x >= p) || (x <= -p)) { - if (p != 0) - x = x / p; + int64_t nvp = p; + d = parm[1]->data()->getIntVal(row, isNull); + + if (!isNull) + helpers::decimalPlaceDec(d, nvp, decimal.scale); + + p = nvp; + } + + if (isNull) + break; + + int64_t x = decimal.value; + + if (d > 0) + { + x = x * p; + } + else if (d < 0) + { + if ((x >= p) || (x <= -p)) + { + if (p != 0) + x = x / p; + else + x = 0; + } else + { x = 0; + } } - else + + // negative scale is not supported by CNX yet, set d to 0. + if (decimal.scale < 0) { - x = 0; + do + x *= 10; + + while (++decimal.scale < 0); } - } - // negative scale is not supported by CNX yet, set d to 0. - if (decimal.scale < 0) + decimal.value = x; + } + else { - do - x *= 10; + //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. + volatile int128_t p = 1; - while (++decimal.scale < 0); + if (!isNull) + { + int128_t nvp = p; + d = parm[1]->data()->getIntVal(row, isNull); + + if (!isNull) + helpers::decimalPlaceDec(d, nvp, decimal.scale); + + p = nvp; + } + + if (isNull) + break; + + if (d < -datatypes::INT128MAXPRECISION) + { + decimal.s128Value = 0; + break; + } + + int128_t x = decimal.s128Value; + + if (d > 0) + { + x = x * p; + } + else if (d < 0) + { + if ((x >= p) || (x <= -p)) + { + if (p != 0) + x = x / p; + else + x = 0; + } + else + { + x = 0; + } + } + + // negative scale is not supported by CNX yet, set d to 0. + if (decimal.scale < 0) + { + do + x *= 10; + + while (++decimal.scale < 0); + } + + decimal.s128Value = x; } - - decimal.value = x; } break; @@ -650,7 +739,10 @@ string Func_truncate::getStrVal(Row& row, break; } - return dataconvert::DataConvert::decimalToString(x.value, x.scale, op_ct.colDataType); + if (!op_ct.isWideDecimalType()) + return x.toString(); + else + return x.toString(true); } diff --git a/utils/funcexp/func_unix_timestamp.cpp b/utils/funcexp/func_unix_timestamp.cpp index 29a8f483a..4c43eaebb 100644 --- a/utils/funcexp/func_unix_timestamp.cpp +++ b/utils/funcexp/func_unix_timestamp.cpp @@ -131,6 +131,7 @@ int64_t Func_unix_timestamp::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -147,6 +148,11 @@ int64_t Func_unix_timestamp::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); } } + else + { + isNull = true; + return -1; + } break; diff --git a/utils/funcexp/func_week.cpp b/utils/funcexp/func_week.cpp index af024cfe4..874e95b42 100644 --- a/utils/funcexp/func_week.cpp +++ b/utils/funcexp/func_week.cpp @@ -139,6 +139,7 @@ int64_t Func_week::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -155,6 +156,11 @@ int64_t Func_week::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); } } + else + { + isNull = true; + return -1; + } break; diff --git a/utils/funcexp/func_weekday.cpp b/utils/funcexp/func_weekday.cpp index 04ca938b4..e8b59e620 100644 --- a/utils/funcexp/func_weekday.cpp +++ b/utils/funcexp/func_weekday.cpp @@ -135,6 +135,7 @@ int64_t Func_weekday::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -151,6 +152,11 @@ int64_t Func_weekday::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); } } + else + { + isNull = true; + return -1; + } break; diff --git a/utils/funcexp/func_year.cpp b/utils/funcexp/func_year.cpp index 2c5ec41f9..68f60ac20 100644 --- a/utils/funcexp/func_year.cpp +++ b/utils/funcexp/func_year.cpp @@ -118,6 +118,7 @@ int64_t Func_year::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); diff --git a/utils/funcexp/func_yearweek.cpp b/utils/funcexp/func_yearweek.cpp index 91bb60a8c..07e111c22 100644 --- a/utils/funcexp/func_yearweek.cpp +++ b/utils/funcexp/func_yearweek.cpp @@ -142,6 +142,7 @@ int64_t Func_yearweek::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -158,6 +159,11 @@ int64_t Func_yearweek::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); } } + else + { + isNull = true; + return -1; + } break; diff --git a/utils/funcexp/funcexp.cpp b/utils/funcexp/funcexp.cpp index 79039c923..3cf0373b9 100644 --- a/utils/funcexp/funcexp.cpp +++ b/utils/funcexp/funcexp.cpp @@ -42,6 +42,8 @@ using namespace joblist; #include "../udfsdk/udfsdk.h" #endif +#include "mcs_decimal.h" + namespace funcexp { @@ -472,10 +474,30 @@ void FuncExp::evaluate(rowgroup::Row& row, std::vector& expressi { IDB_Decimal val = expression[i]->getDecimalVal(row, isNull); - if (isNull) - row.setIntField<8>(BIGINTNULL, expression[i]->outputIndex()); + if (expression[i]->resultType().colWidth + == datatypes::MAXDECIMALWIDTH) + { + if (isNull) + { + row.setBinaryField_offset( + const_cast(&datatypes::Decimal128Null), + expression[i]->resultType().colWidth, + row.getOffset(expression[i]->outputIndex())); + } + else + { + row.setBinaryField_offset(&val.s128Value, + expression[i]->resultType().colWidth, + row.getOffset(expression[i]->outputIndex())); + } + } else - row.setIntField<8>(val.value, expression[i]->outputIndex()); + { + if (isNull) + row.setIntField<8>(BIGINTNULL, expression[i]->outputIndex()); + else + row.setIntField<8>(val.value, expression[i]->outputIndex()); + } break; } diff --git a/utils/funcexp/funchelpers.h b/utils/funcexp/funchelpers.h index 8f017d544..3317a5f69 100644 --- a/utils/funcexp/funchelpers.h +++ b/utils/funcexp/funchelpers.h @@ -470,7 +470,8 @@ inline int power ( int16_t a ) return b; } -inline void decimalPlaceDec(int64_t& d, int64_t& p, int8_t& s) +template +inline void decimalPlaceDec(int64_t& d, T& p, int8_t& s) { // find new scale if D < s if (d < s) @@ -784,10 +785,6 @@ string longDoubleToString(long double ld) return buf; } -//@bug6146, remove duplicate function with incorrect impl. Use the DataConvert::decimalToString() -//string decimalToString( execplan::IDB_Decimal x, int p ) - - uint64_t dateAdd( uint64_t time, const std::string& expr, execplan::IntervalColumn::interval_type unit, bool dateType, execplan::OpType funcType ); const std::string IDB_date_format(const dataconvert::DateTime&, const std::string&); const std::string timediff(int64_t, int64_t, bool isDateTime = true); diff --git a/utils/funcexp/functor_real.h b/utils/funcexp/functor_real.h index 0f5137891..d31a55686 100644 --- a/utils/funcexp/functor_real.h +++ b/utils/funcexp/functor_real.h @@ -281,6 +281,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -318,6 +323,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; diff --git a/utils/joiner/tuplejoiner.cpp b/utils/joiner/tuplejoiner.cpp index 30ad8c3dc..7c6793bf9 100644 --- a/utils/joiner/tuplejoiner.cpp +++ b/utils/joiner/tuplejoiner.cpp @@ -20,16 +20,18 @@ #include #include #include -#ifdef _MSC_VER -#include -#else +#ifndef _MSC_VER #include +#else +#include #endif + #include "hasher.h" #include "lbidlist.h" #include "spinlock.h" #include "vlarray.h" + using namespace std; using namespace rowgroup; using namespace utils; @@ -102,18 +104,38 @@ TupleJoiner::TupleJoiner( smallKeyColumns.push_back(smallJoinColumn); largeKeyColumns.push_back(largeJoinColumn); discreteValues.reset(new bool[1]); - cpValues.reset(new vector[1]); + cpValues.reset(new vector[1]); discreteValues[0] = false; if (smallRG.isUnsigned(smallKeyColumns[0])) { - cpValues[0].push_back(numeric_limits::max()); - cpValues[0].push_back(0); + if (datatypes::isWideDecimalType( + smallRG.getColType(smallKeyColumns[0]), + smallRG.getColumnWidth(smallKeyColumns[0]))) + { + cpValues[0].push_back((int128_t) -1); + cpValues[0].push_back(0); + } + else + { + cpValues[0].push_back((int128_t) numeric_limits::max()); + cpValues[0].push_back(0); + } } else { - cpValues[0].push_back(numeric_limits::max()); - cpValues[0].push_back(numeric_limits::min()); + if (datatypes::isWideDecimalType( + smallRG.getColType(smallKeyColumns[0]), + smallRG.getColumnWidth(smallKeyColumns[0]))) + { + cpValues[0].push_back(utils::maxInt128); + cpValues[0].push_back(utils::minInt128); + } + else + { + cpValues[0].push_back((int128_t) numeric_limits::max()); + cpValues[0].push_back((int128_t) numeric_limits::min()); + } } if (smallRG.isUnsigned(smallJoinColumn) != largeRG.isUnsigned(largeJoinColumn)) @@ -195,20 +217,40 @@ TupleJoiner::TupleJoiner( storedKeyAlloc[i].setAllocSize(keyLength); discreteValues.reset(new bool[smallKeyColumns.size()]); - cpValues.reset(new vector[smallKeyColumns.size()]); + cpValues.reset(new vector[smallKeyColumns.size()]); for (i = 0; i < smallKeyColumns.size(); i++) { discreteValues[i] = false; if (isUnsigned(smallRG.getColTypes()[smallKeyColumns[i]])) { - cpValues[i].push_back(static_cast(numeric_limits::max())); - cpValues[i].push_back(0); + if (datatypes::isWideDecimalType( + smallRG.getColType(smallKeyColumns[i]), + smallRG.getColumnWidth(smallKeyColumns[i]))) + { + cpValues[i].push_back((int128_t) -1); + cpValues[i].push_back(0); + } + else + { + cpValues[i].push_back((int128_t) numeric_limits::max()); + cpValues[i].push_back(0); + } } else { - cpValues[i].push_back(numeric_limits::max()); - cpValues[i].push_back(numeric_limits::min()); + if (datatypes::isWideDecimalType( + smallRG.getColType(smallKeyColumns[i]), + smallRG.getColumnWidth(smallKeyColumns[i]))) + { + cpValues[i].push_back(utils::maxInt128); + cpValues[i].push_back(utils::minInt128); + } + else + { + cpValues[i].push_back(numeric_limits::max()); + cpValues[i].push_back(numeric_limits::min()); + } } } } @@ -678,8 +720,9 @@ void TupleJoiner::doneInserting() for (col = 0; col < smallKeyColumns.size(); col++) { - tr1::unordered_set uniquer; - tr1::unordered_set::iterator uit; + typedef std::tr1::unordered_set unordered_set_int128; + unordered_set_int128 uniquer; + unordered_set_int128::iterator uit; sthash_t::iterator sthit; hash_t::iterator hit; ldhash_t::iterator ldit; @@ -758,6 +801,12 @@ void TupleJoiner::doneInserting() } } } + else if (datatypes::isWideDecimalType( + smallRow.getColType(smallKeyColumns[col]), + smallRow.getColumnWidth(smallKeyColumns[col]))) + { + uniquer.insert(*((int128_t*)smallRow.getBinaryField(smallKeyColumns[col]))); + } else if (smallRow.isUnsigned(smallKeyColumns[col])) { uniquer.insert((int64_t)smallRow.getUintField(smallKeyColumns[col])); @@ -1070,32 +1119,33 @@ void TupleJoiner::updateCPData(const Row& r) for (col = 0; col < smallKeyColumns.size(); col++) { -// if (r.getColumnWidth(smallKeyColumns[col]) > 8) - if (r.isLongString(smallKeyColumns[col])) + auto colIdx = smallKeyColumns[col]; + if (r.isLongString(colIdx)) continue; - int64_t& min = cpValues[col][0], &max = cpValues[col][1]; + auto& min = cpValues[col][0], &max = cpValues[col][1]; - if (r.isCharType(smallKeyColumns[col])) + if (r.isCharType(colIdx)) { - int64_t val = r.getIntField(smallKeyColumns[col]); + int64_t val = r.getIntField(colIdx); - if (order_swap(val) < order_swap(min) || - min == numeric_limits::max()) + if (order_swap(val) < order_swap((int64_t) min) || + ((int64_t) min) == numeric_limits::max()) { min = val; } - if (order_swap(val) > order_swap(max) || - max == numeric_limits::min()) + if (order_swap(val) > order_swap((int64_t) max) || + ((int64_t) max) == numeric_limits::min()) { max = val; } } - else if (r.isUnsigned(smallKeyColumns[col])) + else if (r.isUnsigned(colIdx)) { - uint64_t uval; - if (r.getColType(smallKeyColumns[col]) == CalpontSystemCatalog::LONGDOUBLE) + uint128_t uval; + + if (r.getColType(colIdx) == CalpontSystemCatalog::LONGDOUBLE) { double dval = (double)roundl(r.getLongDoubleField(smallKeyColumns[col])); switch (largeRG.getColType(largeKeyColumns[col])) @@ -1114,23 +1164,30 @@ void TupleJoiner::updateCPData(const Row& r) } } } + else if (datatypes::isWideDecimalType( + r.getColType(colIdx), + r.getColumnWidth(colIdx))) + { + uval = *((int128_t*)r.getBinaryField(colIdx)); + } else { - uval = r.getUintField(smallKeyColumns[col]); + uval = r.getUintField(colIdx); } - if (uval > static_cast(max)) - max = static_cast(uval); + if (uval > static_cast(max)) + max = static_cast(uval); - if (uval < static_cast(min)) - min = static_cast(uval); + if (uval < static_cast(min)) + min = static_cast(uval); } else { - int64_t val; - if (r.getColType(smallKeyColumns[col]) == CalpontSystemCatalog::LONGDOUBLE) + int128_t val = 0; + + if (r.getColType(colIdx) == CalpontSystemCatalog::LONGDOUBLE) { - double dval = (double)roundl(r.getLongDoubleField(smallKeyColumns[col])); + double dval = (double)roundl(r.getLongDoubleField(colIdx)); switch (largeRG.getColType(largeKeyColumns[col])) { case CalpontSystemCatalog::DOUBLE: @@ -1147,9 +1204,15 @@ void TupleJoiner::updateCPData(const Row& r) } } } + else if (datatypes::isWideDecimalType( + r.getColType(colIdx), + r.getColumnWidth(colIdx))) + { + val = *((int128_t*)r.getBinaryField(colIdx)); + } else { - val = r.getIntField(smallKeyColumns[col]); + val = r.getIntField(colIdx); } if (val > max) @@ -1674,20 +1737,40 @@ boost::shared_ptr TupleJoiner::copyForDiskJoin() ret->uniqueLimit = uniqueLimit; ret->discreteValues.reset(new bool[smallKeyColumns.size()]); - ret->cpValues.reset(new vector[smallKeyColumns.size()]); + ret->cpValues.reset(new vector[smallKeyColumns.size()]); for (uint32_t i = 0; i < smallKeyColumns.size(); i++) { ret->discreteValues[i] = false; if (isUnsigned(smallRG.getColTypes()[smallKeyColumns[i]])) { - ret->cpValues[i].push_back(static_cast(numeric_limits::max())); - ret->cpValues[i].push_back(0); + if (datatypes::isWideDecimalType( + smallRG.getColType(smallKeyColumns[i]), + smallRG.getColumnWidth(smallKeyColumns[i]))) + { + ret->cpValues[i].push_back((int128_t) -1); + ret->cpValues[i].push_back(0); + } + else + { + ret->cpValues[i].push_back((int128_t) numeric_limits::max()); + ret->cpValues[i].push_back(0); + } } else { - ret->cpValues[i].push_back(numeric_limits::max()); - ret->cpValues[i].push_back(numeric_limits::min()); + if (datatypes::isWideDecimalType( + smallRG.getColType(smallKeyColumns[i]), + smallRG.getColumnWidth(smallKeyColumns[i]))) + { + ret->cpValues[i].push_back(utils::maxInt128); + ret->cpValues[i].push_back(utils::minInt128); + } + else + { + ret->cpValues[i].push_back(numeric_limits::max()); + ret->cpValues[i].push_back(numeric_limits::min()); + } } } diff --git a/utils/joiner/tuplejoiner.h b/utils/joiner/tuplejoiner.h index b0d38364d..c1be13eac 100644 --- a/utils/joiner/tuplejoiner.h +++ b/utils/joiner/tuplejoiner.h @@ -39,6 +39,7 @@ #include "stlpoolallocator.h" #include "hasher.h" #include "threadpool.h" +#include "columnwidth.h" namespace joiner { @@ -286,7 +287,7 @@ public: { return discreteValues; } - inline const boost::scoped_array >& getCPData() + inline const boost::scoped_array >& getCPData() { return cpValues; } @@ -412,7 +413,7 @@ private: /* Runtime casual partitioning support */ void updateCPData(const rowgroup::Row& r); boost::scoped_array discreteValues; - boost::scoped_array > cpValues; // if !discreteValues, [0] has min, [1] has max + boost::scoped_array > cpValues; // if !discreteValues, [0] has min, [1] has max uint32_t uniqueLimit; bool finished; diff --git a/utils/loggingcpp/exceptclasses.h b/utils/loggingcpp/exceptclasses.h index 805a2f778..3fd77dc8a 100644 --- a/utils/loggingcpp/exceptclasses.h +++ b/utils/loggingcpp/exceptclasses.h @@ -187,6 +187,16 @@ public: std::runtime_error(msg) { } }; +/** @brief Exception for F&E framework to throw on op overflow + * Invalid Operation Exception + */ +class OperationOverflowExcept : public std::runtime_error +{ +public: + /** Takes a character string describing the error. */ + OperationOverflowExcept(const std::string& msg) : + std::runtime_error(msg) { } +}; /** @brief specific error exception class for getSysData in Calpontsystemcatalog. * @bug 2574 * diff --git a/utils/messageqcpp/bytestream.cpp b/utils/messageqcpp/bytestream.cpp index 6ecbf625d..bef473b1c 100644 --- a/utils/messageqcpp/bytestream.cpp +++ b/utils/messageqcpp/bytestream.cpp @@ -37,6 +37,7 @@ using namespace boost; #define BYTESTREAM_DLLEXPORT #include "bytestream.h" #undef BYTESTREAM_DLLEXPORT +#include "datatypes/mcs_int128.h" #define DEBUG_DUMP_STRINGS_LESS_THAN 0 @@ -235,6 +236,24 @@ ByteStream& ByteStream::operator<<(const uint64_t o) return *this; } +ByteStream& ByteStream::operator<<(const uint128_t& o) +{ + if (fBuf == 0 || (fCurInPtr - fBuf + 16U > fMaxLen + ISSOverhead)) + growBuf(fMaxLen + BlockSize); + datatypes::TSInt128::storeUnaligned(fCurInPtr, o); + fCurInPtr += 16; + return *this; +} + +ByteStream& ByteStream::operator<<(const int128_t& o) +{ + if (fBuf == 0 || (fCurInPtr - fBuf + 16U > fMaxLen + ISSOverhead)) + growBuf(fMaxLen + BlockSize); + datatypes::TSInt128::storeUnaligned(fCurInPtr, o); + fCurInPtr += 16; + return *this; +} + ByteStream& ByteStream::operator<<(const string& s) { int32_t len = s.size(); @@ -319,6 +338,20 @@ ByteStream& ByteStream::operator>>(uint64_t& o) return *this; } +ByteStream& ByteStream::operator>>(uint128_t& o) +{ + peek(o); + fCurOutPtr += 16; + return *this; +} + +ByteStream& ByteStream::operator>>(int128_t& o) +{ + peek(o); + fCurOutPtr += 16; + return *this; +} + ByteStream& ByteStream::operator>>(string& s) { peek(s); @@ -399,6 +432,20 @@ void ByteStream::peek(uint64_t& o) const o = *((uint64_t*) fCurOutPtr); } +void ByteStream::peek(uint128_t& o) const +{ + if (length() < 16) + throw underflow_error("ByteStream>uint128_t: not enough data in stream to fill datatype"); + datatypes::TSInt128::assignPtrPtr(&o, fCurOutPtr); +} + +void ByteStream::peek(int128_t& o) const +{ + if (length() < 16) + throw underflow_error("ByteStream>int128_t: not enough data in stream to fill datatype"); + datatypes::TSInt128::assignPtrPtr(&o, fCurOutPtr); +} + void ByteStream::peek(string& s) const { int32_t len; diff --git a/utils/messageqcpp/bytestream.h b/utils/messageqcpp/bytestream.h index 316d9b0a1..970ff9423 100644 --- a/utils/messageqcpp/bytestream.h +++ b/utils/messageqcpp/bytestream.h @@ -33,6 +33,7 @@ #include #include +#include "mcs_basic_types.h" #include "exceptclasses.h" #include "serializeable.h" #include "any.hpp" @@ -45,6 +46,7 @@ class ByteStreamTestSuite; #define EXPORT #endif + namespace messageqcpp { @@ -74,6 +76,7 @@ public: typedef uint16_t doublebyte; typedef uint32_t quadbyte; typedef uint64_t octbyte; + typedef int128_t hexbyte; typedef boost::uuids::uuid uuid; /** @@ -143,6 +146,15 @@ public: * push an uint64_t onto the end of the stream. The byte order is whatever the native byte order is. */ EXPORT ByteStream& operator<<(const uint64_t o); + /** + * push an int128_t onto the end of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator<<(const int128_t& o); + + /** + * push an uint128_t onto the end of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator<<(const uint128_t& o); /** * push a float onto the end of the stream. The byte order is * whatever the native byte order is. @@ -207,6 +219,14 @@ public: * extract an uint64_t from the front of the stream. The byte order is whatever the native byte order is. */ EXPORT ByteStream& operator>>(uint64_t& o); + /** + * extract an int128_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator>>(int128_t& o); + /** + * extract an uint128_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator>>(uint128_t& o); /** * extract a float from the front of the stream. The byte * order is whatever the native byte order is. @@ -277,6 +297,14 @@ public: * Peek at an uint64_t from the front of the stream. The byte order is whatever the native byte order is. */ EXPORT void peek(uint64_t& o) const; + /** + * Peek at an int128_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT void peek(int128_t& o) const; + /** + * Peek at an uint128_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT void peek(uint128_t& o) const; /** * Peek at a float from the front of the stream. The byte order * is whatever the native byte order is. diff --git a/utils/regr/moda.cpp b/utils/regr/moda.cpp index caf6cd1f8..aa095ab53 100644 --- a/utils/regr/moda.cpp +++ b/utils/regr/moda.cpp @@ -21,6 +21,7 @@ #include "moda.h" #include "bytestream.h" #include "objectreader.h" +#include "columnwidth.h" using namespace mcsv1sdk; @@ -103,9 +104,12 @@ mcsv1_UDAF* moda::getImpl(mcsv1Context* context) case 4: data->modaImpl = &moda_impl_int32; break; - default: + case 8: data->modaImpl = &moda_impl_int64; break; + case 16: + data->modaImpl = &moda_impl_int128; + break; } break; case execplan::CalpontSystemCatalog::UTINYINT: @@ -154,7 +158,7 @@ mcsv1_UDAF::ReturnCode moda::init(mcsv1Context* context, return mcsv1_UDAF::ERROR; } - if (!(execplan::isNumeric(colTypes[0].dataType))) + if (!(datatypes::isNumeric(colTypes[0].dataType))) { // The error message will be prepended with // "The storage engine for the table doesn't support " @@ -179,11 +183,18 @@ mcsv1_UDAF::ReturnCode moda::init(mcsv1Context* context, { context->setColWidth(4); } - else + else if (colTypes[0].precision < 19) { context->setColWidth(8); } + else if (utils::widthByPrecision(colTypes[0].precision)) + { + context->setColWidth(16); + } + + context->setScale(colTypes[0].scale); } + context->setPrecision(colTypes[0].precision); mcsv1_UDAF* impl = getImpl(context); @@ -203,10 +214,7 @@ template mcsv1_UDAF::ReturnCode Moda_impl_T::init(mcsv1Context* context, ColumnDatum* colTypes) { - context->setScale(context->getScale()); - context->setPrecision(19); return mcsv1_UDAF::SUCCESS; - } template @@ -224,7 +232,7 @@ mcsv1_UDAF::ReturnCode Moda_impl_T::nextValue(mcsv1Context* context, ColumnDa { static_any::any& valIn = valsIn[0].columnData; ModaData* data = static_cast(context->getUserData()); - std::unordered_map* map = data->getMap(); + std::unordered_map >* map = data->getMap(); if (valIn.empty()) { @@ -261,9 +269,9 @@ mcsv1_UDAF::ReturnCode Moda_impl_T::subEvaluate(mcsv1Context* context, const ModaData* outData = static_cast(context->getUserData()); const ModaData* inData = static_cast(userDataIn); - std::unordered_map* outMap = outData->getMap(); - std::unordered_map* inMap = inData->getMap(); - typename std::unordered_map::const_iterator iter; + std::unordered_map >* outMap = outData->getMap(); + std::unordered_map >* inMap = inData->getMap(); + typename std::unordered_map >::const_iterator iter; for (iter = inMap->begin(); iter != inMap->end(); ++iter) { @@ -283,7 +291,7 @@ mcsv1_UDAF::ReturnCode Moda_impl_T::evaluate(mcsv1Context* context, static_an long double avg = 0; T val = 0; ModaData* data = static_cast(context->getUserData()); - std::unordered_map* map = data->getMap(); + std::unordered_map >* map = data->getMap(); if (map->size() == 0) { @@ -292,7 +300,7 @@ mcsv1_UDAF::ReturnCode Moda_impl_T::evaluate(mcsv1Context* context, static_an } avg = data->fCount ? data->fSum / data->fCount : 0; - typename std::unordered_map::iterator iter; + typename std::unordered_map >::iterator iter; for (iter = map->begin(); iter != map->end(); ++iter) { @@ -303,11 +311,13 @@ mcsv1_UDAF::ReturnCode Moda_impl_T::evaluate(mcsv1Context* context, static_an } else if (iter->second == maxCnt) { + T absval = val >= 0 ? val : -val; + T absfirst = iter->first >= 0 ? iter->first : -iter->first; // Tie breaker: choose the closest to avg. If still tie, choose smallest long double dist1 = val > avg ? (long double)val-avg : avg-(long double)val; long double dist2 = iter->first > avg ? (long double)iter->first-avg : avg-(long double)iter->first; if ((dist1 > dist2) - || ((dist1 == dist2) && (std::fabs(val) > std::fabs(iter->first)))) + || ((dist1 == dist2) && (absval > absfirst))) { val = iter->first; } @@ -328,7 +338,7 @@ mcsv1_UDAF::ReturnCode Moda_impl_T::dropValue(mcsv1Context* context, ColumnDa { static_any::any& valDropped = valsDropped[0].columnData; ModaData* data = static_cast(context->getUserData()); - std::unordered_map* map = data->getMap(); + std::unordered_map >* map = data->getMap(); if (valDropped.empty()) { @@ -379,9 +389,12 @@ void ModaData::serialize(messageqcpp::ByteStream& bs) const case 4: serializeMap(bs); break; - default: + case 8: serializeMap(bs); break; + case 16: + serializeMap(bs); + break; } break; case execplan::CalpontSystemCatalog::UTINYINT: @@ -447,9 +460,12 @@ void ModaData::unserialize(messageqcpp::ByteStream& bs) case 4: unserializeMap(bs); break; - default: + case 8: unserializeMap(bs); break; + case 16: + unserializeMap(bs); + break; } break; case execplan::CalpontSystemCatalog::UTINYINT: @@ -519,10 +535,14 @@ void ModaData::cleanup() clear(); deleteMap(); break; - default: + case 8: clear(); deleteMap(); break; + case 16: + clear(); + deleteMap(); + break; } break; case execplan::CalpontSystemCatalog::UTINYINT: diff --git a/utils/regr/moda.h b/utils/regr/moda.h index f8c0ef822..ae47f8462 100644 --- a/utils/regr/moda.h +++ b/utils/regr/moda.h @@ -45,6 +45,7 @@ #include "mcsv1_udaf.h" #include "calpontsystemcatalog.h" #include "windowfunctioncolumn.h" +#include "hasher.h" #if defined(_MSC_VER) && defined(xxxRGNODE_DLLEXPORT) #define EXPORT __declspec(dllexport) @@ -54,6 +55,38 @@ namespace mcsv1sdk { +// A hasher that handles int128_t +template +struct hasher +{ + inline size_t operator()(T val) const + { + return fHasher((char*) &val, sizeof(T)); + } + +private: + utils::Hasher fHasher; +}; + +template<> +struct hasher +{ + inline size_t operator()(long double val) const + { + if (sizeof(long double) == 8) // Probably just MSC, but you never know. + { + return fHasher((char*) &val, sizeof(long double)); + } + else + { + // For Linux x86_64, long double is stored in 128 bits, but only 80 are significant + return fHasher((char*) &val, 10); + } + } +private: + utils::Hasher fHasher; +}; + // Override UserData for data storage struct ModaData : public UserData { @@ -69,22 +102,22 @@ struct ModaData : public UserData virtual void unserialize(messageqcpp::ByteStream& bs); template - std::unordered_map* getMap() + std::unordered_map >* getMap() { if (!fMap) { // Just in time creation - fMap = new std::unordered_map; + fMap = new std::unordered_map >; } - return (std::unordered_map*) fMap; + return (std::unordered_map >*) fMap; } // The const version is only called by serialize() // It shouldn't (and can't) create a new map. template - std::unordered_map* getMap() const + std::unordered_map >* getMap() const { - return (std::unordered_map*) fMap; + return (std::unordered_map >*) fMap; } template @@ -92,7 +125,7 @@ struct ModaData : public UserData { if (fMap) { - delete (std::unordered_map*) fMap; + delete (std::unordered_map >*) fMap; fMap = NULL; } } @@ -123,10 +156,10 @@ private: template void serializeMap(messageqcpp::ByteStream& bs) const { - std::unordered_map* map = getMap(); + std::unordered_map >* map = getMap(); if (map) { - typename std::unordered_map::const_iterator iter; + typename std::unordered_map >::const_iterator iter; bs << (uint64_t)map->size(); for (iter = map->begin(); iter != map->end(); ++iter) { @@ -147,7 +180,7 @@ private: T num; uint64_t sz; bs >> sz; - std::unordered_map* map = getMap(); + std::unordered_map >* map = getMap(); map->clear(); for (uint64_t i = 0; i < sz; ++i) { @@ -234,6 +267,7 @@ protected: Moda_impl_T moda_impl_int16; Moda_impl_T moda_impl_int32; Moda_impl_T moda_impl_int64; + Moda_impl_T moda_impl_int128; Moda_impl_T moda_impl_uint8; Moda_impl_T moda_impl_uint16; Moda_impl_T moda_impl_uint32; diff --git a/utils/rowgroup/CMakeLists.txt b/utils/rowgroup/CMakeLists.txt index edda8e943..876d620cf 100644 --- a/utils/rowgroup/CMakeLists.txt +++ b/utils/rowgroup/CMakeLists.txt @@ -15,4 +15,3 @@ add_dependencies(rowgroup loggingcpp) target_link_libraries(rowgroup ${NETSNMP_LIBRARIES} funcexp) install(TARGETS rowgroup DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) - diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index c73e751af..05c30f6cb 100755 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -1,6 +1,6 @@ /* Copyright (C) 2014 InfiniDB, Inc. - Copyright (c) 2019 MariaDB Corporation + Copyright (c) 2019-2020 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -56,6 +56,7 @@ //..comment out NDEBUG to enable assertions, uncomment NDEBUG to disable //#define NDEBUG +#include "mcs_decimal.h" using namespace std; using namespace boost; @@ -69,12 +70,16 @@ namespace const int64_t AGG_ROWGROUP_SIZE = 256; template -bool minMax(T d1, T d2, int type) +inline bool minMax(T d1, T d2, int type) { if (type == rowgroup::ROWAGG_MIN) return d1 < d2; else return d1 > d2; } +inline bool minMax(int128_t* d1, int128_t* d2, int type) +{ + return (type == rowgroup::ROWAGG_MIN) ? *d1 < *d2 : *d1 > *d2; +} inline int64_t getIntNullValue(int colType) { @@ -224,7 +229,6 @@ inline string getStringNullValue() } - namespace rowgroup { const std::string typeStr(""); @@ -234,6 +238,7 @@ const static_any::any& RowAggregation::shortTypeId((short)1); const static_any::any& RowAggregation::intTypeId((int)1); const static_any::any& RowAggregation::longTypeId((long)1); const static_any::any& RowAggregation::llTypeId((long long)1); +const static_any::any& RowAggregation::int128TypeId((int128_t)1); const static_any::any& RowAggregation::ucharTypeId((unsigned char)1); const static_any::any& RowAggregation::ushortTypeId((unsigned short)1); const static_any::any& RowAggregation::uintTypeId((unsigned int)1); @@ -244,6 +249,8 @@ const static_any::any& RowAggregation::doubleTypeId((double)1); const static_any::any& RowAggregation::longdoubleTypeId((long double)1); const static_any::any& RowAggregation::strTypeId(typeStr); +using Dec = datatypes::Decimal; + KeyStorage::KeyStorage(const RowGroup& keys, Row** tRow) : tmpRow(tRow), rg(keys) { RGData data(rg); @@ -334,6 +341,16 @@ inline bool ExternalKeyEq::operator()(const RowPosition& pos1, const RowPosition static const string overflowMsg("Aggregation overflow."); +inline void RowAggregation::updateIntMinMax(int128_t* val1, int128_t* val2, int64_t col, int func) +{ + int32_t colOutOffset = fRow.getOffset(col); + if (isNull(fRowGroupOut, fRow, col)) + fRow.setBinaryField_offset(val1, sizeof(int128_t), colOutOffset); + else if (minMax(val1, val2, func)) + fRow.setBinaryField_offset(val1, sizeof(int128_t), colOutOffset); +} + + inline void RowAggregation::updateIntMinMax(int64_t val1, int64_t val2, int64_t col, int func) { if (isNull(fRowGroupOut, fRow, col)) @@ -413,6 +430,7 @@ void RowAggregation::updateStringMinMax(string val1, string val2, int64_t col, i inline bool RowAggregation::isNull(const RowGroup* pRowGroup, const Row& row, int64_t col) { /* TODO: Can we replace all of this with a call to row.isNullValue(col)? */ + // WIP MCOL-641 Yes. We can bool ret = false; int colDataType = (pRowGroup->getColTypes())[col]; @@ -463,7 +481,7 @@ inline bool RowAggregation::isNull(const RowGroup* pRowGroup, const Row& row, in else { //@bug 1821 - ret = (row.equals("", col) || row.equals(joblist::CPNULLSTRMARK, col)); + ret = (row.equals(string(""), col) || row.equals(joblist::CPNULLSTRMARK, col)); } break; @@ -536,18 +554,7 @@ inline bool RowAggregation::isNull(const RowGroup* pRowGroup, const Row& row, in case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - int colWidth = pRowGroup->getColumnWidth(col); - int64_t val = row.getIntField(col); - - if (colWidth == 1) - ret = ((uint8_t)val == joblist::TINYINTNULL); - else if (colWidth == 2) - ret = ((uint16_t)val == joblist::SMALLINTNULL); - else if (colWidth == 4) - ret = ((uint32_t)val == joblist::INTNULL); - else - ret = ((uint64_t)val == joblist::BIGINTNULL); - + ret = row.isNullValue(col); break; } @@ -572,7 +579,7 @@ inline bool RowAggregation::isNull(const RowGroup* pRowGroup, const Row& row, in case execplan::CalpontSystemCatalog::VARBINARY: case execplan::CalpontSystemCatalog::BLOB: { - ret = (row.equals("", col) || row.equals(joblist::CPNULLSTRMARK, col)); + ret = (row.equals(string(""), col) || row.equals(joblist::CPNULLSTRMARK, col)); break; } @@ -614,9 +621,6 @@ RowAggregation::RowAggregation(const RowAggregation& rhs): fSmallSideRGs(NULL), fLargeSideRG(NULL), fSmallSideCount(0), fRGContext(rhs.fRGContext), fOrigFunctionCols(NULL) { - //fGroupByCols.clear(); - //fFunctionCols.clear(); - fGroupByCols.assign(rhs.fGroupByCols.begin(), rhs.fGroupByCols.end()); fFunctionCols.assign(rhs.fFunctionCols.begin(), rhs.fFunctionCols.end()); } @@ -716,31 +720,32 @@ void RowAggregation::setJoinRowGroups(vector* pSmallSideRG, RowGroup* // threads on the PM and by multple threads on the UM. It must remain // thread safe. //------------------------------------------------------------------------------ -void RowAggregation::resetUDAF(RowUDAFFunctionCol* rowUDAF) +void RowAggregation::resetUDAF(RowUDAFFunctionCol* rowUDAF, uint64_t funcColsIdx) { // RowAggregation and it's functions need to be re-entrant which means // each instance (thread) needs its own copy of the context object. // Note: operator=() doesn't copy userData. - fRGContext = rowUDAF->fUDAFContext; + fRGContextColl[funcColsIdx] = rowUDAF->fUDAFContext; // Call the user reset for the group userData. Since, at this point, // context's userData will be NULL, reset will generate a new one. mcsv1sdk::mcsv1_UDAF::ReturnCode rc; - rc = fRGContext.getFunction()->reset(&fRGContext); + rc = fRGContextColl[funcColsIdx].getFunction()->reset(&fRGContextColl[funcColsIdx]); if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) { rowUDAF->bInterrupted = true; - throw logging::QueryDataExcept(fRGContext.getErrorMessage(), logging::aggregateFuncErr); + throw logging::QueryDataExcept(fRGContextColl[funcColsIdx].getErrorMessage(), + logging::aggregateFuncErr); } fRow.setUserDataStore(fRowGroupOut->getRGData()->getUserDataStore()); - fRow.setUserData(fRGContext, - fRGContext.getUserDataSP(), - fRGContext.getUserDataSize(), + fRow.setUserData(fRGContextColl[funcColsIdx], + fRGContextColl[funcColsIdx].getUserDataSP(), + fRGContextColl[funcColsIdx].getUserDataSize(), rowUDAF->fAuxColumnIndex); - - fRGContext.setUserData(NULL); // Prevents calling deleteUserData on the fRGContext. + // Prevents calling deleteUserData on the mcsv1Context. + fRGContextColl[funcColsIdx].setUserData(NULL); } //------------------------------------------------------------------------------ @@ -767,6 +772,9 @@ void RowAggregation::initialize() // save the original output rowgroup data as primary row data fPrimaryRowData = fRowGroupOut->getRGData(); + // Lazy approach w/o a mapping b/w fFunctionCols idx and fRGContextColl idx + fRGContextColl.resize(fFunctionCols.size()); + // Need map only if groupby list is not empty. if (!fGroupByCols.empty()) { @@ -779,13 +787,13 @@ void RowAggregation::initialize() { fRowGroupOut->setRowCount(1); attachGroupConcatAg(); - // For UDAF, reset the data for (uint64_t i = 0; i < fFunctionCols.size(); i++) { if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) { - resetUDAF(dynamic_cast(fFunctionCols[i].get())); + auto rowUDAFColumnPtr = dynamic_cast(fFunctionCols[i].get()); + resetUDAF(rowUDAFColumnPtr, i); } } } @@ -837,7 +845,8 @@ void RowAggregation::aggReset() { if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) { - resetUDAF(dynamic_cast(fFunctionCols[i].get())); + auto rowUDAFColumnPtr = dynamic_cast(fFunctionCols[i].get()); + resetUDAF(rowUDAFColumnPtr, i); } } } @@ -859,7 +868,8 @@ void RowAggregationUM::aggReset() } -void RowAggregationUM::aggregateRowWithRemap(Row& row) +void RowAggregationUM::aggregateRowWithRemap(Row& row, + std::vector* rgContextColl) { pair inserted; RowPosition pos(RowPosition::MSB, 0); @@ -892,7 +902,8 @@ void RowAggregationUM::aggregateRowWithRemap(Row& row) { if ((*fOrigFunctionCols)[i]->fAggFunction == ROWAGG_UDAF) { - resetUDAF(dynamic_cast((*fOrigFunctionCols)[i].get())); + auto rowUDAFColumnPtr = dynamic_cast((*fOrigFunctionCols)[i].get()); + resetUDAF(rowUDAFColumnPtr, i); } } } @@ -902,7 +913,8 @@ void RowAggregationUM::aggregateRowWithRemap(Row& row) { if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) { - resetUDAF(dynamic_cast(fFunctionCols[i].get())); + auto rowUDAFColumnPtr = dynamic_cast(fFunctionCols[i].get()); + resetUDAF(rowUDAFColumnPtr, i); } } } @@ -915,20 +927,22 @@ void RowAggregationUM::aggregateRowWithRemap(Row& row) fResultDataVec[pos.group]->getRow(pos.row, &fRow); } - updateEntry(row); + updateEntry(row, rgContextColl); } -void RowAggregationUM::aggregateRow(Row& row) +void RowAggregationUM::aggregateRow(Row& row, + std::vector* rgContextColl) { if (UNLIKELY(fKeyOnHeap)) - aggregateRowWithRemap(row); + aggregateRowWithRemap(row, rgContextColl); else - RowAggregation::aggregateRow(row); + RowAggregation::aggregateRow(row, rgContextColl); } -void RowAggregation::aggregateRow(Row& row) +void RowAggregation::aggregateRow(Row& row, + std::vector* rgContextColl) { // groupby column list is not empty, find the entry. if (!fGroupByCols.empty()) @@ -967,7 +981,8 @@ void RowAggregation::aggregateRow(Row& row) { if ((*fOrigFunctionCols)[i]->fAggFunction == ROWAGG_UDAF) { - resetUDAF(dynamic_cast((*fOrigFunctionCols)[i].get())); + auto rowUDAFColumnPtr = dynamic_cast((*fOrigFunctionCols)[i].get()); + resetUDAF(rowUDAFColumnPtr, i); } } } @@ -977,7 +992,8 @@ void RowAggregation::aggregateRow(Row& row) { if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) { - resetUDAF(dynamic_cast(fFunctionCols[i].get())); + auto rowUDAFColumnPtr = dynamic_cast(fFunctionCols[i].get()); + resetUDAF(rowUDAFColumnPtr, i); } } } @@ -990,7 +1006,7 @@ void RowAggregation::aggregateRow(Row& row) } } - updateEntry(row); + updateEntry(row, rgContextColl); } @@ -1020,13 +1036,36 @@ void RowAggregation::initMapData(const Row& rowIn) case execplan::CalpontSystemCatalog::MEDINT: case execplan::CalpontSystemCatalog::INT: case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: { fRow.setIntField(rowIn.getIntField(colIn), colOut); break; } + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (LIKELY(rowIn.getColumnWidth(colIn) == datatypes::MAXDECIMALWIDTH)) + { + uint32_t colOutOffset = fRow.getOffset(colOut); + fRow.setBinaryField_offset( + rowIn.getBinaryField(colIn), + sizeof(int128_t), + colOutOffset); + } + else if (rowIn.getColumnWidth(colIn) <= datatypes::MAXLEGACYWIDTH) + { + fRow.setIntField(rowIn.getIntField(colIn), colOut); + } + else + { + idbassert(0); + throw std::logic_error("RowAggregation::initMapData(): DECIMAL bad length."); + } + + break; + } + + case execplan::CalpontSystemCatalog::UTINYINT: case execplan::CalpontSystemCatalog::USMALLINT: case execplan::CalpontSystemCatalog::UMEDINT: @@ -1123,8 +1162,6 @@ void RowAggregation::makeAggFieldsNull(Row& row) fFunctionCols[i]->fAggFunction == ROWAGG_GROUP_CONCAT || fFunctionCols[i]->fAggFunction == ROWAGG_STATS) { -// done by memset -// row.setIntField(0, colOut); continue; } @@ -1170,7 +1207,23 @@ void RowAggregation::makeAggFieldsNull(Row& row) case execplan::CalpontSystemCatalog::UDECIMAL: { int colWidth = fRowGroupOut->getColumnWidth(colOut); - row.setIntField(getUintNullValue(colDataType, colWidth), colOut); + if (LIKELY(colWidth == datatypes::MAXDECIMALWIDTH)) + { + uint32_t offset = row.getOffset(colOut); + row.setBinaryField_offset( + const_cast(&datatypes::Decimal128Null), + colWidth, + offset); + } + else if (colWidth <= datatypes::MAXLEGACYWIDTH) + { + row.setIntField(getUintNullValue(colDataType, colWidth), colOut); + } + else + { + idbassert(0); + throw std::logic_error("RowAggregation::makeAggFieldsNull(): DECIMAL bad length."); + } break; } @@ -1182,7 +1235,7 @@ void RowAggregation::makeAggFieldsNull(Row& row) { int colWidth = fRowGroupOut->getColumnWidth(colOut); - if (colWidth <= 8) + if (colWidth <= datatypes::MAXLEGACYWIDTH) { row.setUintField(getUintNullValue(colDataType, colWidth), colOut); } @@ -1255,8 +1308,6 @@ void RowAggregation::doMinMax(const Row& rowIn, int64_t colIn, int64_t colOut, i case execplan::CalpontSystemCatalog::MEDINT: case execplan::CalpontSystemCatalog::INT: case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: { int64_t valIn = rowIn.getIntField(colIn); int64_t valOut = fRow.getIntField(colOut); @@ -1264,6 +1315,30 @@ void RowAggregation::doMinMax(const Row& rowIn, int64_t colIn, int64_t colOut, i break; } + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (LIKELY(rowIn.getColumnWidth(colIn) == datatypes::MAXDECIMALWIDTH)) + { + updateIntMinMax(rowIn.getBinaryField(colIn), + fRow.getBinaryField(colOut), + colOut, funcType); + } + else if (rowIn.getColumnWidth(colIn) <= datatypes::MAXLEGACYWIDTH) + { + int64_t valIn = rowIn.getIntField(colIn); + int64_t valOut = fRow.getIntField(colOut); + updateIntMinMax(valIn, valOut, colOut, funcType); + } + else + { + idbassert(0); + throw std::logic_error("RowAggregation::doMinMax(): DECIMAL bad length."); + } + + break; + } + case execplan::CalpontSystemCatalog::UTINYINT: case execplan::CalpontSystemCatalog::USMALLINT: case execplan::CalpontSystemCatalog::UMEDINT: @@ -1343,7 +1418,9 @@ void RowAggregation::doSum(const Row& rowIn, int64_t colIn, int64_t colOut, int { int colDataType = (fRowGroupIn.getColTypes())[colIn]; long double valIn = 0; - long double valOut = fRow.getLongDoubleField(colOut); + bool isWideDataType = false; + void *wideValInPtr = nullptr; + if (isNull(&fRowGroupIn, rowIn, colIn) == true) return; @@ -1372,12 +1449,28 @@ void RowAggregation::doSum(const Row& rowIn, int64_t colIn, int64_t colOut, int case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - valIn = rowIn.getIntField(colIn); - double scale = (double)(fRowGroupIn.getScale())[colIn]; - if (valIn != 0 && scale > 0) + uint32_t width = fRowGroupIn.getColumnWidth(colIn); + isWideDataType = width == datatypes::MAXDECIMALWIDTH; + if(LIKELY(isWideDataType)) { - valIn /= pow(10.0, scale); + int128_t *dec = rowIn.getBinaryField(colIn); + wideValInPtr = reinterpret_cast(dec); } + else if (width <= datatypes::MAXLEGACYWIDTH) + { + valIn = rowIn.getIntField(colIn); + double scale = (double)(fRowGroupIn.getScale())[colIn]; + if (valIn != 0 && scale > 0) + { + valIn /= pow(10.0, scale); + } + } + else + { + idbassert(0); + throw std::logic_error("RowAggregation::doSum(): DECIMAL bad length."); + } + break; } @@ -1428,14 +1521,33 @@ void RowAggregation::doSum(const Row& rowIn, int64_t colIn, int64_t colOut, int break; } } - if (isNull(fRowGroupOut, fRow, colOut)) + if (LIKELY(!isWideDataType)) { - fRow.setLongDoubleField(valIn, colOut); + if (LIKELY(!isNull(fRowGroupOut, fRow, colOut))) + { + long double valOut = fRow.getLongDoubleField(colOut); + fRow.setLongDoubleField(valIn+valOut, colOut); + } + else + { + fRow.setLongDoubleField(valIn, colOut); + } } else { - fRow.setLongDoubleField(valIn+valOut, colOut); - } + uint32_t offset = fRow.getOffset(colOut); + int128_t* dec = reinterpret_cast(wideValInPtr); + if (LIKELY(!isNull(fRowGroupOut, fRow, colOut))) + { + int128_t *valOutPtr = fRow.getBinaryField(colOut); + int128_t sum = *valOutPtr + *dec; + fRow.setBinaryField_offset(&sum, sizeof(sum), offset); + } + else + { + fRow.setBinaryField_offset(dec, sizeof(*dec), offset); + } + } // end-of isWideDataType block } //------------------------------------------------------------------------------ @@ -1696,8 +1808,10 @@ void RowAggregation::deserialize(messageqcpp::ByteStream& bs) // NULL values are recognized and ignored for all agg functions except for // COUNT(*), which counts all rows regardless of value. // rowIn(in) - Row to be included in aggregation. +// rgContextColl(in) - ptr to a vector of UDAF contexts //------------------------------------------------------------------------------ -void RowAggregation::updateEntry(const Row& rowIn) +void RowAggregation::updateEntry(const Row& rowIn, + std::vector* rgContextColl) { for (uint64_t i = 0; i < fFunctionCols.size(); i++) { @@ -1727,7 +1841,7 @@ void RowAggregation::updateEntry(const Row& rowIn) case ROWAGG_AVG: // count(column) for average is inserted after the sum, - // colOut+1 is the position of the count column. + // colOut+1 is the position of the aux count column. doAvg(rowIn, colIn, colOut, colOut + 1); break; @@ -1754,7 +1868,7 @@ void RowAggregation::updateEntry(const Row& rowIn) case ROWAGG_UDAF: { - doUDAF(rowIn, colIn, colOut, colOut + 1, i); + doUDAF(rowIn, colIn, colOut, colOut + 1, i, rgContextColl); break; } @@ -1787,6 +1901,8 @@ void RowAggregation::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, int6 int colDataType = (fRowGroupIn.getColTypes())[colIn]; long double valIn = 0; long double valOut = fRow.getLongDoubleField(colOut); + bool isWideDataType = false; + void *wideValInPtr = nullptr; switch (colDataType) { @@ -1798,7 +1914,6 @@ void RowAggregation::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, int6 { valIn = rowIn.getIntField(colIn); break; - break; } case execplan::CalpontSystemCatalog::UTINYINT: @@ -1814,12 +1929,28 @@ void RowAggregation::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, int6 case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - valIn = rowIn.getIntField(colIn); - double scale = (double)(fRowGroupIn.getScale())[colIn]; - if (valIn != 0 && scale > 0) + uint32_t width = fRowGroupIn.getColumnWidth(colIn); + isWideDataType = width == datatypes::MAXDECIMALWIDTH; + if(LIKELY(isWideDataType)) { - valIn /= pow(10.0, scale); + int128_t* dec = rowIn.getBinaryField(colIn); + wideValInPtr = reinterpret_cast(dec); } + else if (width <= datatypes::MAXLEGACYWIDTH) + { + valIn = rowIn.getIntField(colIn); + double scale = (double)(fRowGroupIn.getScale())[colIn]; + if (valIn != 0 && scale > 0) + { + valIn /= pow(10.0, scale); + } + } + else + { + idbassert(0); + throw std::logic_error("RowAggregation::doAvg(): DECIMAL bad length."); + } + break; } @@ -1853,16 +1984,32 @@ void RowAggregation::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, int6 } } - if (fRow.getUintField(colAux) == 0) + // min(count) = 0 + uint64_t count = fRow.getUintField(colAux) + 1; + fRow.setUintField<8>(count, colAux); + bool notFirstValue = count > 1; + + if (LIKELY(!isWideDataType)) { - // This is the first value - fRow.setLongDoubleField(valIn, colOut); - fRow.setUintField(1, colAux); + if (LIKELY(notFirstValue)) + fRow.setLongDoubleField(valIn + valOut, colOut); + else // This is the first value + fRow.setLongDoubleField(valIn, colOut); } else { - fRow.setLongDoubleField(valIn + valOut, colOut); - fRow.setUintField(fRow.getUintField(colAux) + 1, colAux); + uint32_t offset = fRow.getOffset(colOut); + int128_t* dec = reinterpret_cast(wideValInPtr); + if (LIKELY(notFirstValue)) + { + int128_t *valOutPtr = fRow.getBinaryField(colOut); + int128_t sum = *valOutPtr + *dec; + fRow.setBinaryField_offset(&sum, sizeof(sum), offset); + } + else + { + fRow.setBinaryField_offset(dec, sizeof(*dec), offset); + } } } @@ -1891,9 +2038,24 @@ void RowAggregation::doStatistics(const Row& rowIn, int64_t colIn, int64_t colOu case execplan::CalpontSystemCatalog::MEDINT: case execplan::CalpontSystemCatalog::INT: case execplan::CalpontSystemCatalog::BIGINT: + valIn = (long double) rowIn.getIntField(colIn); + break; + case execplan::CalpontSystemCatalog::DECIMAL: // handle scale later case execplan::CalpontSystemCatalog::UDECIMAL: // handle scale later - valIn = (long double) rowIn.getIntField(colIn); + if (LIKELY(fRowGroupIn.getColumnWidth(colIn) == datatypes::MAXDECIMALWIDTH)) + { + int128_t* val128InPtr = rowIn.getBinaryField(colIn); + valIn = Dec::getLongDoubleFromWideDecimal(*val128InPtr); + } + else if (fRowGroupIn.getColumnWidth(colIn) <= datatypes::MAXLEGACYWIDTH) + { + valIn = (long double) rowIn.getIntField(colIn); + } + else + { + idbassert(false); + } break; case execplan::CalpontSystemCatalog::UTINYINT: @@ -1931,10 +2093,24 @@ void RowAggregation::doStatistics(const Row& rowIn, int64_t colIn, int64_t colOu fRow.setLongDoubleField(fRow.getLongDoubleField(colAux + 1) + valIn * valIn, colAux + 1); } -void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, - int64_t colAux, uint64_t& funcColsIdx) +void RowAggregation::doUDAF(const Row& rowIn, + int64_t colIn, + int64_t colOut, + int64_t colAux, + uint64_t& funcColsIdx, + std::vector* rgContextColl) { - uint32_t paramCount = fRGContext.getParameterCount(); + std::vector* udafContextsCollPtr = &fRGContextColl; + if (UNLIKELY(rgContextColl != nullptr)) + { + udafContextsCollPtr = rgContextColl; + } + + std::vector& udafContextsColl = *udafContextsCollPtr; + uint32_t paramCount = udafContextsColl[funcColsIdx].getParameterCount(); + // doUDAF changes funcColsIdx to skip UDAF arguments so the real UDAF + // column idx is the initial value of the funcColsIdx + uint64_t origFuncColsIdx = funcColsIdx; // The vector of parameters to be sent to the UDAF utils::VLArray valsIn(paramCount); utils::VLArray dataFlags(paramCount); @@ -1960,7 +2136,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, if ((cc && cc->type() == execplan::ConstantColumn::NULLDATA) || (!cc && isNull(&fRowGroupIn, rowIn, colIn) == true)) { - if (fRGContext.getRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS)) + if (udafContextsColl[origFuncColsIdx].getRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS)) { // When Ignore nulls, if there are multiple parameters and any // one of them is NULL, we ignore the entry. We need to increment @@ -2005,7 +2181,6 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, datum.scale = fRowGroupIn.getScale()[colIn]; datum.precision = fRowGroupIn.getPrecision()[colIn]; } - break; } @@ -2022,7 +2197,22 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, } else { - datum.columnData = rowIn.getIntField(colIn); + if (LIKELY(fRowGroupIn.getColumnWidth(colIn) + == datatypes::MAXDECIMALWIDTH)) + { + // We can't control boost::any asignment + // so let's get an aligned memory + datatypes::TSInt128 val = rowIn.getTSInt128Field(colIn); + datum.columnData = val.s128Value; + } + else if (fRowGroupIn.getColumnWidth(colIn) <= datatypes::MAXLEGACYWIDTH) + { + datum.columnData = rowIn.getIntField(colIn); + } + else + { + idbassert(false); + } datum.scale = fRowGroupIn.getScale()[colIn]; datum.precision = fRowGroupIn.getPrecision()[colIn]; } @@ -2188,7 +2378,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, default: { std::ostringstream errmsg; - errmsg << "RowAggregation " << fRGContext.getName() << + errmsg << "RowAggregation " << udafContextsColl[origFuncColsIdx].getName() << ": No logic for data type: " << colDataType; throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); break; @@ -2214,18 +2404,20 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, } // The intermediate values are stored in userData referenced by colAux. - fRGContext.setDataFlags(dataFlags); - fRGContext.setUserData(fRow.getUserData(colAux)); + udafContextsColl[origFuncColsIdx].setDataFlags(dataFlags); + udafContextsColl[origFuncColsIdx].setUserData(fRow.getUserData(colAux)); mcsv1sdk::mcsv1_UDAF::ReturnCode rc; - rc = fRGContext.getFunction()->nextValue(&fRGContext, valsIn); - fRGContext.setUserData(NULL); + rc = udafContextsColl[origFuncColsIdx].getFunction()->nextValue(&udafContextsColl[origFuncColsIdx], + valsIn); + udafContextsColl[origFuncColsIdx].setUserData(NULL); if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) { - RowUDAFFunctionCol* rowUDAF = dynamic_cast(fFunctionCols[funcColsIdx].get()); + RowUDAFFunctionCol* rowUDAF = dynamic_cast(fFunctionCols[origFuncColsIdx].get()); rowUDAF->bInterrupted = true; - throw logging::QueryDataExcept(fRGContext.getErrorMessage(), logging::aggregateFuncErr); + throw logging::QueryDataExcept(udafContextsColl[origFuncColsIdx].getErrorMessage(), + logging::aggregateFuncErr); } } @@ -2455,8 +2647,10 @@ void RowAggregationUM::attachGroupConcatAg() // Update the aggregation totals in the internal hashmap for the specified row. // NULL values are recognized and ignored for all agg functions except for count // rowIn(in) - Row to be included in aggregation. +// rgContextColl(in) - ptr to a vector of UDAF contexts //------------------------------------------------------------------------------ -void RowAggregationUM::updateEntry(const Row& rowIn) +void RowAggregationUM::updateEntry(const Row& rowIn, + std::vector* rgContextColl) { for (uint64_t i = 0; i < fFunctionCols.size(); i++) { @@ -2524,7 +2718,7 @@ void RowAggregationUM::updateEntry(const Row& rowIn) case ROWAGG_UDAF: { - doUDAF(rowIn, colIn, colOut, colAux, i); + doUDAF(rowIn, colIn, colOut, colAux, i, rgContextColl); break; } @@ -2568,11 +2762,6 @@ void RowAggregationUM::calculateAvgColumns() int64_t colOut = fFunctionCols[i]->fOutputColumnIndex; int64_t colAux = fFunctionCols[i]->fAuxColumnIndex; -// int scale = fRowGroupOut->getScale()[colOut]; -// int scale1 = scale >> 8; -// int scale2 = scale & 0x000000FF; -// long double factor = pow(10.0, scale2 - scale1); - for (uint64_t j = 0; j < fRowGroupOut->getRowCount(); j++) { fRowGroupOut->getRow(j, &fRow); @@ -2581,14 +2770,38 @@ void RowAggregationUM::calculateAvgColumns() if (cnt == 0) // empty set, value is initialized to null. continue; - long double sum = 0.0; - long double avg = 0.0; + uint32_t precision = fRow.getPrecision(colOut); + bool isWideDecimal = + datatypes::Decimal::isWideDecimalTypeByPrecision(precision); - // MCOL-1822 Always long double - sum = fRow.getLongDoubleField(colOut); - avg = sum / cnt; -// avg *= factor; - fRow.setLongDoubleField(avg, colOut); + if (!isWideDecimal) + { + long double sum = 0.0; + long double avg = 0.0; + sum = fRow.getLongDoubleField(colOut); + avg = sum / cnt; + fRow.setLongDoubleField(avg, colOut); + } + else + { + uint32_t offset = fRow.getOffset(colOut); + uint32_t scale = fRow.getScale(colOut); + // Get multiplied to deliver AVG with the scale closest + // to the expected original scale + 4. + // There is a counterpart in buildAggregateColumn. + datatypes::Decimal::setScalePrecision4Avg(precision, scale); + int128_t* sumPnt = fRow.getBinaryField_offset(offset); + uint32_t scaleDiff = scale - fRow.getScale(colOut); + // multiplication overflow check + datatypes::MultiplicationOverflowCheck multOp; + int128_t sum = 0; + if (scaleDiff > 0) + multOp(*sumPnt, datatypes::mcs_pow_10[scaleDiff], sum); + else + sum = *sumPnt; + int128_t avg = sum / cnt; + fRow.setBinaryField_offset(&avg, sizeof(avg), offset); + } } } } @@ -2605,11 +2818,11 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) return; } - int64_t intOut = 0; - uint64_t uintOut = 0; - float floatOut = 0.0; - double doubleOut = 0.0; - long double longdoubleOut = 0.0; + int64_t intOut; + uint64_t uintOut; + float floatOut; + double doubleOut; + long double longdoubleOut; ostringstream oss; std::string strOut; @@ -2676,6 +2889,12 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) fRow.setIntField<8>(intOut, colOut); bSetSuccess = true; } + else if (valOut.compatible(int128TypeId)) + { + int128_t int128Out = valOut.cast(); + fRow.setInt128Field(int128Out, colOut); + bSetSuccess = true; + } break; @@ -2799,6 +3018,8 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) if (!bSetSuccess) { + // This means the return from the UDAF doesn't match the field + // This handles the mismatch SetUDAFAnyValue(valOut, colOut); } } @@ -2811,104 +3032,102 @@ void RowAggregationUM::SetUDAFAnyValue(static_any::any& valOut, int64_t colOut) // that they didn't set in mcsv1_UDAF::init(), but this // handles whatever return type is given and casts // it to whatever they said to return. - // TODO: Save cpu cycles here. + // TODO: Save cpu cycles here. For one, we don't need to initialize these + int64_t intOut = 0; uint64_t uintOut = 0; - float floatOut = 0.0; double doubleOut = 0.0; long double longdoubleOut = 0.0; + int128_t int128Out = 0; ostringstream oss; std::string strOut; if (valOut.compatible(charTypeId)) { - uintOut = intOut = valOut.cast(); - floatOut = intOut; + int128Out = uintOut = intOut = valOut.cast(); + doubleOut = intOut; oss << intOut; } else if (valOut.compatible(scharTypeId)) { - uintOut = intOut = valOut.cast(); - floatOut = intOut; + int128Out = uintOut = intOut = valOut.cast(); + doubleOut = intOut; oss << intOut; } else if (valOut.compatible(shortTypeId)) { - uintOut = intOut = valOut.cast(); - floatOut = intOut; + int128Out = uintOut = intOut = valOut.cast(); + doubleOut = intOut; oss << intOut; } else if (valOut.compatible(intTypeId)) { - uintOut = intOut = valOut.cast(); - floatOut = intOut; + int128Out = uintOut = intOut = valOut.cast(); + doubleOut = intOut; oss << intOut; } else if (valOut.compatible(longTypeId)) { - uintOut = intOut = valOut.cast(); - floatOut = intOut; + int128Out = uintOut = intOut = valOut.cast(); + doubleOut = intOut; oss << intOut; } else if (valOut.compatible(llTypeId)) { - uintOut = intOut = valOut.cast(); - floatOut = intOut; + int128Out = uintOut = intOut = valOut.cast(); + doubleOut = intOut; oss << intOut; } else if (valOut.compatible(ucharTypeId)) { - intOut = uintOut = valOut.cast(); - floatOut = uintOut; + int128Out = intOut = uintOut = valOut.cast(); + doubleOut = uintOut; oss << uintOut; } else if (valOut.compatible(ushortTypeId)) { - intOut = uintOut = valOut.cast(); - floatOut = uintOut; + int128Out = intOut = uintOut = valOut.cast(); + doubleOut = uintOut; oss << uintOut; } else if (valOut.compatible(uintTypeId)) { - intOut = uintOut = valOut.cast(); - floatOut = uintOut; + int128Out = intOut = uintOut = valOut.cast(); + doubleOut = uintOut; oss << uintOut; } else if (valOut.compatible(ulongTypeId)) { - intOut = uintOut = valOut.cast(); - floatOut = uintOut; + int128Out = intOut = uintOut = valOut.cast(); + doubleOut = uintOut; oss << uintOut; } else if (valOut.compatible(ullTypeId)) { - intOut = uintOut = valOut.cast(); - floatOut = uintOut; + int128Out = intOut = uintOut = valOut.cast(); + doubleOut = uintOut; oss << uintOut; } - else if (valOut.compatible(floatTypeId)) + else if (valOut.compatible(int128TypeId)) { - floatOut = valOut.cast(); - doubleOut = floatOut; - longdoubleOut = doubleOut; - intOut = uintOut = floatOut; - oss << floatOut; + intOut = uintOut = int128Out = valOut.cast(); + doubleOut = uintOut; + oss << uintOut; } - else if (valOut.compatible(doubleTypeId)) + else if (valOut.compatible(floatTypeId) || valOut.compatible(doubleTypeId)) { - doubleOut = valOut.cast(); - longdoubleOut = doubleOut; - floatOut = (float)doubleOut; - uintOut = (uint64_t)doubleOut; - intOut = (int64_t)doubleOut; + // Should look at scale for decimal and adjust + doubleOut = valOut.cast(); + int128Out = doubleOut; + intOut = uintOut = doubleOut; oss << doubleOut; } - else if (valOut.compatible(longdoubleTypeId)) { + // Should look at scale for decimal and adjust longdoubleOut = valOut.cast(); + int128Out = longdoubleOut; doubleOut = (double)longdoubleOut; - floatOut = (float)doubleOut; uintOut = (uint64_t)doubleOut; intOut = (int64_t)doubleOut; oss << doubleOut; @@ -2922,7 +3141,7 @@ void RowAggregationUM::SetUDAFAnyValue(static_any::any& valOut, int64_t colOut) uintOut = strtoul(strOut.c_str(), NULL, 10); doubleOut = strtod(strOut.c_str(), NULL); longdoubleOut = strtold(strOut.c_str(), NULL); - floatOut = (float)doubleOut; + int128Out = longdoubleOut; } else { @@ -2946,11 +3165,20 @@ void RowAggregationUM::SetUDAFAnyValue(static_any::any& valOut, int64_t colOut) break; case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: fRow.setIntField<8>(intOut, colOut); break; + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + uint32_t width = fRowGroupOut->getColumnWidth(colOut); + if (width == datatypes::MAXDECIMALWIDTH) + fRow.setInt128Field(int128Out, colOut); + else + fRow.setIntField<8>(intOut, colOut); + break; + } + case execplan::CalpontSystemCatalog::UTINYINT: fRow.setUintField<1>(uintOut, colOut); break; @@ -2982,8 +3210,11 @@ void RowAggregationUM::SetUDAFAnyValue(static_any::any& valOut, int64_t colOut) case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::UFLOAT: + { + float floatOut = (float)doubleOut; fRow.setFloatField(floatOut, colOut); break; + } case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::UDOUBLE: @@ -3025,7 +3256,6 @@ void RowAggregationUM::calculateUDAFColumns() RowUDAFFunctionCol* rowUDAF = NULL; static_any::any valOut; - for (uint64_t i = 0; i < fFunctionCols.size(); i++) { if (fFunctionCols[i]->fAggFunction != ROWAGG_UDAF) @@ -3254,10 +3484,28 @@ void RowAggregationUM::doNullConstantAggregate(const ConstantAggData& aggData, u case execplan::CalpontSystemCatalog::MEDINT: case execplan::CalpontSystemCatalog::INT: case execplan::CalpontSystemCatalog::BIGINT: + { + fRow.setIntField(getIntNullValue(colDataType), colOut); + } + break; + case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - fRow.setIntField(getIntNullValue(colDataType), colOut); + auto width = fRow.getColumnWidth(colOut); + if (fRow.getColumnWidth(colOut) == datatypes::MAXDECIMALWIDTH) + { + fRow.setInt128Field(datatypes::Decimal128Null, colOut); + } + else if (width <= datatypes::MAXLEGACYWIDTH) + { + fRow.setIntField(getIntNullValue(colDataType), colOut); + } + else + { + idbassert(0); + throw std::logic_error("RowAggregationUM::doNullConstantAggregate(): DECIMAL bad length."); + } } break; @@ -3417,7 +3665,7 @@ void RowAggregationUM::doNullConstantAggregate(const ConstantAggData& aggData, u void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData, uint64_t i) { int64_t colOut = fFunctionCols[i]->fOutputColumnIndex; - int colDataType = (fRowGroupOut->getColTypes())[colOut]; + auto colDataType = (fRowGroupOut->getColTypes())[colOut]; int64_t rowCnt = fRow.getIntField(fFunctionCols[i]->fAuxColumnIndex); switch (aggData.fOp) @@ -3439,7 +3687,7 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData { fRow.setIntField(strtol(aggData.fConstValue.c_str(), 0, 10), colOut); } - break; + break; // AVG should not be uint32_t result type. case execplan::CalpontSystemCatalog::UTINYINT: @@ -3455,9 +3703,27 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - double dbl = strtod(aggData.fConstValue.c_str(), 0); - double scale = pow(10.0, (double) fRowGroupOut->getScale()[i]); - fRow.setIntField((int64_t)(scale * dbl), colOut); + auto width = fRow.getColumnWidth(colOut); + if (width == datatypes::MAXDECIMALWIDTH) + { + execplan::CalpontSystemCatalog::TypeHolderStd colType; + colType.colWidth = width; + colType.precision = fRow.getPrecision(i); + colType.scale = fRow.getScale(i); + colType.colDataType = colDataType; + fRow.setInt128Field(colType.decimal128FromString(aggData.fConstValue), colOut); + } + else if (width <= datatypes::MAXLEGACYWIDTH) + { + double dbl = strtod(aggData.fConstValue.c_str(), 0); + double scale = pow(10.0, (double) fRowGroupOut->getScale()[i]); + fRow.setIntField((int64_t)(scale * dbl), colOut); + } + else + { + idbassert(0); + throw std::logic_error("RowAggregationUM::doNotNullConstantAggregate(): DECIMAL bad length."); + } } break; @@ -3562,15 +3828,38 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - double dbl = strtod(aggData.fConstValue.c_str(), 0); - dbl *= pow(10.0, (double) fRowGroupOut->getScale()[i]); - dbl *= rowCnt; + auto width = fRow.getColumnWidth(colOut); + if (width == datatypes::MAXDECIMALWIDTH) + { + execplan::CalpontSystemCatalog::TypeHolderStd colType; + colType.colWidth = width; + colType.precision = fRow.getPrecision(i); + colType.scale = fRow.getScale(i); + colType.colDataType = colDataType; + int128_t constValue = colType.decimal128FromString(aggData.fConstValue); + int128_t sum; - if ((dbl > 0 && dbl > (double) numeric_limits::max()) || - (dbl < 0 && dbl < (double) numeric_limits::min())) - throw logging::QueryDataExcept(overflowMsg, logging::aggregateDataErr); + datatypes::MultiplicationOverflowCheck multOp; + multOp(constValue, rowCnt, sum); + fRow.setInt128Field(sum, colOut); + } + else if (width == datatypes::MAXLEGACYWIDTH) + { + double dbl = strtod(aggData.fConstValue.c_str(), 0); + dbl *= pow(10.0, (double) fRowGroupOut->getScale()[i]); + dbl *= rowCnt; + + if ((dbl > 0 && dbl > (double) numeric_limits::max()) || + (dbl < 0 && dbl < (double) numeric_limits::min())) + throw logging::QueryDataExcept(overflowMsg, logging::aggregateDataErr); + fRow.setIntField((int64_t) dbl, colOut); + } + else + { + idbassert(0); + throw std::logic_error("RowAggregationUM::doNotNullConstantAggregate(): sum() DECIMAL bad length."); + } - fRow.setIntField((int64_t) dbl, colOut); } break; @@ -4010,8 +4299,10 @@ RowAggregationUMP2::~RowAggregationUMP2() // Update the aggregation totals in the internal hashmap for the specified row. // NULL values are recognized and ignored for all agg functions except for count // rowIn(in) - Row to be included in aggregation. +// rgContextColl(in) - ptr to a vector of UDAF contexts //------------------------------------------------------------------------------ -void RowAggregationUMP2::updateEntry(const Row& rowIn) +void RowAggregationUMP2::updateEntry(const Row& rowIn, + std::vector* rgContextColl) { for (uint64_t i = 0; i < fFunctionCols.size(); i++) { @@ -4077,7 +4368,7 @@ void RowAggregationUMP2::updateEntry(const Row& rowIn) case ROWAGG_UDAF: { - doUDAF(rowIn, colIn, colOut, colAux, i); + doUDAF(rowIn, colIn, colOut, colAux, i, rgContextColl); break; } @@ -4110,6 +4401,8 @@ void RowAggregationUMP2::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, int colDataType = (fRowGroupIn.getColTypes())[colIn]; long double valIn = 0; long double valOut = fRow.getLongDoubleField(colOut); + bool isWideDataType = false; + void *wideValInPtr = nullptr; switch (colDataType) { @@ -4136,13 +4429,28 @@ void RowAggregationUMP2::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - valIn = rowIn.getIntField(colIn); - break; - double scale = (double)(fRowGroupIn.getScale())[colIn]; - if (valIn != 0 && scale > 0) + uint32_t width = fRowGroupIn.getColumnWidth(colIn); + isWideDataType = width == datatypes::MAXDECIMALWIDTH; + if(LIKELY(isWideDataType)) { - valIn /= pow(10.0, scale); + int128_t* dec = rowIn.getBinaryField(colIn); + wideValInPtr = reinterpret_cast(dec); } + else if (width <= datatypes::MAXLEGACYWIDTH) + { + valIn = rowIn.getIntField(colIn); + double scale = (double)(fRowGroupIn.getScale())[colIn]; + if (valIn != 0 && scale > 0) + { + valIn /= pow(10.0, scale); + } + } + else + { + idbassert(0); + throw std::logic_error("RowAggregationUMP2::doAvg(): DECIMAL bad length."); + } + break; } @@ -4176,16 +4484,36 @@ void RowAggregationUMP2::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, } } - int64_t cnt = fRow.getUintField(colAux); - if (cnt == 0) + uint64_t cnt = fRow.getUintField(colAux); + if (LIKELY(!isWideDataType)) { - fRow.setLongDoubleField(valIn, colOut); - fRow.setUintField(rowIn.getUintField(colIn + 1), colAux); + if (LIKELY(cnt > 0)) + { + fRow.setLongDoubleField(valIn + valOut, colOut); + fRow.setUintField(rowIn.getUintField(colIn + 1) + cnt, colAux); + } + else + { + fRow.setLongDoubleField(valIn, colOut); + fRow.setUintField(rowIn.getUintField(colIn + 1), colAux); + } } else { - fRow.setLongDoubleField(valIn + valOut, colOut); - fRow.setUintField(rowIn.getUintField(colIn + 1) + cnt, colAux); + uint32_t offset = fRow.getOffset(colOut); + int128_t* dec = reinterpret_cast(wideValInPtr); + if (LIKELY(cnt > 0)) + { + int128_t *valOutPtr = fRow.getBinaryField(colOut); + int128_t sum = *valOutPtr + *dec; + fRow.setBinaryField_offset(&sum, sizeof(sum), offset); + fRow.setUintField(rowIn.getUintField(colIn + 1) + cnt, colAux); + } + else + { + fRow.setBinaryField_offset(dec, sizeof(*dec), offset); + fRow.setUintField(rowIn.getUintField(colIn + 1), colAux); + } } } @@ -4252,11 +4580,23 @@ void RowAggregationUMP2::doBitOp(const Row& rowIn, int64_t colIn, int64_t colOut // colOut(in) - column in the output row group // colAux(in) - Where the UDAF userdata resides // rowUDAF(in) - pointer to the RowUDAFFunctionCol for this UDAF instance +// rgContextColl(in) - ptr to a vector that brings UDAF contextx in //------------------------------------------------------------------------------ -void RowAggregationUMP2::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, - int64_t colAux, uint64_t& funcColsIdx) +void RowAggregationUMP2::doUDAF(const Row& rowIn, + int64_t colIn, + int64_t colOut, + int64_t colAux, + uint64_t& funcColsIdx, + std::vector* rgContextColl) { static_any::any valOut; + std::vector* udafContextsCollPtr = &fRGContextColl; + if (UNLIKELY(rgContextColl != nullptr)) + { + udafContextsCollPtr = rgContextColl; + } + + std::vector& udafContextsColl = *udafContextsCollPtr; // Get the user data boost::shared_ptr userDataIn = rowIn.getUserData(colIn + 1); @@ -4269,7 +4609,7 @@ void RowAggregationUMP2::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, if (!userDataIn) { - if (fRGContext.getRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS)) + if (udafContextsColl[funcColsIdx].getRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS)) { return; } @@ -4278,14 +4618,14 @@ void RowAggregationUMP2::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, flags[0] |= mcsv1sdk::PARAM_IS_NULL; } - fRGContext.setDataFlags(flags); + udafContextsColl[funcColsIdx].setDataFlags(flags); // The intermediate values are stored in colAux. - fRGContext.setUserData(fRow.getUserData(colAux)); + udafContextsColl[funcColsIdx].setUserData(fRow.getUserData(colAux)); // Call the UDAF subEvaluate method mcsv1sdk::mcsv1_UDAF::ReturnCode rc; - rc = fRGContext.getFunction()->subEvaluate(&fRGContext, userDataIn.get()); + rc = udafContextsColl[funcColsIdx].getFunction()->subEvaluate(&udafContextsColl[funcColsIdx], userDataIn.get()); fRGContext.setUserData(NULL); if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) @@ -4404,8 +4744,10 @@ void RowAggregationDistinct::doDistinctAggregation_rowVec(vector& // Update the aggregation totals in the internal hashmap for the specified row. // for non-DISTINCT columns works partially aggregated results // rowIn(in) - Row to be included in aggregation. +// rgContextColl(in) - ptr to a vector of UDAF contexts //------------------------------------------------------------------------------ -void RowAggregationDistinct::updateEntry(const Row& rowIn) +void RowAggregationDistinct::updateEntry(const Row& rowIn, + std::vector* rgContextColl) { for (uint64_t i = 0; i < fFunctionCols.size(); i++) { @@ -4487,7 +4829,7 @@ void RowAggregationDistinct::updateEntry(const Row& rowIn) case ROWAGG_UDAF: { - doUDAF(rowIn, colIn, colOut, colAux, i); + doUDAF(rowIn, colIn, colOut, colAux, i, rgContextColl); break; } @@ -4775,6 +5117,7 @@ void RowAggregationMultiDistinct::doDistinctAggregation() { fFunctionCols = fSubFunctions[i]; fRowGroupIn = fSubRowGroups[i]; + auto* rgContextColl = fSubAggregators[i]->rgContextColl(); Row rowIn; fRowGroupIn.initRow(&rowIn); @@ -4790,14 +5133,14 @@ void RowAggregationMultiDistinct::doDistinctAggregation() for (uint64_t j = 0; j < fRowGroupIn.getRowCount(); ++j, rowIn.nextRow()) { - aggregateRow(rowIn); + aggregateRow(rowIn, rgContextColl); } } } // restore the function column vector fFunctionCols = origFunctionCols; - fOrigFunctionCols = NULL; + fOrigFunctionCols = nullptr; } @@ -4812,20 +5155,21 @@ void RowAggregationMultiDistinct::doDistinctAggregation_rowVec(vectorrgContextColl(); Row rowIn; fRowGroupIn.initRow(&rowIn); for (uint64_t j = 0; j < inRows[i].size(); ++j) { rowIn.setData(inRows[i][j]); - aggregateRow(rowIn); + aggregateRow(rowIn, rgContextColl); } inRows[i].clear(); } // restore the function column vector fFunctionCols = origFunctionCols; - fOrigFunctionCols = NULL; + fOrigFunctionCols = nullptr; } diff --git a/utils/rowgroup/rowaggregation.h b/utils/rowgroup/rowaggregation.h index f0d50381f..d41cd87f8 100644 --- a/utils/rowgroup/rowaggregation.h +++ b/utils/rowgroup/rowaggregation.h @@ -605,7 +605,8 @@ public: return fResultDataVec; } - virtual void aggregateRow(Row& row); + virtual void aggregateRow(Row& row, + std::vector* rgContextColl = nullptr); inline uint32_t aggMapKeyLength() { return fAggMapKeyCount; @@ -619,19 +620,29 @@ public: { return fTimeZone; } + inline std::vector* rgContextColl() + { + return &fRGContextColl; + } protected: virtual void initialize(); virtual void initMapData(const Row& row); virtual void attachGroupConcatAg(); - virtual void updateEntry(const Row& row); + virtual void updateEntry(const Row& row, + std::vector* rgContextColl = nullptr); virtual void doMinMax(const Row&, int64_t, int64_t, int); virtual void doSum(const Row&, int64_t, int64_t, int); virtual void doAvg(const Row&, int64_t, int64_t, int64_t); virtual void doStatistics(const Row&, int64_t, int64_t, int64_t); virtual void doBitOp(const Row&, int64_t, int64_t, int); - virtual void doUDAF(const Row&, int64_t, int64_t, int64_t, uint64_t& funcColsIdx); + virtual void doUDAF(const Row&, + int64_t, + int64_t, + int64_t, + uint64_t& funcColsIdx, + std::vector* rgContextColl = nullptr); virtual bool countSpecial(const RowGroup* pRG) { fRow.setUintField<8>(fRow.getUintField<8>(0) + pRG->getRowCount(), 0); @@ -645,6 +656,7 @@ protected: } void resetUDAF(RowUDAFFunctionCol* rowUDAF); + void resetUDAF(RowUDAFFunctionCol* rowUDAF, uint64_t funcColIdx); inline bool isNull(const RowGroup* pRowGroup, const Row& row, int64_t col); inline void makeAggFieldsNull(Row& row); @@ -653,6 +665,7 @@ protected: copyRow(fNullRow, &row); } + inline void updateIntMinMax(int128_t* val1, int128_t* val2, int64_t col, int func); inline void updateIntMinMax(int64_t val1, int64_t val2, int64_t col, int func); inline void updateUintMinMax(uint64_t val1, uint64_t val2, int64_t col, int func); inline void updateCharMinMax(uint64_t val1, uint64_t val2, int64_t col, int func); @@ -709,6 +722,7 @@ protected: // We need a separate copy for each thread. mcsv1sdk::mcsv1Context fRGContext; + std::vector fRGContextColl; // These are handy for testing the actual type of static_any for UDAF static const static_any::any& charTypeId; @@ -717,6 +731,7 @@ protected: static const static_any::any& intTypeId; static const static_any::any& longTypeId; static const static_any::any& llTypeId; + static const static_any::any& int128TypeId; static const static_any::any& ucharTypeId; static const static_any::any& ushortTypeId; static const static_any::any& uintTypeId; @@ -819,19 +834,21 @@ public: return fGroupConcat; } - void aggregateRow(Row&); - //void initialize(); + void aggregateRow(Row&, + std::vector* rgContextColl = nullptr) override; virtual void aggReset(); void setInputOutput(const RowGroup& pRowGroupIn, RowGroup* pRowGroupOut); protected: - // virtual methods from base - void initialize(); - void aggregateRowWithRemap(Row&); + void initialize() override; + void updateEntry(const Row& row, + std::vector* rgContextColl = nullptr) override; + + void aggregateRowWithRemap(Row&, + std::vector* rgContextColl = nullptr); void attachGroupConcatAg(); - void updateEntry(const Row& row); bool countSpecial(const RowGroup* pRG) { fRow.setIntField<8>( @@ -944,12 +961,18 @@ public: protected: // virtual methods from base - void updateEntry(const Row& row); + void updateEntry(const Row& row, + std::vector* rgContextColl = nullptr) override; void doAvg(const Row&, int64_t, int64_t, int64_t); void doStatistics(const Row&, int64_t, int64_t, int64_t); void doGroupConcat(const Row&, int64_t, int64_t); void doBitOp(const Row&, int64_t, int64_t, int); - void doUDAF(const Row&, int64_t, int64_t, int64_t, uint64_t& funcColsIdx); + void doUDAF(const Row&, + int64_t, + int64_t, + int64_t, + uint64_t& funcColsIdx, + std::vector* rgContextColl = nullptr) override; bool countSpecial(const RowGroup* pRG) { return false; @@ -1017,8 +1040,8 @@ public: } protected: - // virtual methods from base - void updateEntry(const Row& row); + void updateEntry(const Row& row, + std::vector* rgContextColl = nullptr) override; boost::shared_ptr fAggregator; RowGroup fRowGroupDist; diff --git a/utils/rowgroup/rowgroup-tests.cpp b/utils/rowgroup/rowgroup-tests.cpp new file mode 100644 index 000000000..3ebfb6c0b --- /dev/null +++ b/utils/rowgroup/rowgroup-tests.cpp @@ -0,0 +1,241 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include // googletest header file +#include + +#include "rowgroup.h" +#include "columnwidth.h" +#include "joblisttypes.h" +#include "dataconvert.h" + +#define WIDE_DEC_PRECISION 38U +#define INITIAL_ROW_OFFSET 2 + +using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; + +class RowDecimalTest : public ::testing::Test { + protected: + void SetUp() override { + uint32_t precision = WIDE_DEC_PRECISION; + uint32_t oid =3001; + + std::vectortypes; + std::vectorprecisionVec; + std::vector roids, tkeys, cscale; + types.push_back(execplan::CalpontSystemCatalog::DECIMAL); + types.push_back(execplan::CalpontSystemCatalog::UDECIMAL); + for (size_t i=0; i <= 3; i++) { + types.push_back(execplan::CalpontSystemCatalog::DECIMAL); + } + precisionVec.push_back(precision); + precisionVec.push_back(precision); + precisionVec.push_back(18); + precisionVec.push_back(9); + precisionVec.push_back(4); + precisionVec.push_back(2); + std::vectorwidthVec; + uint32_t offset = INITIAL_ROW_OFFSET; + offsets.push_back(offset); + for (size_t i=0; i < types.size(); i++) { + uint8_t width = utils::widthByPrecision(precisionVec[i]); + widthVec.push_back(width); + offset += width; + offsets.push_back(offset); + roids.push_back(oid+i); + tkeys.push_back(i+1); cscale.push_back(0); + } + /*offsets.push_back(INITIAL_ROW_OFFSET); + offsets.push_back(16+INITIAL_ROW_OFFSET); + offsets.push_back(16*2+INITIAL_ROW_OFFSET); + roids.push_back(oid); roids.push_back(oid+1); + tkeys.push_back(1); tkeys.push_back(1); + cscale.push_back(0); cscale.push_back(0);*/ + + rowgroup::RowGroup inRG(roids.size(), //column count + offsets, //oldOffset + roids, // column oids + tkeys, //keys + types, // types + cscale, //scale + precisionVec, // precision + 20, // sTableThreshold + false //useStringTable + ); + + rg = inRG; + rgD.reinit(rg); + rg.setData(&rgD); + + rg.initRow(&r); + rg.initRow(&rOutMappingCheck); + rowSize = r.getSize(); + rg.getRow(0, &r); + + int128_t nullValue = 0; + int128_t bigValue = 0; + uint64_t* uint128_pod = reinterpret_cast(&nullValue); + uint128_pod[0] = joblist::UBIGINTNULL; + uint128_pod[1] = joblist::UBIGINTEMPTYROW; + bigValue = -static_cast(0xFFFFFFFF)*0xFFFFFFFFFFFFFFFF; + sValueVector.push_back(nullValue); + sValueVector.push_back(-42); + sValueVector.push_back(bigValue); + sValueVector.push_back(0); + sValueVector.push_back(nullValue-1); + + uValueVector.push_back(nullValue); + uValueVector.push_back(42); + uValueVector.push_back(bigValue); + uValueVector.push_back(0); + uValueVector.push_back(nullValue-1); + + s8ValueVector.push_back(joblist::TINYINTNULL); + s8ValueVector.push_back(-0x79); + s8ValueVector.push_back(0); + s8ValueVector.push_back(0x81); + s8ValueVector.push_back(joblist::TINYINTNULL-1); + + s16ValueVector.push_back(joblist::SMALLINTNULL); + s16ValueVector.push_back(-0x79); + s16ValueVector.push_back(0); + s16ValueVector.push_back(0x81); + s16ValueVector.push_back(joblist::SMALLINTNULL-1); + + s32ValueVector.push_back(joblist::INTNULL); + s32ValueVector.push_back(-0x79); + s32ValueVector.push_back(0); + s32ValueVector.push_back(0x81); + s32ValueVector.push_back(joblist::INTNULL-1); + + s64ValueVector.push_back(joblist::BIGINTNULL); + s64ValueVector.push_back(-0x79); + s64ValueVector.push_back(0); + s64ValueVector.push_back(0x81); + s64ValueVector.push_back(joblist::BIGINTNULL-1); + + for(size_t i = 0; i < sValueVector.size(); i++) { + r.setBinaryField_offset(&sValueVector[i], + sizeof(sValueVector[0]), offsets[0]); + r.setBinaryField_offset(&uValueVector[i], + sizeof(uValueVector[0]), offsets[1]); + r.setIntField(s64ValueVector[i], 2); + r.setIntField(s32ValueVector[i], 3); + r.setIntField(s16ValueVector[i], 4); + r.setIntField(s8ValueVector[i], 5); + r.nextRow(rowSize); + } + rowCount = sValueVector.size(); + } + // void TearDown() override {} + + rowgroup::Row r; + rowgroup::Row rOutMappingCheck; + rowgroup::RowGroup rg; + rowgroup::RGData rgD; + uint32_t rowSize; + size_t rowCount; + std::vector sValueVector; + std::vector uValueVector; + std::vector s8ValueVector; + std::vector s16ValueVector; + std::vector s32ValueVector; + std::vector s64ValueVector; + std::vector offsets; +}; + +TEST_F(RowDecimalTest, NonNULLValuesCheck) { + rg.getRow(1, &r); + for (size_t i = 1; i <= sValueVector.size(); i++) { + EXPECT_FALSE(r.isNullValue(0)); + EXPECT_FALSE(r.isNullValue(1)); + EXPECT_FALSE(r.isNullValue(2)); + EXPECT_FALSE(r.isNullValue(3)); + EXPECT_FALSE(r.isNullValue(4)); + EXPECT_FALSE(r.isNullValue(5)); + r.nextRow(rowSize); + } +} + +TEST_F(RowDecimalTest, initToNullANDisNullValueValueCheck) { + rg.getRow(0, &r); + r.initToNull(); + EXPECT_TRUE(r.isNullValue(0)); + EXPECT_TRUE(r.isNullValue(1)); + EXPECT_TRUE(r.isNullValue(2)); + EXPECT_TRUE(r.isNullValue(3)); + EXPECT_TRUE(r.isNullValue(4)); + EXPECT_TRUE(r.isNullValue(5)); +} + +TEST_F(RowDecimalTest, getBinaryFieldCheck) { + rg.getRow(0, &r); + uint128_t* u128Value; + int128_t* s128Value; +// std::remove_reference::type uType; +// std::remove_reference::type sType; + + for (size_t i = 0; i < sValueVector.size(); i++) { + s128Value = r.getBinaryField(0); + EXPECT_EQ(sValueVector[i], *s128Value); + u128Value = r.getBinaryField(1); + EXPECT_EQ(uValueVector[i], *u128Value); + //EXPECT_EQ(s64ValueVector[i], r.getIntField(2)); + //EXPECT_EQ(s32ValueVector[i],r.getIntField(3)); + //EXPECT_EQ(r.getIntField(4),s16ValueVector[i]); + //EXPECT_EQ(r.getIntField(5),s8ValueVector[i]); + r.nextRow(rowSize); + } +} + +TEST_F(RowDecimalTest, toStringCheck) { + std::vector exemplarVector; + exemplarVector.push_back(std::string("0: NULL NULL NULL NULL NULL NULL ")); + exemplarVector.push_back(std::string("0: -42 42 -121 -121 -121 -121 ")); + exemplarVector.push_back(std::string("0: -79228162495817593515539431425 -79228162495817593515539431425 0 0 0 0 ")); + exemplarVector.push_back(std::string("0: 0 0 129 129 129 -127 ")); + exemplarVector.push_back(std::string("0: -3 -3 9223372036854775807 2147483647 32767 127 ")); + + rg.getRow(0, &r); + r.initToNull(); + for (auto &el: exemplarVector) { + EXPECT_EQ(el, r.toString()); + r.nextRow(rowSize); + } +} + +TEST_F(RowDecimalTest, toCSVCheck) { +} + +TEST_F(RowDecimalTest, applyMappingCheck) { + int mapping[] = {0, 1, -1, -1, -1, -1}; + rg.getRow(1, &r); + rg.getRow(2, &rOutMappingCheck); + int128_t* s128Value = rOutMappingCheck.getBinaryField(0); + uint128_t* u128Value = rOutMappingCheck.getBinaryField(1); + EXPECT_NE(sValueVector[1], *s128Value); + EXPECT_NE(uValueVector[1], *u128Value); + applyMapping(mapping, r, &rOutMappingCheck); + s128Value = rOutMappingCheck.getBinaryField(0); + EXPECT_EQ(sValueVector[1], *s128Value); + u128Value = rOutMappingCheck.getBinaryField(1); + EXPECT_EQ(uValueVector[1], *u128Value); +} + +// WIP +TEST_F(RowDecimalTest, RowEqualsCheck) { +} diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 9f16123cf..d477bafc3 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -42,12 +42,16 @@ using namespace execplan; #include "nullvaluemanip.h" #include "rowgroup.h" +#include "dataconvert.h" +#include "columnwidth.h" #include "collation.h" namespace rowgroup { +using cscType = execplan::CalpontSystemCatalog::ColDataType; + StringStore::StringStore() : empty(true), fUseStoreStringMutex(false) { } StringStore::StringStore(const StringStore&) @@ -628,7 +632,22 @@ string Row::toString() const os << " " << dec; break; } - + // WIP MCOL-641 + case CalpontSystemCatalog::BINARY: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; + break; + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + if (colWidths[i] == datatypes::MAXDECIMALWIDTH) + { + datatypes::VDecimal dec(0, + scale[i], + precision[i], + getBinaryField(i)); + os << dec << " "; + break; + } + //fallthrough default: os << getIntField(i) << " "; break; @@ -690,6 +709,9 @@ string Row::toCSV() const os << dec; break; } + case CalpontSystemCatalog::BINARY: + std::cout << __FILE__<< __LINE__ << ":" << "toCSV"<< std::endl; + //fallthrough default: os << getIntField(i); @@ -797,7 +819,6 @@ void Row::initToNull() default: *((uint64_t*) &data[offsets[i]]) = *((uint64_t*) joblist::CPNULLSTRMARK.c_str()); memset(&data[offsets[i] + 8], 0, len - 8); - //strcpy((char *) &data[offsets[i]], joblist::CPNULLSTRMARK.c_str()); break; } @@ -828,6 +849,9 @@ void Row::initToNull() *((int32_t*) &data[offsets[i]]) = static_cast(joblist::INTNULL); break; + case 16 : + datatypes::Decimal::setWideDecimalNullValue(reinterpret_cast(data[offsets[i]])); + break; default: *((int64_t*) &data[offsets[i]]) = static_cast(joblist::BIGINTNULL); break; @@ -852,6 +876,11 @@ void Row::initToNull() case CalpontSystemCatalog::UBIGINT: *((uint64_t*) &data[offsets[i]]) = joblist::UBIGINTNULL; break; + case CalpontSystemCatalog::BINARY: + { + datatypes::Decimal::setWideDecimalNullValue(reinterpret_cast(data[offsets[i]])); + } + break; default: ostringstream os; @@ -863,6 +892,84 @@ void Row::initToNull() } } +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + ostringstream os; + os << "Row::isNullValue(): got bad column type at offset("; + os << offset; + os << "). Width="; + os << width << endl; + throw logic_error(os.str()); +} + +// WIP Method template resolution could impose some perf degradation +// Compare perf with switch-case +template<> +inline bool +Row::isNullValue_offset( + uint32_t offset) const +{ + const uint64_t *intPtr = reinterpret_cast(&data[offset]); + return ((intPtr[0] == static_cast(utils::BINARYNULLVALUELOW)) && + (intPtr[1] == static_cast(utils::BINARYNULLVALUELOW)) && + (intPtr[2] == static_cast(utils::BINARYNULLVALUELOW)) && + (intPtr[3] == static_cast(utils::BINARYEMPTYVALUEHIGH))); +} + +template<> +inline bool +Row::isNullValue_offset( + uint32_t offset) const +{ + const int128_t *intPtr = reinterpret_cast(&data[offset]); + return datatypes::Decimal::isWideDecimalNullValue (*intPtr); +} + +template<> +inline bool +Row::isNullValue_offset( + uint32_t offset) const +{ + const int128_t *intPtr = reinterpret_cast(&data[offset]); + return datatypes::Decimal::isWideDecimalNullValue (*intPtr); +} + +template<> +inline bool +Row::isNullValue_offset( + uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) + == static_cast(joblist::BIGINTNULL)); +} + +template<> +inline bool +Row::isNullValue_offset( + uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) + == static_cast(joblist::INTNULL)); +} + +template<> +inline bool +Row::isNullValue_offset( + uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) + == static_cast(joblist::SMALLINTNULL)); +} + +template<> +inline bool +Row::isNullValue_offset( + uint32_t offset) const +{ + return (data[offset] == joblist::TINYINTNULL); +} + bool Row::isNullValue(uint32_t colIndex) const { switch (types[colIndex]) @@ -934,7 +1041,6 @@ bool Row::isNullValue(uint32_t colIndex) const case 8: return (*((uint64_t*) &data[offsets[colIndex]]) == joblist::CHAR8NULL); - default: return (*((uint64_t*) &data[offsets[colIndex]]) == *((uint64_t*) joblist::CPNULLSTRMARK.c_str())); } @@ -945,10 +1051,14 @@ bool Row::isNullValue(uint32_t colIndex) const case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - uint32_t len = getColumnWidth(colIndex); - - switch (len) + // WIP MCOL-641 Allmighty hack. + switch (getColumnWidth(colIndex)) { + // MCOL-641 + case 16: + return isNullValue_offset + (offsets[colIndex]); + case 1 : return (data[offsets[colIndex]] == joblist::TINYINTNULL); @@ -1004,6 +1114,15 @@ bool Row::isNullValue(uint32_t colIndex) const return (*((long double*) &data[offsets[colIndex]]) == joblist::LONGDOUBLENULL); break; + case CalpontSystemCatalog::BINARY: + { + const uint32_t len = 16; + uint32_t* lenPtr = const_cast(&len); + *lenPtr = getColumnWidth(colIndex); + return isNullValue_offset + (offsets[colIndex]); + } + default: { ostringstream os; @@ -1055,10 +1174,11 @@ bool Row::equals(const Row& r2, const std::vector& keyCols) const for (uint32_t i = 0; i < keyCols.size(); i++) { const uint32_t& col = keyCols[i]; + cscDataType columnType = getColType(col); - if (UNLIKELY(getColType(col) == execplan::CalpontSystemCatalog::VARCHAR || - (getColType(col) == execplan::CalpontSystemCatalog::CHAR && (colWidths[col] > 1)) || - getColType(col) == execplan::CalpontSystemCatalog::TEXT)) + if (UNLIKELY(columnType == execplan::CalpontSystemCatalog::VARCHAR || + (columnType == execplan::CalpontSystemCatalog::CHAR && (colWidths[col] > 1)) || + columnType == execplan::CalpontSystemCatalog::TEXT)) { CHARSET_INFO* cs = getCharset(col); if (cs->strnncollsp(getStringPointer(col), getStringLength(col), @@ -1067,7 +1187,7 @@ bool Row::equals(const Row& r2, const std::vector& keyCols) const return false; } } - else if (UNLIKELY(getColType(col) == execplan::CalpontSystemCatalog::BLOB)) + else if (UNLIKELY(columnType == execplan::CalpontSystemCatalog::BLOB)) { if (getStringLength(col) != r2.getStringLength(col)) return false; @@ -1077,13 +1197,20 @@ bool Row::equals(const Row& r2, const std::vector& keyCols) const } else { - if (getColType(col) == execplan::CalpontSystemCatalog::LONGDOUBLE) + if (UNLIKELY(columnType == execplan::CalpontSystemCatalog::LONGDOUBLE)) { if (getLongDoubleField(col) != r2.getLongDoubleField(col)) return false; } + else if (UNLIKELY(datatypes::isWideDecimalType(columnType, colWidths[col]))) + { + if (*getBinaryField(col) != *r2.getBinaryField(col)) + return false; + } else if (getUintField(col) != r2.getUintField(col)) + { return false; + } } } @@ -1107,9 +1234,10 @@ bool Row::equals(const Row& r2, uint32_t lastCol) const // because binary equality is not equality for many charsets/collations for (uint32_t col = 0; col <= lastCol; col++) { - if (UNLIKELY(getColType(col) == execplan::CalpontSystemCatalog::VARCHAR || - (getColType(col) == execplan::CalpontSystemCatalog::CHAR && (colWidths[col] > 1)) || - getColType(col) == execplan::CalpontSystemCatalog::TEXT)) + cscDataType columnType = getColType(col); + if (UNLIKELY(columnType == execplan::CalpontSystemCatalog::VARCHAR || + (columnType == execplan::CalpontSystemCatalog::CHAR && (colWidths[col] > 1)) || + columnType == execplan::CalpontSystemCatalog::TEXT)) { CHARSET_INFO* cs = getCharset(col); if (cs->strnncollsp(getStringPointer(col), getStringLength(col), @@ -1118,7 +1246,7 @@ bool Row::equals(const Row& r2, uint32_t lastCol) const return false; } } - else if (UNLIKELY(getColType(col) == execplan::CalpontSystemCatalog::BLOB)) + else if (UNLIKELY(columnType == execplan::CalpontSystemCatalog::BLOB)) { if (getStringLength(col) != r2.getStringLength(col)) return false; @@ -1128,13 +1256,20 @@ bool Row::equals(const Row& r2, uint32_t lastCol) const } else { - if (getColType(col) == execplan::CalpontSystemCatalog::LONGDOUBLE) + if (UNLIKELY(columnType == execplan::CalpontSystemCatalog::LONGDOUBLE)) { if (getLongDoubleField(col) != r2.getLongDoubleField(col)) return false; } + else if (UNLIKELY(datatypes::isWideDecimalType(columnType, colWidths[col]))) + { + if (*getBinaryField(col) != *r2.getBinaryField(col)) + return false; + } else if (getUintField(col) != r2.getUintField(col)) + { return false; + } } } return true; @@ -1509,11 +1644,17 @@ void applyMapping(const int* mapping, const Row& in, Row* out) out->setVarBinaryField(in.getVarBinaryField(i), in.getVarBinaryLength(i), mapping[i]); else if (UNLIKELY(in.isLongString(i))) out->setStringField(in.getStringPointer(i), in.getStringLength(i), mapping[i]); - //out->setStringField(in.getStringField(i), mapping[i]); else if (UNLIKELY(in.isShortString(i))) out->setUintField(in.getUintField(i), mapping[i]); else if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::LONGDOUBLE)) out->setLongDoubleField(in.getLongDoubleField(i), mapping[i]); + // WIP this doesn't look right b/c we can pushdown colType + // Migrate to offset based methods here + // code precision 2 width convertor + else if (UNLIKELY(datatypes::isWideDecimalType(in.getColTypes()[i], + in.getColumnWidth(i)))) + out->setBinaryField_offset(in.getBinaryField(i), 16, + out->getOffset(mapping[i])); else if (in.isUnsigned(i)) out->setUintField(in.getUintField(i), mapping[i]); else @@ -1624,7 +1765,8 @@ void RowGroup::addToSysDataList(execplan::CalpontSystemCatalog::NJLSysDataList& case 8: cr->PutData(row.getUintField<8>(j)); break; - + case 16: + default: { string s = row.getStringField(j); @@ -1645,6 +1787,9 @@ void RowGroup::addToSysDataList(execplan::CalpontSystemCatalog::NJLSysDataList& cr->PutData(row.getUintField<4>(j)); break; + case CalpontSystemCatalog::BINARY: + std::cout << __FILE__<< __LINE__ << __func__<< std::endl; + //fallthrough default: cr->PutData(row.getIntField<8>(j)); } diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index e3b3652e9..45f2be327 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -55,6 +55,7 @@ #include "mcsv1_udaf.h" #include "branchpred.h" +#include "datatypes/mcs_int128.h" #include "../winport/winport.h" @@ -372,10 +373,13 @@ public: inline uint64_t getUintField(uint32_t colIndex) const; template inline int64_t getIntField(uint32_t colIndex) const; inline int64_t getIntField(uint32_t colIndex) const; + template + inline bool equals(T* value, uint32_t colIndex) const; template inline bool equals(uint64_t val, uint32_t colIndex) const; inline bool equals(long double val, uint32_t colIndex) const; bool equals(const std::string& val, uint32_t colIndex) const; - + inline bool equals(const int128_t& val, uint32_t colIndex) const; + inline double getDoubleField(uint32_t colIndex) const; inline float getFloatField(uint32_t colIndex) const; inline double getDecimalField(uint32_t colIndex) const @@ -383,6 +387,8 @@ public: return 0.0; // TODO: Do something here } inline long double getLongDoubleField(uint32_t colIndex) const; + inline void getInt128Field(uint32_t colIndex, int128_t& x) const; + inline datatypes::TSInt128 getTSInt128Field(uint32_t colIndex) const; inline uint64_t getBaseRid() const; inline uint64_t getRid() const; @@ -411,7 +417,8 @@ public: inline void setDoubleField(double val, uint32_t colIndex); inline void setFloatField(float val, uint32_t colIndex); inline void setDecimalField(double val, uint32_t colIndex) { }; // TODO: Do something here - inline void setLongDoubleField(long double val, uint32_t colIndex); + inline void setLongDoubleField(const long double& val, uint32_t colIndex); + inline void setInt128Field(const int128_t& val, uint32_t colIndex); inline void setRid(uint64_t rid); @@ -421,7 +428,12 @@ public: inline uint32_t getStringLength(uint32_t colIndex) const; void setStringField(const std::string& val, uint32_t colIndex); inline void setStringField(const uint8_t*, uint32_t len, uint32_t colIndex); - + template + inline void setBinaryField(const T* value, uint32_t width, uint32_t colIndex); + template + inline void setBinaryField(const T* value, uint32_t colIndex); + template + inline void setBinaryField_offset(const T* value, uint32_t width, uint32_t colIndex); // support VARBINARY // Add 2-byte length at the CHARSET_INFO*beginning of the field. NULL and zero length field are // treated the same, could use one of the length bit to distinguish these two cases. @@ -433,6 +445,15 @@ public: inline const uint8_t* getVarBinaryField(uint32_t& len, uint32_t colIndex) const; inline void setVarBinaryField(const uint8_t* val, uint32_t len, uint32_t colIndex); + //inline std::string getBinaryField(uint32_t colIndex) const; + template + inline T* getBinaryField(uint32_t colIndex) const; + // To simplify parameter type deduction. + template + inline T* getBinaryField(T* argtype, uint32_t colIndex) const; + template + inline T* getBinaryField_offset(uint32_t offset) const; + inline boost::shared_ptr getUserData(uint32_t colIndex) const; inline void setUserData(mcsv1sdk::mcsv1Context& context, boost::shared_ptr userData, @@ -440,6 +461,8 @@ public: uint64_t getNullValue(uint32_t colIndex) const; bool isNullValue(uint32_t colIndex) const; + template + inline bool isNullValue_offset(uint32_t offset) const; // when NULLs are pulled out via getIntField(), they come out with these values. // Ex: the 1-byte int null value is 0x80. When it gets cast to an int64_t @@ -456,6 +479,9 @@ public: // that's not string-table safe, this one is inline void copyField(Row& dest, uint32_t destIndex, uint32_t srcIndex) const; + template + inline void copyBinaryField(Row& dest, uint32_t destIndex, uint32_t srcIndex) const; + std::string toString() const; std::string toCSV() const; @@ -624,12 +650,12 @@ inline uint32_t Row::getCharsetNumber(uint32_t col) const inline bool Row::isCharType(uint32_t colIndex) const { - return execplan::isCharType(types[colIndex]); + return datatypes::isCharType(types[colIndex]); } inline bool Row::isUnsigned(uint32_t colIndex) const { - return execplan::isUnsigned(types[colIndex]); + return datatypes::isUnsigned(types[colIndex]); } inline bool Row::isShortString(uint32_t colIndex) const @@ -647,6 +673,12 @@ inline bool Row::inStringTable(uint32_t col) const return strings && getColumnWidth(col) >= sTableThreshold && !forceInline[col]; } +template +inline bool Row::equals(T* value, uint32_t colIndex) const +{ + return *reinterpret_cast(&data[offsets[colIndex]]) == *value; +} + template inline bool Row::equals(uint64_t val, uint32_t colIndex) const { @@ -664,7 +696,6 @@ inline bool Row::equals(uint64_t val, uint32_t colIndex) const case 8: return *((uint64_t*) &data[offsets[colIndex]]) == val; - default: idbassert(0); throw std::logic_error("Row::equals(): bad length."); @@ -675,6 +706,12 @@ inline bool Row::equals(long double val, uint32_t colIndex) const { return *((long double*) &data[offsets[colIndex]]) == val; } + +inline bool Row::equals(const int128_t& val, uint32_t colIndex) const +{ + return *((int128_t*) &data[offsets[colIndex]]) == val; +} + template inline uint64_t Row::getUintField(uint32_t colIndex) const { @@ -692,7 +729,6 @@ inline uint64_t Row::getUintField(uint32_t colIndex) const case 8: return *((uint64_t*) &data[offsets[colIndex]]); - default: idbassert(0); throw std::logic_error("Row::getUintField(): bad length."); @@ -711,7 +747,6 @@ inline uint64_t Row::getUintField(uint32_t colIndex) const case 4: return *((uint32_t*) &data[offsets[colIndex]]); - case 8: return *((uint64_t*) &data[offsets[colIndex]]); @@ -740,6 +775,7 @@ inline int64_t Row::getIntField(uint32_t colIndex) const return *((int64_t*) &data[offsets[colIndex]]); default: + std::cout << "Row::getIntField getColumnWidth(colIndex) " << getColumnWidth(colIndex) << std::endl; idbassert(0); throw std::logic_error("Row::getIntField(): bad length."); } @@ -784,6 +820,44 @@ inline uint32_t Row::getStringLength(uint32_t colIndex) const return strnlen((char*) &data[offsets[colIndex]], getColumnWidth(colIndex)); } +template +inline void Row::setBinaryField(const T* value, uint32_t width, uint32_t colIndex) +{ + memcpy(&data[offsets[colIndex]], value, width); +} + +template +inline void Row::setBinaryField(const T* value, uint32_t colIndex) +{ + *reinterpret_cast(&data[offsets[colIndex]]) = *value; +} + +template<> +inline void Row::setBinaryField(const int128_t* value, uint32_t colIndex) +{ + datatypes::TSInt128::assignPtrPtr(&data[offsets[colIndex]], value); +} + + +// This method !cannot! be applied to uint8_t* buffers. +template +inline void Row::setBinaryField_offset(const T* value, uint32_t width, uint32_t offset) +{ + *reinterpret_cast(&data[offset]) = *value; +} + +template<> +inline void Row::setBinaryField_offset(const uint8_t* value, uint32_t width, uint32_t offset) +{ + memcpy(&data[offset], value, width); +} + +template<> +inline void Row::setBinaryField_offset(const int128_t* value, uint32_t width, uint32_t offset) +{ + datatypes::TSInt128::assignPtrPtr(&data[offset], value); +} + inline void Row::setStringField(const uint8_t* strdata, uint32_t length, uint32_t colIndex) { uint64_t offset; @@ -817,6 +891,24 @@ inline std::string Row::getStringField(uint32_t colIndex) const strnlen((char*) &data[offsets[colIndex]], getColumnWidth(colIndex))); } +template +inline T* Row::getBinaryField(uint32_t colIndex) const +{ + return getBinaryField_offset(offsets[colIndex]); +} + +template +inline T* Row::getBinaryField(T* argtype, uint32_t colIndex) const +{ + return getBinaryField_offset(offsets[colIndex]); +} + +template +inline T* Row::getBinaryField_offset(uint32_t offset) const +{ + return reinterpret_cast(&data[offset]); +} + inline std::string Row::getVarBinaryStringField(uint32_t colIndex) const { if (inStringTable(colIndex)) @@ -880,6 +972,17 @@ inline long double Row::getLongDoubleField(uint32_t colIndex) const return *((long double*) &data[offsets[colIndex]]); } +inline void Row::getInt128Field(uint32_t colIndex, int128_t& x) const +{ + datatypes::TSInt128::assignPtrPtr(&x, &data[offsets[colIndex]]); +} + +inline datatypes::TSInt128 Row::getTSInt128Field(uint32_t colIndex) const +{ + const int128_t* ptr = getBinaryField(colIndex); + return datatypes::TSInt128(ptr); +} + inline uint64_t Row::getRid() const { return baseRid + *((uint16_t*) data); @@ -1076,7 +1179,7 @@ inline void Row::setFloatField(float val, uint32_t colIndex) *((float*) &data[offsets[colIndex]]) = val; } -inline void Row::setLongDoubleField(long double val, uint32_t colIndex) +inline void Row::setLongDoubleField(const long double& val, uint32_t colIndex) { uint8_t* p = &data[offsets[colIndex]]; *((long double*)p) = val; @@ -1087,6 +1190,11 @@ inline void Row::setLongDoubleField(long double val, uint32_t colIndex) } } +inline void Row::setInt128Field(const int128_t& val, uint32_t colIndex) +{ + setBinaryField(&val, colIndex); +} + inline void Row::setVarBinaryField(const std::string& val, uint32_t colIndex) { if (inStringTable(colIndex)) @@ -1140,16 +1248,36 @@ inline void Row::copyField(Row& out, uint32_t destIndex, uint32_t srcIndex) cons if (UNLIKELY(types[srcIndex] == execplan::CalpontSystemCatalog::VARBINARY || types[srcIndex] == execplan::CalpontSystemCatalog::BLOB || types[srcIndex] == execplan::CalpontSystemCatalog::TEXT)) + { out.setVarBinaryField(getVarBinaryStringField(srcIndex), destIndex); + } else if (UNLIKELY(isLongString(srcIndex))) + { out.setStringField(getStringPointer(srcIndex), getStringLength(srcIndex), destIndex); - //out.setStringField(getStringField(srcIndex), destIndex); + } else if (UNLIKELY(isShortString(srcIndex))) + { out.setUintField(getUintField(srcIndex), destIndex); + } else if (UNLIKELY(types[srcIndex] == execplan::CalpontSystemCatalog::LONGDOUBLE)) + { out.setLongDoubleField(getLongDoubleField(srcIndex), destIndex); + } + else if (UNLIKELY(datatypes::isWideDecimalType( + types[srcIndex], colWidths[srcIndex]))) + { + copyBinaryField(out, destIndex, srcIndex); + } else + { out.setIntField(getIntField(srcIndex), destIndex); + } +} + +template +inline void Row::copyBinaryField(Row& out, uint32_t destIndex, uint32_t srcIndex) const +{ + out.setBinaryField(getBinaryField(srcIndex), destIndex); } inline void Row::setRid(uint64_t rid) @@ -1582,12 +1710,12 @@ inline uint64_t RowGroup::getSizeWithStrings() const inline bool RowGroup::isCharType(uint32_t colIndex) const { - return execplan::isCharType(types[colIndex]); + return datatypes::isCharType(types[colIndex]); } inline bool RowGroup::isUnsigned(uint32_t colIndex) const { - return execplan::isUnsigned(types[colIndex]); + return datatypes::isUnsigned(types[colIndex]); } inline bool RowGroup::isShortString(uint32_t colIndex) const @@ -1774,16 +1902,30 @@ inline void copyRow(const Row& in, Row* out, uint32_t colCount) in.getColTypes()[i] == execplan::CalpontSystemCatalog::BLOB || in.getColTypes()[i] == execplan::CalpontSystemCatalog::TEXT || in.getColTypes()[i] == execplan::CalpontSystemCatalog::CLOB)) + { out->setVarBinaryField(in.getVarBinaryStringField(i), i); + } else if (UNLIKELY(in.isLongString(i))) - //out->setStringField(in.getStringField(i), i); + { out->setStringField(in.getStringPointer(i), in.getStringLength(i), i); + } else if (UNLIKELY(in.isShortString(i))) + { out->setUintField(in.getUintField(i), i); + } else if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::LONGDOUBLE)) + { out->setLongDoubleField(in.getLongDoubleField(i), i); + } + else if (UNLIKELY(datatypes::isWideDecimalType( + in.getColType(i), in.getColumnWidth(i)))) + { + in.copyBinaryField(*out, i, i); + } else + { out->setIntField(in.getIntField(i), i); + } } } diff --git a/utils/thrift/thrift/protocol/TCompactProtocol.tcc b/utils/thrift/thrift/protocol/TCompactProtocol.tcc index 62d6485d7..89c943a8c 100644 --- a/utils/thrift/thrift/protocol/TCompactProtocol.tcc +++ b/utils/thrift/thrift/protocol/TCompactProtocol.tcc @@ -808,6 +808,8 @@ TType TCompactProtocolT::getTType(int8_t type) { case detail::compact::CT_STRUCT: return T_STRUCT; default: + + cout << __FILE__<< __LINE__ << __func__<< endl; throw TException(std::string("don't know what type: ") + (char)type); } return T_STOP; diff --git a/utils/udfsdk/mcsv1_udaf.cpp b/utils/udfsdk/mcsv1_udaf.cpp index 326c7fee0..b9d7cdac1 100755 --- a/utils/udfsdk/mcsv1_udaf.cpp +++ b/utils/udfsdk/mcsv1_udaf.cpp @@ -18,6 +18,7 @@ #include #include #include +#include "mcs_basic_types.h" #include "mcsv1_udaf.h" #include "bytestream.h" #include "objectreader.h" @@ -151,7 +152,7 @@ const std::string mcsv1Context::toString() const std::ostringstream output; output << "mcsv1Context: " << getName() << std::endl; output << " RunFlags=" << fRunFlags << " ContextFlags=" << fContextFlags << std::endl; - output << " UserDataSize=" << fUserDataSize << " ResultType=" << colDataTypeToString(fResultType) << std::endl; + output << " UserDataSize=" << fUserDataSize << " ResultType=" << execplan::colDataTypeToString(fResultType) << std::endl; output << " Resultscale=" << fResultscale << " ResultPrecision=" << fResultPrecision << std::endl; output << " ErrorMsg=" << errorMsg << std::endl; output << " bInterrupted=" << bInterrupted << std::endl; @@ -278,6 +279,7 @@ const static_any::any& mcsv1_UDAF::shortTypeId((short)1); const static_any::any& mcsv1_UDAF::intTypeId((int)1); const static_any::any& mcsv1_UDAF::longTypeId((long)1); const static_any::any& mcsv1_UDAF::llTypeId((long long)1); +const static_any::any& mcsv1_UDAF::int128TypeId((int128_t)1); const static_any::any& mcsv1_UDAF::ucharTypeId((unsigned char)1); const static_any::any& mcsv1_UDAF::ushortTypeId((unsigned short)1); const static_any::any& mcsv1_UDAF::uintTypeId((unsigned int)1); diff --git a/utils/udfsdk/mcsv1_udaf.h b/utils/udfsdk/mcsv1_udaf.h index d7096c052..ff37457ef 100755 --- a/utils/udfsdk/mcsv1_udaf.h +++ b/utils/udfsdk/mcsv1_udaf.h @@ -638,6 +638,7 @@ protected: static const static_any::any& intTypeId; static const static_any::any& longTypeId; static const static_any::any& llTypeId; + static const static_any::any& int128TypeId; static const static_any::any& ucharTypeId; static const static_any::any& ushortTypeId; static const static_any::any& uintTypeId; @@ -1069,6 +1070,10 @@ inline T mcsv1_UDAF::convertAnyTo(static_any::any& valIn) { val = valIn.cast(); } + else if (valIn.compatible(int128TypeId)) + { + val = valIn.cast(); + } else { throw std::runtime_error("mcsv1_UDAF::convertAnyTo(): input param has unrecognized type"); diff --git a/utils/windowfunction/CMakeLists.txt b/utils/windowfunction/CMakeLists.txt index 636f72735..218a43803 100755 --- a/utils/windowfunction/CMakeLists.txt +++ b/utils/windowfunction/CMakeLists.txt @@ -29,9 +29,3 @@ add_library(windowfunction SHARED ${windowfunction_LIB_SRCS}) add_dependencies(windowfunction loggingcpp) install(TARGETS windowfunction DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) - -if (WITH_SORTING_COMPARATORS_UT) - add_executable(comparators_tests comparators-tests.cpp) - target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) - install(TARGETS comparators_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) -endif() diff --git a/utils/windowfunction/frameboundrange.cpp b/utils/windowfunction/frameboundrange.cpp index 21323f823..e2ae172e1 100644 --- a/utils/windowfunction/frameboundrange.cpp +++ b/utils/windowfunction/frameboundrange.cpp @@ -285,7 +285,6 @@ void FrameBoundExpressionRange::validate() case execplan::CalpontSystemCatalog::MEDINT: case execplan::CalpontSystemCatalog::INT: case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::DECIMAL: { int64_t tmp = this->fRow.getIntField(this->fIndex[1]); this->fIsZero = (tmp == 0); @@ -299,6 +298,50 @@ void FrameBoundExpressionRange::validate() break; } + case execplan::CalpontSystemCatalog::DECIMAL: + { + if (UNLIKELY(this->fRow.getColumnWidth(this->fIndex[1]) + < datatypes::MAXDECIMALWIDTH)) + { + int64_t tmp = this->fRow.getIntField(this->fIndex[1]); + this->fIsZero = (tmp == 0); + + if (tmp < 0) + { + invalid = true; + oss << ""; + } + } + else + { + datatypes::TSInt128 tmp = this->fRow.getTSInt128Field(this->fIndex[1]); + this->fIsZero = (tmp == 0); + + if (tmp < 0) + { + invalid = true; + oss << ""; + } + } + break; + } + + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (UNLIKELY(this->fRow.getColumnWidth(this->fIndex[1]) + < datatypes::MAXDECIMALWIDTH)) + { + uint64_t tmp = this->fRow.getUintField(this->fIndex[1]); + this->fIsZero = (tmp == 0); + } + else + { + datatypes::TSInt128 tmp = this->fRow.getTSInt128Field(this->fIndex[1]); + this->fIsZero = (tmp == 0); + } + break; + } + case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::UDOUBLE: { @@ -348,10 +391,9 @@ void FrameBoundExpressionRange::validate() case execplan::CalpontSystemCatalog::UMEDINT: case execplan::CalpontSystemCatalog::UINT: case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UDECIMAL: default: { - int64_t tmp = this->fRow.getIntField(this->fIndex[1]); + uint64_t tmp = this->fRow.getUintField(this->fIndex[1]); this->fIsZero = (tmp == 0); break; } diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index f92e1a5c0..93e2d468e 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -47,7 +47,7 @@ using namespace rowgroup; #include "idborderby.h" #include "joblisttypes.h" - +#include "mcs_decimal.h" #include "collation.h" // See agg_arg_charsets in sql_type.h to see conversion rules for @@ -166,6 +166,35 @@ int BigIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) return ret; } +int WideDecimalCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) +{ + l->row1().setData(r1); + l->row2().setData(r2); + + int ret = 0; + int128_t v1 = *(l->row1().getBinaryField_offset(keyColumnOffset)); + int128_t v2 = *(l->row2().getBinaryField_offset(keyColumnOffset)); + bool v1IsNull = v1 == datatypes::Decimal128Null; + bool v2IsNull = v2 == datatypes::Decimal128Null; + + if (v1IsNull || v2IsNull) + { + if (!v1IsNull && v2IsNull) + ret = fSpec.fNf; + else if (v1IsNull && !v2IsNull) + ret = -fSpec.fNf; + } + else + { + if (v1 > v2) + ret = fSpec.fAsc; + else if (v1 < v2) + ret = -fSpec.fAsc; + } + + return ret; +} + int UTinyIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) { l->row1().setData(r1); @@ -537,6 +566,7 @@ void CompareRule::revertRules() void CompareRule::compileRules(const std::vector& spec, const rowgroup::RowGroup& rg) { const vector& types = rg.getColTypes(); + const auto& offsets = rg.getOffsets(); for (vector::const_iterator i = spec.begin(); i != spec.end(); i++) { @@ -574,14 +604,16 @@ void CompareRule::compileRules(const std::vector& spec, const rowgr uint32_t len = rg.getColumnWidth(i->fIndex); switch (len) { + case datatypes::MAXDECIMALWIDTH: + c = new WideDecimalCompare(*i, offsets[i->fIndex]); break; + case datatypes::MAXLEGACYWIDTH: + c = new BigIntCompare(*i); break; case 1 : c = new TinyIntCompare(*i); break; case 2 : c = new SmallIntCompare(*i); break; case 4 : c = new IntCompare(*i); break; - default: - c = new BigIntCompare(*i); } fCompares.push_back(c); @@ -802,8 +834,6 @@ bool EqualCompData::operator()(Row::Pointer a, Row::Pointer b) case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::UTINYINT: case CalpontSystemCatalog::USMALLINT: case CalpontSystemCatalog::UMEDINT: @@ -819,6 +849,22 @@ bool EqualCompData::operator()(Row::Pointer a, Row::Pointer b) break; } + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + // equal compare. ignore sign and null + if (UNLIKELY(fRow1.getColumnWidth(*i) < datatypes::MAXDECIMALWIDTH)) + { + eq = (fRow1.getUintField(*i) == fRow2.getUintField(*i)); + } + else if (fRow1.getColumnWidth(*i) == datatypes::MAXDECIMALWIDTH) + { + eq = (*fRow1.getBinaryField(*i) == + *fRow2.getBinaryField(*i)); + } + break; + } + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: { diff --git a/utils/windowfunction/idborderby.h b/utils/windowfunction/idborderby.h index 98dce4b59..30f49c1de 100644 --- a/utils/windowfunction/idborderby.h +++ b/utils/windowfunction/idborderby.h @@ -143,6 +143,15 @@ public: int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); }; +class WideDecimalCompare : public Compare +{ + int keyColumnOffset; +public: + WideDecimalCompare(const IdbSortSpec& spec, int offset) : Compare(spec), keyColumnOffset(offset) { } + + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); +}; + // End of comparators for signed types // Comparators for unsigned types diff --git a/utils/windowfunction/wf_count.cpp b/utils/windowfunction/wf_count.cpp index 1e18c6a74..67bc82ed4 100644 --- a/utils/windowfunction/wf_count.cpp +++ b/utils/windowfunction/wf_count.cpp @@ -19,6 +19,7 @@ //#define NDEBUG +#include #include #include #include @@ -54,7 +55,7 @@ namespace windowfunction template -boost::shared_ptr WF_count::makeFunction(int id, const string& name, int ct) +boost::shared_ptr WF_count::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) { boost::shared_ptr func; @@ -68,6 +69,27 @@ boost::shared_ptr WF_count::makeFunction(int id, const st break; } + case CalpontSystemCatalog::BINARY: + { + std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + decltype(datatypes::MAXDECIMALWIDTH) width = + wc->functionParms()[0]->resultType().colWidth; + if (width < datatypes::MAXDECIMALWIDTH) + { + func.reset(new WF_count(id, name)); + } + else if (width == datatypes::MAXDECIMALWIDTH) + { + func.reset(new WF_count(id, name)); + } + break; + } default: { func.reset(new WF_count(id, name)); @@ -174,7 +196,7 @@ void WF_count::operator()(int64_t b, int64_t e, int64_t c) template -boost::shared_ptr WF_count::makeFunction(int, const string&, int); +boost::shared_ptr WF_count::makeFunction(int, const string&, int, WindowFunctionColumn*); } //namespace diff --git a/utils/windowfunction/wf_count.h b/utils/windowfunction/wf_count.h index cc4b63959..934489815 100644 --- a/utils/windowfunction/wf_count.h +++ b/utils/windowfunction/wf_count.h @@ -43,7 +43,7 @@ public: WindowFunctionType* clone() const; void resetData(); - static boost::shared_ptr makeFunction(int, const string&, int); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); protected: diff --git a/utils/windowfunction/wf_lead_lag.cpp b/utils/windowfunction/wf_lead_lag.cpp index cb15f3490..ff363151b 100644 --- a/utils/windowfunction/wf_lead_lag.cpp +++ b/utils/windowfunction/wf_lead_lag.cpp @@ -53,7 +53,7 @@ namespace windowfunction template -boost::shared_ptr WF_lead_lag::makeFunction(int id, const string& name, int ct) +boost::shared_ptr WF_lead_lag::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) { boost::shared_ptr func; @@ -64,7 +64,6 @@ boost::shared_ptr WF_lead_lag::makeFunction(int id, const case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: { func.reset(new WF_lead_lag(id, name)); break; @@ -75,7 +74,6 @@ boost::shared_ptr WF_lead_lag::makeFunction(int id, const case CalpontSystemCatalog::UMEDINT: case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: case CalpontSystemCatalog::TIMESTAMP: @@ -85,6 +83,24 @@ boost::shared_ptr WF_lead_lag::makeFunction(int id, const break; } + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + decltype(datatypes::MAXDECIMALWIDTH) width = + wc->functionParms()[0]->resultType().colWidth; + if (width < datatypes::MAXDECIMALWIDTH) + { + if (ct == CalpontSystemCatalog::UDECIMAL) + func.reset(new WF_lead_lag(id, name)); + else + func.reset(new WF_lead_lag(id, name)); + } + else if (width == datatypes::MAXDECIMALWIDTH) + { + func.reset(new WF_lead_lag(id, name)); + } + break; + } case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { @@ -167,12 +183,15 @@ void WF_lead_lag::parseParms(const std::vector& parms) // parms[3]: respect null | ignore null cc = dynamic_cast(parms[3].get()); - idbassert(cc != NULL); - bool isNull = false; // dummy, harded coded - fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); + if (cc != NULL) + { + bool isNull = false; // dummy. Return not used + fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); + } } + template void WF_lead_lag::operator()(int64_t b, int64_t e, int64_t c) { @@ -196,13 +215,7 @@ void WF_lead_lag::operator()(int64_t b, int64_t e, int64_t c) if (!fOffsetNull) { implicit2T(idx, tmp, 0); - - if (tmp > e) // prevent integer overflow - tmp = e + 1; - else if (tmp + e < 0) - tmp += e - 1; - - fOffset = (int64_t) tmp; + fOffset = round(tmp); fOffset *= fLead; } } @@ -293,10 +306,11 @@ void WF_lead_lag::operator()(int64_t b, int64_t e, int64_t c) template -boost::shared_ptr WF_lead_lag::makeFunction(int, const string&, int); +boost::shared_ptr WF_lead_lag::makeFunction(int, const string&, int, WindowFunctionColumn*); template void WF_lead_lag::parseParms(const std::vector&); template void WF_lead_lag::parseParms(const std::vector&); +template void WF_lead_lag::parseParms(const std::vector&); template void WF_lead_lag::parseParms(const std::vector&); template void WF_lead_lag::parseParms(const std::vector&); template void WF_lead_lag::parseParms(const std::vector&); diff --git a/utils/windowfunction/wf_lead_lag.h b/utils/windowfunction/wf_lead_lag.h index de14ca702..81b707ae2 100644 --- a/utils/windowfunction/wf_lead_lag.h +++ b/utils/windowfunction/wf_lead_lag.h @@ -43,7 +43,7 @@ public: void resetData(); void parseParms(const std::vector&); - static boost::shared_ptr makeFunction(int, const string&, int); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); protected: T fValue; diff --git a/utils/windowfunction/wf_min_max.cpp b/utils/windowfunction/wf_min_max.cpp index 4ae76e84d..036a393fc 100644 --- a/utils/windowfunction/wf_min_max.cpp +++ b/utils/windowfunction/wf_min_max.cpp @@ -52,7 +52,7 @@ namespace windowfunction template -boost::shared_ptr WF_min_max::makeFunction(int id, const string& name, int ct) +boost::shared_ptr WF_min_max::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) { boost::shared_ptr func; @@ -63,7 +63,6 @@ boost::shared_ptr WF_min_max::makeFunction(int id, const case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: { func.reset(new WF_min_max(id, name)); break; @@ -74,7 +73,6 @@ boost::shared_ptr WF_min_max::makeFunction(int id, const case CalpontSystemCatalog::UMEDINT: case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: case CalpontSystemCatalog::TIMESTAMP: @@ -84,6 +82,26 @@ boost::shared_ptr WF_min_max::makeFunction(int id, const break; } + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + decltype(datatypes::MAXDECIMALWIDTH) width = + wc->functionParms()[0]->resultType().colWidth; + + if (width < datatypes::MAXDECIMALWIDTH) + { + if (ct == CalpontSystemCatalog::UDECIMAL) + func.reset(new WF_min_max(id, name)); + else + func.reset(new WF_min_max(id, name)); + } + else + { + func.reset(new WF_min_max(id, name)); + } + break; + } + case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { @@ -173,7 +191,7 @@ void WF_min_max::operator()(int64_t b, int64_t e, int64_t c) template -boost::shared_ptr WF_min_max::makeFunction(int, const string&, int); +boost::shared_ptr WF_min_max::makeFunction(int, const string&, int, WindowFunctionColumn*); } //namespace diff --git a/utils/windowfunction/wf_min_max.h b/utils/windowfunction/wf_min_max.h index aac8d579c..ebe12df65 100644 --- a/utils/windowfunction/wf_min_max.h +++ b/utils/windowfunction/wf_min_max.h @@ -42,7 +42,7 @@ public: WindowFunctionType* clone() const; void resetData(); - static boost::shared_ptr makeFunction(int, const string&, int); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); protected: T fValue; diff --git a/utils/windowfunction/wf_nth_value.cpp b/utils/windowfunction/wf_nth_value.cpp index c736c0e92..d838c8ec3 100644 --- a/utils/windowfunction/wf_nth_value.cpp +++ b/utils/windowfunction/wf_nth_value.cpp @@ -53,7 +53,7 @@ namespace windowfunction template -boost::shared_ptr WF_nth_value::makeFunction(int id, const string& name, int ct) +boost::shared_ptr WF_nth_value::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) { boost::shared_ptr func; @@ -64,7 +64,6 @@ boost::shared_ptr WF_nth_value::makeFunction(int id, cons case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: { func.reset(new WF_nth_value(id, name)); break; @@ -75,7 +74,6 @@ boost::shared_ptr WF_nth_value::makeFunction(int id, cons case CalpontSystemCatalog::UMEDINT: case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: case CalpontSystemCatalog::TIMESTAMP: @@ -85,6 +83,25 @@ boost::shared_ptr WF_nth_value::makeFunction(int id, cons break; } + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + decltype(datatypes::MAXDECIMALWIDTH) width = + wc->functionParms()[0]->resultType().colWidth; + if (width < datatypes::MAXDECIMALWIDTH) + { + if (ct == CalpontSystemCatalog::UDECIMAL) + func.reset(new WF_nth_value(id, name)); + else + func.reset(new WF_nth_value(id, name)); + } + else if (width == datatypes::MAXDECIMALWIDTH) + { + func.reset(new WF_nth_value(id, name)); + } + break; + } + case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { @@ -191,23 +208,10 @@ void WF_nth_value::operator()(int64_t b, int64_t e, int64_t c) if (!fNthNull) { implicit2T(idx, tmp, 0); - - if (tmp <= 0) - { - ostringstream oss; - oss << tmp; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_ARG_OUT_OF_RANGE, - oss.str()), ERR_WF_ARG_OUT_OF_RANGE); - } - - if (tmp > e) // prevent integer overflow - tmp = e + 1; - - fNth = (int64_t) tmp; + fNth = round(tmp); } } - bool isNull = true; if ((!fNthNull) && ((b + fNth - 1) <= e)) @@ -232,7 +236,7 @@ void WF_nth_value::operator()(int64_t b, int64_t e, int64_t c) int64_t n = k + fNth - 1; - if (n <= e) + if (n <= e && n >= 0) { fRow.setData(getPointer(fRowData->at(n))); getValue(colIn, fValue); @@ -273,7 +277,7 @@ void WF_nth_value::operator()(int64_t b, int64_t e, int64_t c) template -boost::shared_ptr WF_nth_value::makeFunction(int, const string&, int); +boost::shared_ptr WF_nth_value::makeFunction(int, const string&, int, WindowFunctionColumn*); } //namespace diff --git a/utils/windowfunction/wf_nth_value.h b/utils/windowfunction/wf_nth_value.h index 313313210..8ea294b02 100644 --- a/utils/windowfunction/wf_nth_value.h +++ b/utils/windowfunction/wf_nth_value.h @@ -43,7 +43,7 @@ public: void resetData(); void parseParms(const std::vector&); - static boost::shared_ptr makeFunction(int, const string&, int); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); protected: T fValue; diff --git a/utils/windowfunction/wf_ntile.cpp b/utils/windowfunction/wf_ntile.cpp index 3bf2a3c24..716b10a2c 100644 --- a/utils/windowfunction/wf_ntile.cpp +++ b/utils/windowfunction/wf_ntile.cpp @@ -54,7 +54,7 @@ namespace windowfunction { -boost::shared_ptr WF_ntile::makeFunction(int id, const string& name, int ct) +boost::shared_ptr WF_ntile::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) { boost::shared_ptr func(new WF_ntile(id, name)); return func; @@ -118,10 +118,7 @@ void WF_ntile::operator()(int64_t b, int64_t e, int64_t c) oss.str()), ERR_WF_ARG_OUT_OF_RANGE); } - if (tmp > e) // prevent integer overflow - tmp = e + 1; - - fNtile = (int64_t) tmp; + fNtile = round(tmp); } } diff --git a/utils/windowfunction/wf_ntile.h b/utils/windowfunction/wf_ntile.h index 783ee1983..45d5ba64a 100644 --- a/utils/windowfunction/wf_ntile.h +++ b/utils/windowfunction/wf_ntile.h @@ -43,7 +43,7 @@ public: void resetData(); void parseParms(const std::vector&); - static boost::shared_ptr makeFunction(int, const string&, int); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); protected: diff --git a/utils/windowfunction/wf_percentile.cpp b/utils/windowfunction/wf_percentile.cpp index 0c93b1434..d23f9e712 100644 --- a/utils/windowfunction/wf_percentile.cpp +++ b/utils/windowfunction/wf_percentile.cpp @@ -56,7 +56,7 @@ namespace windowfunction { template -boost::shared_ptr WF_percentile::makeFunction(int id, const string& name, int ct) +boost::shared_ptr WF_percentile::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) { boost::shared_ptr func; @@ -69,7 +69,6 @@ boost::shared_ptr WF_percentile::makeFunction(int id, con case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: { func.reset(new WF_percentile(id, name)); break; @@ -80,7 +79,6 @@ boost::shared_ptr WF_percentile::makeFunction(int id, con case CalpontSystemCatalog::UMEDINT: case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: case CalpontSystemCatalog::TIMESTAMP: @@ -90,6 +88,26 @@ boost::shared_ptr WF_percentile::makeFunction(int id, con break; } + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + decltype(datatypes::MAXDECIMALWIDTH) width = + wc->resultType().colWidth; + + if (width < datatypes::MAXDECIMALWIDTH) + { + if (ct == CalpontSystemCatalog::UDECIMAL) + func.reset(new WF_percentile(id, name)); + else + func.reset(new WF_percentile(id, name)); + } + else if (width == datatypes::MAXDECIMALWIDTH) + { + func.reset(new WF_percentile(id, name)); + } + break; + } + case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { @@ -371,7 +389,7 @@ void WF_percentile::operator()(int64_t b, int64_t e, int64_t c) template -boost::shared_ptr WF_percentile::makeFunction(int, const string&, int); +boost::shared_ptr WF_percentile::makeFunction(int, const string&, int, WindowFunctionColumn*); } //namespace diff --git a/utils/windowfunction/wf_percentile.h b/utils/windowfunction/wf_percentile.h index afc2f69b3..ae2c8db50 100644 --- a/utils/windowfunction/wf_percentile.h +++ b/utils/windowfunction/wf_percentile.h @@ -44,7 +44,7 @@ public: void resetData(); void parseParms(const std::vector&); - static boost::shared_ptr makeFunction(int, const string&, int); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); protected: diff --git a/utils/windowfunction/wf_ranking.cpp b/utils/windowfunction/wf_ranking.cpp index 4e9765630..818820ee5 100644 --- a/utils/windowfunction/wf_ranking.cpp +++ b/utils/windowfunction/wf_ranking.cpp @@ -53,7 +53,7 @@ namespace windowfunction { -boost::shared_ptr WF_ranking::makeFunction(int id, const string& name, int ct) +boost::shared_ptr WF_ranking::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) { boost::shared_ptr func(new WF_ranking(id, name)); return func; diff --git a/utils/windowfunction/wf_ranking.h b/utils/windowfunction/wf_ranking.h index c8be23ba4..ce95bd122 100644 --- a/utils/windowfunction/wf_ranking.h +++ b/utils/windowfunction/wf_ranking.h @@ -42,7 +42,7 @@ public: WindowFunctionType* clone() const; void resetData(); - static boost::shared_ptr makeFunction(int, const string&, int); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); protected: diff --git a/utils/windowfunction/wf_row_number.cpp b/utils/windowfunction/wf_row_number.cpp index cac718317..048422614 100644 --- a/utils/windowfunction/wf_row_number.cpp +++ b/utils/windowfunction/wf_row_number.cpp @@ -53,7 +53,7 @@ namespace windowfunction { -boost::shared_ptr WF_row_number::makeFunction(int id, const string& name, int ct) +boost::shared_ptr WF_row_number::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) { boost::shared_ptr func(new WF_row_number(id, name)); return func; diff --git a/utils/windowfunction/wf_row_number.h b/utils/windowfunction/wf_row_number.h index 65a1b6618..4b6a61c08 100644 --- a/utils/windowfunction/wf_row_number.h +++ b/utils/windowfunction/wf_row_number.h @@ -42,7 +42,7 @@ public: WindowFunctionType* clone() const; void resetData(); - static boost::shared_ptr makeFunction(int, const string&, int); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); protected: diff --git a/utils/windowfunction/wf_stats.cpp b/utils/windowfunction/wf_stats.cpp index 8b06eda61..d736704f8 100644 --- a/utils/windowfunction/wf_stats.cpp +++ b/utils/windowfunction/wf_stats.cpp @@ -55,7 +55,7 @@ namespace windowfunction template -boost::shared_ptr WF_stats::makeFunction(int id, const string& name, int ct) +boost::shared_ptr WF_stats::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) { boost::shared_ptr func; @@ -66,7 +66,6 @@ boost::shared_ptr WF_stats::makeFunction(int id, const st case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: { func.reset(new WF_stats(id, name)); break; @@ -77,12 +76,30 @@ boost::shared_ptr WF_stats::makeFunction(int id, const st case CalpontSystemCatalog::UMEDINT: case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: { func.reset(new WF_stats(id, name)); break; } + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + decltype(datatypes::MAXDECIMALWIDTH) width = + wc->functionParms()[0]->resultType().colWidth; + if (width < datatypes::MAXDECIMALWIDTH) + { + if (ct == CalpontSystemCatalog::UDECIMAL) + func.reset(new WF_stats(id, name)); + else + func.reset(new WF_stats(id, name)); + } + else if (width == datatypes::MAXDECIMALWIDTH) + { + func.reset(new WF_stats(id, name)); + } + break; + } + case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { @@ -177,17 +194,19 @@ void WF_stats::operator()(int64_t b, int64_t e, int64_t c) { int scale = fRow.getScale(colIn); long double factor = pow(10.0, scale); + long double ldSum1 = fSum1; + long double ldSum2 = fSum2; // adjust the scale if necessary if (scale != 0 && cdt != CalpontSystemCatalog::LONGDOUBLE) { - fSum1 /= factor; - fSum2 /= factor * factor; + ldSum1 /= factor; + ldSum2 /= factor * factor; } - long double stat = fSum1 * fSum1 / fCount; - stat = fSum2 - stat; + long double stat = ldSum1 * ldSum1 / fCount; + stat = ldSum2 - stat; if (fFunctionId == WF__STDDEV_POP) stat = sqrt(stat / fCount); @@ -220,7 +239,7 @@ void WF_stats::operator()(int64_t b, int64_t e, int64_t c) template -boost::shared_ptr WF_stats::makeFunction(int, const string&, int); +boost::shared_ptr WF_stats::makeFunction(int, const string&, int, WindowFunctionColumn*); } //namespace diff --git a/utils/windowfunction/wf_stats.h b/utils/windowfunction/wf_stats.h index 381920482..6bff926a6 100644 --- a/utils/windowfunction/wf_stats.h +++ b/utils/windowfunction/wf_stats.h @@ -42,7 +42,7 @@ public: WindowFunctionType* clone() const; void resetData(); - static boost::shared_ptr makeFunction(int, const string&, int); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); protected: long double fSum1; diff --git a/utils/windowfunction/wf_sum_avg.cpp b/utils/windowfunction/wf_sum_avg.cpp index d0d5f46f7..5a0fbbfc0 100644 --- a/utils/windowfunction/wf_sum_avg.cpp +++ b/utils/windowfunction/wf_sum_avg.cpp @@ -1,4 +1,4 @@ -/* Copyright (C) 2014 InfiniDB, Inc. +/* CopyrighT (C) 2014 InfiniDB, Inc. Copyright (c) 2019 MariaDB Corporation This program is free software; you can redistribute it and/or @@ -50,112 +50,109 @@ using namespace joblist; #include "wf_sum_avg.h" -#if 0 -namespace -{ - -template -void checkSumLimit(T sum, T val) -{ -} - - -template<> -void checkSumLimit(int64_t sum, int64_t val) -{ - if (((sum >= 0) && ((numeric_limits::max() - sum) < val)) || - ((sum < 0) && ((numeric_limits::min() - sum) > val))) - { - string errStr = "SUM(int):"; - - ostringstream oss; - oss << sum << "+" << val; - - if (sum > 0) - oss << " > " << numeric_limits::max(); - else - oss << " < " << numeric_limits::min(); - - errStr += oss.str(); - - errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_OVERFLOW, errStr); - cerr << errStr << endl; - throw IDBExcept(errStr, ERR_WF_OVERFLOW); - } -} - - -template<> -void checkSumLimit(uint64_t sum, uint64_t val) -{ - if ((sum >= 0) && ((numeric_limits::max() - sum) < val)) - { - string errStr = "SUM(unsigned):"; - - ostringstream oss; - oss << sum << "+" << val << " > " << numeric_limits::max(); - errStr += oss.str(); - - errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_OVERFLOW, errStr); - cerr << errStr << endl; - throw IDBExcept(errStr, ERR_WF_OVERFLOW); - } -} - - -template<> -long double calculateAvg(long double sum, uint64_t count, int s) -{ - return sum / count; -} - -long double avgWithLimit(long double sum, uint64_t count, int scale, long double u, long double l) -{ - long double factor = pow(10.0, scale); - long double avg = sum / count; - avg *= factor; - avg += (avg < 0) ? (-0.5) : (0.5); - - if (avg > u || avg < l) - { - string errStr = string("AVG") + (l < 0 ? "(int):" : "(unsign)"); - ostringstream oss; - oss << avg; - errStr += oss.str(); - - errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_OVERFLOW, errStr); - cerr << errStr << endl; - throw IDBExcept(errStr, ERR_WF_OVERFLOW); - } - - return avg; -} - - -template<> -int64_t calculateAvg(int64_t sum, uint64_t count, int scale) -{ - int64_t t = (int64_t) avgWithLimit(sum, count, scale, - numeric_limits::max(), numeric_limits::min()); - return t; -} - - -template<> -uint64_t calculateAvg(uint64_t sum, uint64_t count, int scale) -{ - uint64_t t = (uint64_t) avgWithLimit(sum, count, scale, numeric_limits::max(), 0); - return t; -} - -} -#endif - namespace windowfunction { -template -boost::shared_ptr WF_sum_avg::makeFunction(int id, const string& name, int ct) +template +inline void WF_sum_avg::checkSumLimit(const T_IN& val, + const T_OUT& sum) +{ } + +template<> +inline void WF_sum_avg::checkSumLimit(const int128_t& val, + const int128_t& sum) +{ + datatypes::AdditionOverflowCheck ofCheckOp; + ofCheckOp(sum, val); +} + +template<> +inline void WF_sum_avg::checkSumLimit(const long double& val, + const long double& sum) +{ } + +template<> +inline void WF_sum_avg::checkSumLimit(const float&, + const long double&) +{ } + +template<> +inline void WF_sum_avg::checkSumLimit(const long&, + const long double&) +{ } + +template<> +inline void WF_sum_avg::checkSumLimit(const unsigned long&, + const long double&) +{ } +template<> +inline void WF_sum_avg::checkSumLimit(const double&, + const long double&) +{ } + +template<> +void WF_sum_avg::checkSumLimit(const int128_t&, const int128_t&); +template<> +void WF_sum_avg::checkSumLimit(const long double& val, const long double&); +template<> +void WF_sum_avg::checkSumLimit(const float&, const long double&); +template<> +void WF_sum_avg::checkSumLimit(const long&, const long double&); +template<> +void WF_sum_avg::checkSumLimit(const unsigned long&, + const long double&); +template<> +void WF_sum_avg::checkSumLimit(const double&, const long double&); + +template +int128_t WF_sum_avg::calculateAvg(const int128_t& sum, + const uint64_t count, + const int scale) +{ + int128_t avg = 0; + int128_t factor; + datatypes::getScaleDivisor(factor, scale); + if (scale > 0) + { + if ((sum * factor) / factor == sum) + { + avg = sum * factor; + avg /= count; + } + else + { + // scale won't fit before divide, we're gonna lose precision. + avg = sum / count; + if ((avg * factor) / factor != avg) // Still won't fit + { + string errStr = string("AVG(int)"); + errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_OVERFLOW, errStr); + cerr << errStr << endl; + throw IDBExcept(errStr, ERR_WF_OVERFLOW); + } + avg *= factor; + } + } + else + { + avg = sum / count; + } + + avg += (avg < 0) ? (-0.5) : (0.5); + return avg; +} + +template +inline long double WF_sum_avg::calculateAvg(const long double& sum, + const uint64_t count, + const int scale) +{ + return sum / count; +} + +// For the static function makeFunction, the template parameters are ignored +template +boost::shared_ptr WF_sum_avg::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) { boost::shared_ptr func; switch (ct) @@ -165,9 +162,9 @@ boost::shared_ptr WF_sum_avg::makeFunction(int id, const case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: { - func.reset(new WF_sum_avg(id, name)); + // Look into using int128_t instead of long double + func.reset(new WF_sum_avg(id, name)); break; } @@ -176,29 +173,48 @@ boost::shared_ptr WF_sum_avg::makeFunction(int id, const case CalpontSystemCatalog::UMEDINT: case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: { - func.reset(new WF_sum_avg(id, name)); + func.reset(new WF_sum_avg(id, name)); break; } + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + decltype(datatypes::MAXDECIMALWIDTH) width = + wc->functionParms()[0]->resultType().colWidth; + + if (width < datatypes::MAXDECIMALWIDTH) + { + if (ct == CalpontSystemCatalog::UDECIMAL) + func.reset(new WF_sum_avg(id, name)); + else + func.reset(new WF_sum_avg(id, name)); + } + else if (width == datatypes::MAXDECIMALWIDTH) + { + func.reset(new WF_sum_avg(id, name)); + } + break; + } + case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { - func.reset(new WF_sum_avg(id, name)); + func.reset(new WF_sum_avg(id, name)); break; } case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: { - func.reset(new WF_sum_avg(id, name)); + func.reset(new WF_sum_avg(id, name)); break; } case CalpontSystemCatalog::LONGDOUBLE: { - func.reset(new WF_sum_avg(id, name)); + func.reset(new WF_sum_avg(id, name)); break; } default: @@ -216,15 +232,15 @@ boost::shared_ptr WF_sum_avg::makeFunction(int id, const } -template -WindowFunctionType* WF_sum_avg::clone() const +template +WindowFunctionType* WF_sum_avg::clone() const { - return new WF_sum_avg(*this); + return new WF_sum_avg(*this); } -template -void WF_sum_avg::resetData() +template +void WF_sum_avg::resetData() { fAvg = 0; fSum = 0; @@ -235,8 +251,8 @@ void WF_sum_avg::resetData() } -template -void WF_sum_avg::operator()(int64_t b, int64_t e, int64_t c) +template +void WF_sum_avg::operator()(int64_t b, int64_t e, int64_t c) { uint64_t colOut = fFieldIndex[0]; @@ -251,8 +267,7 @@ void WF_sum_avg::operator()(int64_t b, int64_t e, int64_t c) e = c; uint64_t colIn = fFieldIndex[1]; - double scale = fRow.getScale(colIn); - + int scale = fRow.getScale(colOut) - fRow.getScale(colIn); for (int64_t i = b; i <= e; i++) { if (i % 1000 == 0 && fStep->cancelled()) @@ -263,34 +278,27 @@ void WF_sum_avg::operator()(int64_t b, int64_t e, int64_t c) if (fRow.isNullValue(colIn) == true) continue; - T valIn; CDT cdt; - getValue(colIn, valIn, &cdt); -// checkSumLimit(fSum, valIn); + getValue(colIn, fVal, &cdt); - if ((!fDistinct) || (fSet.find(valIn) == fSet.end())) + if ((!fDistinct) || (fSet.find(fVal) == fSet.end())) { - long double val = valIn; - if (scale && - cdt != CalpontSystemCatalog::LONGDOUBLE) - { - val /= pow(10.0, scale); - } - fSum += val; + checkSumLimit(fVal, fSum); + fSum += (T_OUT)fVal; fCount++; if (fDistinct) - fSet.insert(valIn); + fSet.insert(fVal); } } if ((fCount > 0) && (fFunctionId == WF__AVG || fFunctionId == WF__AVG_DISTINCT)) { - fAvg = fSum / fCount; + fAvg = calculateAvg(fSum, fCount, scale); } } - long double* v = NULL; + T_OUT* v = NULL; if (fCount > 0) { @@ -307,7 +315,7 @@ void WF_sum_avg::operator()(int64_t b, int64_t e, int64_t c) template -boost::shared_ptr WF_sum_avg::makeFunction(int, const string&, int); +boost::shared_ptr WF_sum_avg::makeFunction(int, const string&, int, WindowFunctionColumn*); } //namespace diff --git a/utils/windowfunction/wf_sum_avg.h b/utils/windowfunction/wf_sum_avg.h index 48d46453b..0e2a4ffe2 100644 --- a/utils/windowfunction/wf_sum_avg.h +++ b/utils/windowfunction/wf_sum_avg.h @@ -29,8 +29,9 @@ namespace windowfunction { - -template +// T_IN is the data type of the input values. +// T_OUT is the data type we are using for output and internal values +template class WF_sum_avg : public WindowFunctionType { public: @@ -45,16 +46,21 @@ public: WindowFunctionType* clone() const; void resetData(); - static boost::shared_ptr makeFunction(int, const string&, int); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); protected: - long double fAvg; - long double fSum; + T_IN fVal; + T_OUT fAvg; + T_OUT fSum; uint64_t fCount; bool fDistinct; - std::set fSet; -}; + std::set fSet; + void checkSumLimit(const T_IN& val, const T_OUT& sum); + + int128_t calculateAvg(const int128_t& sum, const uint64_t count, const int scale); + long double calculateAvg(const long double& sum, const uint64_t count, const int scale); +}; } // namespace diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index 2b8d7a4a9..b68173850 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -19,6 +19,7 @@ //#define NDEBUG +#include #include #include #include @@ -55,7 +56,7 @@ using namespace joblist; namespace windowfunction { -boost::shared_ptr WF_udaf::makeFunction(int id, const string& name, int ct, mcsv1sdk::mcsv1Context& context) +boost::shared_ptr WF_udaf::makeFunction(int id, const string& name, int ct, mcsv1sdk::mcsv1Context& context, WindowFunctionColumn* wc) { boost::shared_ptr func; @@ -490,7 +491,9 @@ bool WF_udaf::dropValues(int64_t b, int64_t e) datum.columnData = valIn; break; } - + case CalpontSystemCatalog::BINARY: + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + //fallthrough default: { string errStr = "(" + colType2String[(int)datum.dataType] + ")"; @@ -542,6 +545,7 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, static const static_any::any& intTypeId = (int)1; static const static_any::any& longTypeId = (long)1; static const static_any::any& llTypeId = (long long)1; + static const static_any::any& int128TypeId = (int128_t)1; static const static_any::any& ucharTypeId = (unsigned char)1; static const static_any::any& ushortTypeId = (unsigned short)1; static const static_any::any& uintTypeId = (unsigned int)1; @@ -560,6 +564,7 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, // it to whatever they said to return. int64_t intOut = 0; uint64_t uintOut = 0; + int128_t int128Out = 0; float floatOut = 0.0; double doubleOut = 0.0; long double longdoubleOut = 0.0; @@ -649,6 +654,15 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, intOut = (int64_t)doubleOut; oss << doubleOut; } + else if (valOut.compatible(int128TypeId)) + { + int128Out = valOut.cast(); + uintOut = intOut = int128Out; // may truncate + floatOut = int128Out; + doubleOut = int128Out; + longdoubleOut = int128Out; + oss << longdoubleOut; + } if (valOut.compatible(strTypeId)) { @@ -754,7 +768,9 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, setValue(colDataType, b, e, c, &strOut); } break; - + case CalpontSystemCatalog::BINARY: + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + //fallthrough default: { std::ostringstream errmsg; @@ -898,33 +914,66 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - int64_t valIn; - - if (cc) + if (fRow.getColumnWidth(colIn) < 16) { - valIn = cc->getDecimalVal(fRow, isNull).value; + int64_t valIn; + + if (cc) + { + valIn = cc->getDecimalVal(fRow, isNull).value; + } + else + { + getValue(colIn, valIn); + } + + // Check for distinct, if turned on. + // Currently, distinct only works on the first parameter. + if (k == 0 && fDistinct) + { + std::pair val = make_pair(valIn, 1); + std::pair distinct; + distinct = fDistinctMap.insert(val); + if (distinct.second == false) + { + ++(*distinct.first).second; + bSkipIt = true; + continue; + } + } + + datum.columnData = valIn; } else { - getValue(colIn, valIn); - } + int128_t valIn; - // Check for distinct, if turned on. - // Currently, distinct only works on the first parameter. - if (k == 0 && fDistinct) - { - std::pair val = make_pair(valIn, 1); - std::pair distinct; - distinct = fDistinctMap.insert(val); - if (distinct.second == false) + if (cc) { - ++(*distinct.first).second; - bSkipIt = true; - continue; + valIn = cc->getDecimalVal(fRow, isNull).s128Value; + } + else + { + getValue(colIn, valIn); } - } - datum.columnData = valIn; + // Check for distinct, if turned on. + // Currently, distinct only works on the first parameter. + if (k == 0 && fDistinct) + { + std::pair val = make_pair(valIn, 1); + std::pair distinct; + distinct = fDistinctMap.insert(val); + if (distinct.second == false) + { + ++(*distinct.first).second; + bSkipIt = true; + continue; + } + } + + datum.columnData = valIn; + } break; } @@ -1101,7 +1150,9 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) datum.columnData = valIn; break; } - + case CalpontSystemCatalog::BINARY: + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + //fallthrough default: { string errStr = "(" + colType2String[(int)datum.dataType] + ")"; @@ -1150,7 +1201,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) fPrev = c; } -boost::shared_ptr WF_udaf::makeFunction(int id, const string& name, int ct, mcsv1sdk::mcsv1Context& context); +boost::shared_ptr WF_udaf::makeFunction(int id, const string& name, int ct, mcsv1sdk::mcsv1Context& context, WindowFunctionColumn*); } //namespace // vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_udaf.h b/utils/windowfunction/wf_udaf.h index 38515285f..e87b39759 100644 --- a/utils/windowfunction/wf_udaf.h +++ b/utils/windowfunction/wf_udaf.h @@ -104,7 +104,7 @@ protected: public: static boost::shared_ptr makeFunction(int id, const string& name, - int ct, mcsv1sdk::mcsv1Context& context); + int ct, mcsv1sdk::mcsv1Context& context, WindowFunctionColumn* wc); }; diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index cea636402..d5bb92047 100644 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -19,6 +19,7 @@ */ //#define NDEBUG +#include #include #include #include @@ -39,7 +40,6 @@ using namespace logging; using namespace ordering; #include "calpontsystemcatalog.h" -#include "dataconvert.h" // int64_t IDB_pow[19] using namespace execplan; #include "windowfunctionstep.h" @@ -149,67 +149,67 @@ WindowFunctionType::makeWindowFunction(const string& name, int ct, WindowFunctio int functionId = windowFunctionId[algorithm::to_upper_copy(name)]; // The template parameters here are dummies to execute the static makeFunction - // which sets the real type based on ct. + // which sets the real types based on ct. switch (functionId) { case WF__COUNT_ASTERISK: case WF__COUNT: case WF__COUNT_DISTINCT: - af = WF_count::makeFunction(functionId, name, ct); + af = WF_count::makeFunction(functionId, name, ct, wc); break; case WF__MIN: case WF__MAX: - af = WF_min_max::makeFunction(functionId, name, ct); + af = WF_min_max::makeFunction(functionId, name, ct, wc); break; case WF__SUM: case WF__AVG: case WF__SUM_DISTINCT: case WF__AVG_DISTINCT: - af = WF_sum_avg::makeFunction(functionId, name, ct); + af = WF_sum_avg::makeFunction(functionId, name, ct, wc); break; case WF__STDDEV_POP: case WF__STDDEV_SAMP: case WF__VAR_POP: case WF__VAR_SAMP: - af = WF_stats::makeFunction(functionId, name, ct); + af = WF_stats::makeFunction(functionId, name, ct, wc); break; case WF__ROW_NUMBER: - af = WF_row_number::makeFunction(functionId, name, ct); + af = WF_row_number::makeFunction(functionId, name, ct, wc); break; case WF__RANK: case WF__DENSE_RANK: case WF__PERCENT_RANK: case WF__CUME_DIST: - af = WF_ranking::makeFunction(functionId, name, ct); + af = WF_ranking::makeFunction(functionId, name, ct, wc); break; case WF__FIRST_VALUE: case WF__LAST_VALUE: case WF__NTH_VALUE: - af = WF_nth_value::makeFunction(functionId, name, ct); + af = WF_nth_value::makeFunction(functionId, name, ct, wc); break; case WF__LEAD: case WF__LAG: - af = WF_lead_lag::makeFunction(functionId, name, ct); + af = WF_lead_lag::makeFunction(functionId, name, ct, wc); break; case WF__NTILE: - af = WF_ntile::makeFunction(functionId, name, ct); + af = WF_ntile::makeFunction(functionId, name, ct, wc); break; case WF__PERCENTILE_CONT: case WF__PERCENTILE_DISC: - af = WF_percentile::makeFunction(functionId, name, ct); + af = WF_percentile::makeFunction(functionId, name, ct, wc); break; case WF__UDAF: - af = WF_udaf::makeFunction(functionId, name, ct, wc->getUDAFContext()); + af = WF_udaf::makeFunction(functionId, name, ct, wc->getUDAFContext(), wc); break; case WF__REGR_SLOPE: @@ -228,7 +228,7 @@ WindowFunctionType::makeWindowFunction(const string& name, int ct, WindowFunctio break; } - // Copy the only the constant parameter pointers + // Copy only the constant parameter pointers af->constParms(wc->functionParms()); return af; @@ -307,6 +307,16 @@ template<> void WindowFunctionType::getValue(uint64_t i, string& t, CDT* // By not setting cdt, we let it default to the column's type } +template<> void WindowFunctionType::getValue(uint64_t i, int128_t& t, CDT* cdt) +{ + fRow.getInt128Field(i, t); + + if (cdt) + { + *cdt = execplan::CalpontSystemCatalog::DECIMAL; + } +} + template void WindowFunctionType::setValue(uint64_t i, T& t) { } @@ -336,6 +346,11 @@ template<> void WindowFunctionType::setValue(uint64_t i, long doubl fRow.setLongDoubleField(t, i); } +template<> void WindowFunctionType::setValue(uint64_t i, int128_t& t) +{ + fRow.setInt128Field(t, i); +} + template<> void WindowFunctionType::setValue(uint64_t i, string& t) { fRow.setStringField(t, i); @@ -388,7 +403,6 @@ void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, T* v) case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: { int64_t iv = *v; setValue(i, iv); @@ -400,13 +414,42 @@ void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, T* v) case CalpontSystemCatalog::UMEDINT: case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: { uint64_t uv = *v; setValue(i, uv); break; } + case CalpontSystemCatalog::DECIMAL: + { + if (sizeof(T) == 8) + { + int64_t iv = *v; + setValue(i, iv); + } + else + { + int128_t iv = *v; + setValue(i, iv); + } + break; + } + + case CalpontSystemCatalog::UDECIMAL: + { + if (sizeof(T) == 8) + { + uint64_t iv = *v; + setValue(i, iv); + } + else + { + int128_t iv = *v; + setValue(i, iv); + } + break; + } + case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { @@ -442,7 +485,6 @@ template void WindowFunctionType::implicit2T(uint64_t i, T& t, int s) { int ct = fRow.getColType(i); - int pw = 0; switch (ct) { @@ -451,16 +493,8 @@ void WindowFunctionType::implicit2T(uint64_t i, T& t, int s) case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: { t = (T) fRow.getIntField(i); - pw = s - fRow.getScale(i); // pw is difference of scales, will be in [-18, 18] - - if (pw > 0) - t *= IDB_pow[pw]; - else if (pw < 0) - t /= IDB_pow[-pw]; - break; } @@ -469,48 +503,48 @@ void WindowFunctionType::implicit2T(uint64_t i, T& t, int s) case CalpontSystemCatalog::UMEDINT: case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: { t = (T) fRow.getUintField(i); - pw = s - fRow.getScale(i); // pw is difference of scales, will be in [-18, 18] + break; + } - if (pw > 0) - t *= IDB_pow[pw]; - else if (pw < 0) - t /= IDB_pow[-pw]; + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + decltype(datatypes::MAXDECIMALWIDTH) width = + fRow.getColumnWidth(i);; + if (width < datatypes::MAXDECIMALWIDTH) + { + t = (ct == execplan::CalpontSystemCatalog::DECIMAL) ? + (T) fRow.getIntField(i) : + (T) fRow.getUintField(i); + } + else if (width == datatypes::MAXDECIMALWIDTH) + { + datatypes::TSInt128::assignPtrPtr(&t, + fRow.getBinaryField(i)); + } break; } case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { - if (s == 0) - t = (T) fRow.getDoubleField(i); - else - t = (T) (fRow.getDoubleField(i) * IDB_pow[s]); // s is scale, [0, 18] - + t = (T) fRow.getDoubleField(i); break; } case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: { - if (s == 0) - t = (T) fRow.getFloatField(i); - else - t = (T) (fRow.getFloatField(i) * IDB_pow[s]); // s is scale, [0, 18] - + t = (T) fRow.getFloatField(i); break; } case CalpontSystemCatalog::LONGDOUBLE: { - if (s == 0) - t = (T) fRow.getLongDoubleField(i); - else - t = (T) (fRow.getLongDoubleField(i) * IDB_pow[s]); // s is scale, [0, 18] - + t = (T) fRow.getLongDoubleField(i); break; } @@ -526,6 +560,14 @@ void WindowFunctionType::implicit2T(uint64_t i, T& t, int s) break; } } + + T divisor = 1; + s -= fRow.getScale(i); // we scale only the difference of scales + datatypes::getScaleDivisor(divisor, abs(s)); + if (s > 0) + t *= divisor; + else if (s < 0) + t /= divisor; } template<> @@ -551,6 +593,12 @@ void WindowFunctionType::getConstValue(ConstantColumn* cc, uint64_t& t t = cc->getUintVal(fRow, b); } +template<> +void WindowFunctionType::getConstValue(ConstantColumn* cc, int128_t& t, bool& b) +{ + t = cc->getDecimalVal(fRow, b).s128Value; +} + template<> void WindowFunctionType::getConstValue(ConstantColumn* cc, double& t, bool& b) { @@ -580,12 +628,13 @@ template void WindowFunctionType::implicit2T(uint64_t, uint64_t&, int) template void WindowFunctionType::implicit2T(uint64_t, float&, int); template void WindowFunctionType::implicit2T(uint64_t, double&, int); template void WindowFunctionType::implicit2T(uint64_t, long double&, int); - +template void WindowFunctionType::implicit2T(uint64_t, int128_t&, int); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, int64_t*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, uint64_t*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, float*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, double*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, long double*); +template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, int128_t*); void* WindowFunctionType::getNullValueByType(int ct, int pos) { @@ -609,7 +658,8 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos) // static uint64_t char4Null = joblist::CHAR4NULL; // static uint64_t char8Null = joblist::CHAR8NULL; static string stringNull(""); - + static int128_t int128Null; // Set at runtime; + void* v = NULL; switch (ct) @@ -713,9 +763,18 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos) v = &intNull; break; - default: + case 8: v = &bigIntNull; break; + + case 16: + datatypes::Decimal::setWideDecimalNullValue(int128Null); + v = &int128Null; + break; + + default: + break; + } break; @@ -741,7 +800,9 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos) case CalpontSystemCatalog::LONGDOUBLE: v = &longDoubleNull; break; - + case CalpontSystemCatalog::BINARY: + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + //fallthrough case CalpontSystemCatalog::VARBINARY: default: std::ostringstream oss; diff --git a/utils/windowfunction/windowfunctiontype.h b/utils/windowfunction/windowfunctiontype.h index 9260cc4a2..dfda6e053 100644 --- a/utils/windowfunction/windowfunctiontype.h +++ b/utils/windowfunction/windowfunctiontype.h @@ -33,6 +33,7 @@ #include "rowgroup.h" #include "windowframe.h" #include "constantcolumn.h" +#include "mcs_decimal.h" namespace ordering { @@ -222,6 +223,12 @@ protected: virtual void* getNullValueByType(int, int); + // There are two types of getters for integral types and for + // DTs wider then 8 bytes. + void getInt128Value(uint64_t i, int128_t& x) + { + return fRow.getInt128Field(i, x); + } int64_t getIntValue(uint64_t i) { return fRow.getIntField(i); @@ -238,11 +245,15 @@ protected: { fRow.setIntField(v, i); } + void setInt128Value(int64_t i, const int128_t& v) + { + fRow.setInt128Field(v, i); + } void setDoubleValue(int64_t i, double v) { fRow.setDoubleField(v, i); } - void setLongDoubleValue(int64_t i, long double v) + void setLongDoubleValue(int64_t i, const long double& v) { fRow.setLongDoubleField(v, i); } diff --git a/versioning/BRM/CMakeLists.txt b/versioning/BRM/CMakeLists.txt index 6f39c645d..db6b809b5 100644 --- a/versioning/BRM/CMakeLists.txt +++ b/versioning/BRM/CMakeLists.txt @@ -28,7 +28,8 @@ set(brm_LIB_SRCS transactionnode.cpp undoable.cpp vbbm.cpp - vss.cpp) + vss.cpp + ../../datatypes/mcs_datatype.cpp) add_library(brm SHARED ${brm_LIB_SRCS}) diff --git a/versioning/BRM/blockresolutionmanager.cpp b/versioning/BRM/blockresolutionmanager.cpp index 7127f3f43..6004a3832 100644 --- a/versioning/BRM/blockresolutionmanager.cpp +++ b/versioning/BRM/blockresolutionmanager.cpp @@ -107,10 +107,10 @@ int BlockResolutionManager::saveState(string filename) throw() saveExtentMap(emFilename); // truncate teh file if already exists since no truncate in HDFS. - const char* filename = journalFilename.c_str(); + const char* filename_p = journalFilename.c_str(); IDBDataFile* journal = IDBDataFile::open( - IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "wb", 0); + IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), filename_p, "wb", 0); delete journal; vbbm.save(vbbmFilename); diff --git a/versioning/BRM/brmtypes.h b/versioning/BRM/brmtypes.h index e0e3f64e7..623720263 100644 --- a/versioning/BRM/brmtypes.h +++ b/versioning/BRM/brmtypes.h @@ -31,6 +31,7 @@ #include #include #include +#include "mcs_basic_types.h" #include "logicalpartition.h" #ifndef _MSC_VER @@ -151,6 +152,17 @@ struct CPInfo int64_t max; int64_t min; int32_t seqNum; + union + { + int128_t bigMax; + int64_t max_; + }; + union + { + int128_t bigMin; + int64_t min_; + }; + bool isBinaryColumn; }; typedef std::vector CPInfoList_t; @@ -160,6 +172,17 @@ struct CPMaxMin int64_t max; int64_t min; int32_t seqNum; + union + { + int128_t bigMax; + int64_t max_; + }; + union + { + int128_t bigMin; + int64_t min_; + }; + bool isBinaryColumn; }; typedef std::tr1::unordered_map CPMaxMinMap_t; @@ -172,7 +195,18 @@ struct CPInfoMerge int64_t min; // min value to be merged with current min value int32_t seqNum; // sequence number (not currently used) execplan::CalpontSystemCatalog::ColDataType type; + int32_t colWidth; bool newExtent; // is this to be treated as a new extent + union + { + int128_t bigMax; + int64_t max_; + }; + union + { + int128_t bigMin; + int64_t min_; + }; }; typedef std::vector CPInfoMergeList_t; @@ -184,7 +218,18 @@ struct CPMaxMinMerge int64_t min; int32_t seqNum; execplan::CalpontSystemCatalog::ColDataType type; + int32_t colWidth; bool newExtent; + union + { + int128_t bigMax; + int64_t max_; + }; + union + { + int128_t bigMin; + int64_t min_; + }; }; typedef std::tr1::unordered_map CPMaxMinMergeMap_t; diff --git a/versioning/BRM/dbrm.cpp b/versioning/BRM/dbrm.cpp index e6079f252..7c8b3c860 100644 --- a/versioning/BRM/dbrm.cpp +++ b/versioning/BRM/dbrm.cpp @@ -30,6 +30,7 @@ //#define NDEBUG #include +#include "dataconvert.h" #include "oamcache.h" #include "rwlock.h" #include "mastersegmenttable.h" @@ -461,7 +462,8 @@ int DBRM::markExtentsInvalid(const vector& lbids, return err; } -int DBRM::getExtentMaxMin(const LBID_t lbid, int64_t& max, int64_t& min, int32_t& seqNum) throw() +template +int DBRM::getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) throw() { #ifdef BRM_INFO @@ -556,7 +558,14 @@ int DBRM::setExtentsMaxMin(const CPInfoList_t& cpInfos) DBRM_THROW for (it = cpInfos.begin(); it != cpInfos.end(); it++) { - command << (uint64_t)it->firstLbid << (uint64_t)it->max << (uint64_t)it->min << (uint32_t)it->seqNum; + if (it->isBinaryColumn) + { + command << (uint8_t)1 << (uint64_t)it->firstLbid << (uint128_t)it->bigMax << (uint128_t)it->bigMin << (uint32_t)it->seqNum; + } + else + { + command << (uint8_t)0 << (uint64_t)it->firstLbid << (uint64_t)it->max << (uint64_t)it->min << (uint32_t)it->seqNum; + } } err = send_recv(command, response); @@ -4526,15 +4535,34 @@ void DBRM::invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN // lookup the column oid for that lbid (all we care about is oid here) if (em->lookupLocal(lbid, oid, dbRoot, partitionNum, segmentNum, fileBlockOffset) == 0) { - if (execplan::isUnsigned(csc->colType(oid).colDataType)) + execplan::CalpontSystemCatalog::ColType colType = csc->colType(oid); + bool isBinaryColumn = colType.colWidth > 8; + aInfo.isBinaryColumn = isBinaryColumn; + if (!isBinaryColumn) { - aInfo.max = 0; - aInfo.min = numeric_limits::max(); + if (datatypes::isUnsigned(colType.colDataType)) + { + aInfo.max = 0; + aInfo.min = numeric_limits::max(); + } + else + { + aInfo.max = numeric_limits::min(); + aInfo.min = numeric_limits::max(); + } } else { - aInfo.max = numeric_limits::min(); - aInfo.min = numeric_limits::max(); + if (datatypes::isUnsigned(colType.colDataType)) + { + aInfo.bigMax = 0; + aInfo.bigMin = -1; + } + else + { + utils::int128Min(aInfo.bigMax); + utils::int128Max(aInfo.bigMin); + } } } else @@ -4542,6 +4570,8 @@ void DBRM::invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN // We have a problem, but we need to put something in. This should never happen. aInfo.max = numeric_limits::min(); aInfo.min = numeric_limits::max(); + // MCOL-641 is this correct? + aInfo.isBinaryColumn = false; } aInfo.seqNum = -2; @@ -4552,4 +4582,10 @@ void DBRM::invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN setExtentsMaxMin(cpInfos); } +template +int DBRM::getExtentMaxMin(const LBID_t lbid, int128_t& max, int128_t& min, int32_t& seqNum) throw(); + +template +int DBRM::getExtentMaxMin(const LBID_t lbid, int64_t& max, int64_t& min, int32_t& seqNum) throw(); + } //namespace diff --git a/versioning/BRM/dbrm.h b/versioning/BRM/dbrm.h index 00165f496..90b24d203 100644 --- a/versioning/BRM/dbrm.h +++ b/versioning/BRM/dbrm.h @@ -822,7 +822,8 @@ public: execplan::CalpontSystemCatalog::ColDataType colDataType) DBRM_THROW; EXPORT int markExtentsInvalid(const std::vector& lbids, const std::vector& colDataTypes) DBRM_THROW; - EXPORT int getExtentMaxMin(const LBID_t lbid, int64_t& max, int64_t& min, int32_t& seqNum) throw(); + template + EXPORT int getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) throw(); EXPORT int setExtentMaxMin(const LBID_t lbid, const int64_t max, const int64_t min, const int32_t seqNum) DBRM_THROW; diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index 2b79fc6ea..bb48844b8 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -55,6 +55,7 @@ namespace bi = boost::interprocess; #include "mastersegmenttable.h" #include "blocksize.h" #include "dataconvert.h" +#include "mcs_decimal.h" #include "oamcache.h" #include "IDBDataFile.h" #include "IDBPolicy.h" @@ -73,6 +74,7 @@ namespace bi = boost::interprocess; #define EM_MAGIC_V2 0x76f78b1d #define EM_MAGIC_V3 0x76f78b1e #define EM_MAGIC_V4 0x76f78b1f +#define EM_MAGIC_V5 0x76f78b20 #ifndef NDEBUG #define ASSERT(x) \ @@ -116,39 +118,47 @@ namespace BRM EMCasualPartition_struct::EMCasualPartition_struct() { - lo_val = numeric_limits::min(); - hi_val = numeric_limits::max(); + utils::int128Max(bigLoVal); + utils::int128Min(bigHiVal); sequenceNum = 0; isValid = CP_INVALID; } EMCasualPartition_struct::EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum) { - lo_val = lo; - hi_val = hi; + loVal = lo; + hiVal = hi; + sequenceNum = seqNum; + isValid = CP_INVALID; +} + +EMCasualPartition_struct::EMCasualPartition_struct(const int128_t bigLo, const int128_t bigHi, const int32_t seqNum) +{ + bigLoVal = bigLo; + bigHiVal = bigHi; sequenceNum = seqNum; isValid = CP_INVALID; } EMCasualPartition_struct::EMCasualPartition_struct(const EMCasualPartition_struct& em) { - lo_val = em.lo_val; - hi_val = em.hi_val; + bigLoVal = em.bigLoVal; + bigHiVal = em.bigHiVal; sequenceNum = em.sequenceNum; isValid = em.isValid; } EMCasualPartition_struct& EMCasualPartition_struct::operator= (const EMCasualPartition_struct& em) { - lo_val = em.lo_val; - hi_val = em.hi_val; + bigLoVal = em.bigLoVal; + bigHiVal = em.bigHiVal; sequenceNum = em.sequenceNum; isValid = em.isValid; return *this; } //------------------------------------------------------------------------------ -// Version 4 EmEntry methods +// Version 5 EmEntry methods //------------------------------------------------------------------------------ EMEntry::EMEntry() @@ -337,13 +347,21 @@ int ExtentMap::_markInvalid(const LBID_t lbid, const execplan::CalpontSystemCata if (isUnsigned(colDataType)) { - fExtentMap[i].partition.cprange.lo_val = numeric_limits::max(); - fExtentMap[i].partition.cprange.hi_val = 0; + fExtentMap[i].partition.cprange.bigLoVal = -1; + fExtentMap[i].partition.cprange.bigHiVal = 0; } else { - fExtentMap[i].partition.cprange.lo_val = numeric_limits::max(); - fExtentMap[i].partition.cprange.hi_val = numeric_limits::min(); + if (fExtentMap[i].colWid != datatypes::MAXDECIMALWIDTH) + { + fExtentMap[i].partition.cprange.loVal = numeric_limits::max(); + fExtentMap[i].partition.cprange.hiVal = numeric_limits::min(); + } + else + { + utils::int128Max(fExtentMap[i].partition.cprange.bigLoVal); + utils::int128Min(fExtentMap[i].partition.cprange.bigHiVal); + } } incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); @@ -352,8 +370,8 @@ int ExtentMap::_markInvalid(const LBID_t lbid, const execplan::CalpontSystemCata os << "ExtentMap::_markInvalid(): casual partitioning update: firstLBID=" << fExtentMap[i].range.start << " lastLBID=" << fExtentMap[i].range.start + fExtentMap[i].range.size * 1024 - 1 << " OID=" << fExtentMap[i].fileID << - " min=" << fExtentMap[i].partition.cprange.lo_val << - " max=" << fExtentMap[i].partition.cprange.hi_val << + " min=" << fExtentMap[i].partition.cprange.loVal << + " max=" << fExtentMap[i].partition.cprange.hiVal << "seq=" << fExtentMap[i].partition.cprange.sequenceNum; log(os.str(), logging::LOG_TYPE_DEBUG); #endif @@ -452,7 +470,7 @@ int ExtentMap::markInvalid(const vector& lbids, /** * @brief set the max/min values for the extent if the seqNum matches the extents sequenceNum * -* reset the lbid's hi_val to max and lo_val to min +* reset the lbid's hiVal to max and loVal to min * the seqNum matches the ExtentMap.sequenceNum. Then increments * the current sequenceNum value by 1. If the sequenceNum does not * match the seqNum value do not update the lbid's max/min values @@ -460,6 +478,8 @@ int ExtentMap::markInvalid(const vector& lbids, **/ +// TODO MCOL-641 Not adding support here since this function appears to be unused anywhere. + int ExtentMap::setMaxMin(const LBID_t lbid, const int64_t max, const int64_t min, @@ -521,8 +541,8 @@ int ExtentMap::setMaxMin(const LBID_t lbid, if (curSequence == seqNum) { makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); - fExtentMap[i].partition.cprange.hi_val = max; - fExtentMap[i].partition.cprange.lo_val = min; + fExtentMap[i].partition.cprange.hiVal = max; + fExtentMap[i].partition.cprange.loVal = min; fExtentMap[i].partition.cprange.isValid = CP_VALID; incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); return 0; @@ -532,7 +552,7 @@ int ExtentMap::setMaxMin(const LBID_t lbid, else if (seqNum == -1) { makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); - // We set hi_val and lo_val to correct values for signed or unsigned + // We set hiVal and loVal to correct values for signed or unsigned // during the markinvalid step, which sets the invalid variable to CP_UPDATING. // During this step (seqNum == -1), the min and max passed in are not reliable // and should not be used. @@ -626,8 +646,16 @@ void ExtentMap::setExtentsMaxMin(const CPMaxMinMap_t& cpMap, bool firstNode, boo fExtentMap[i].partition.cprange.isValid == CP_INVALID) { makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); - fExtentMap[i].partition.cprange.hi_val = it->second.max; - fExtentMap[i].partition.cprange.lo_val = it->second.min; + if (it->second.isBinaryColumn) + { + fExtentMap[i].partition.cprange.bigHiVal = it->second.bigMax; + fExtentMap[i].partition.cprange.bigLoVal = it->second.bigMin; + } + else + { + fExtentMap[i].partition.cprange.hiVal = it->second.max; + fExtentMap[i].partition.cprange.loVal = it->second.min; + } fExtentMap[i].partition.cprange.isValid = CP_VALID; incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); extentsUpdated++; @@ -651,7 +679,7 @@ void ExtentMap::setExtentsMaxMin(const CPMaxMinMap_t& cpMap, bool firstNode, boo else if (it->second.seqNum == -1) { makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); - // We set hi_val and lo_val to correct values for signed or unsigned + // We set hiVal and loVal to correct values for signed or unsigned // during the markinvalid step, which sets the invalid variable to CP_UPDATING. // During this step (seqNum == -1), the min and max passed in are not reliable // and should not be used. @@ -663,8 +691,16 @@ void ExtentMap::setExtentsMaxMin(const CPMaxMinMap_t& cpMap, bool firstNode, boo else if (it->second.seqNum == -2) { makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); - fExtentMap[i].partition.cprange.hi_val = it->second.max; - fExtentMap[i].partition.cprange.lo_val = it->second.min; + if (it->second.isBinaryColumn) + { + fExtentMap[i].partition.cprange.bigHiVal = it->second.bigMax; + fExtentMap[i].partition.cprange.bigLoVal = it->second.bigMin; + } + else + { + fExtentMap[i].partition.cprange.hiVal = it->second.max; + fExtentMap[i].partition.cprange.loVal = it->second.min; + } fExtentMap[i].partition.cprange.isValid = CP_INVALID; incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); extentsUpdated++; @@ -707,6 +743,7 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) { CPMaxMinMergeMap_t::const_iterator it; + // TODO MCOL-641 Add support in the debugging outputs here. #ifdef BRM_DEBUG log("ExtentMap::mergeExtentsMaxMin()", logging::LOG_TYPE_DEBUG); @@ -777,21 +814,22 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) os << "ExtentMap::mergeExtentsMaxMin(): casual partitioning update: firstLBID=" << fExtentMap[i].range.start << " lastLBID=" << fExtentMap[i].range.start + fExtentMap[i].range.size * 1024 - 1 << " OID=" << fExtentMap[i].fileID << - " hi_val=" << fExtentMap[i].partition.cprange.hi_val << - " lo_val=" << fExtentMap[i].partition.cprange.lo_val << + " hiVal=" << fExtentMap[i].partition.cprange.hiVal << + " loVal=" << fExtentMap[i].partition.cprange.loVal << " min=" << it->second.min << " max=" << it->second.max << " seq=" << it->second.seqNum; log(os.str(), logging::LOG_TYPE_DEBUG); #endif + bool isBinaryColumn = it->second.colWidth > 8; + switch (fExtentMap[i].partition.cprange.isValid) { // Merge input min/max with current min/max case CP_VALID: { - if (!isValidCPRange( it->second.max, - it->second.min, - it->second.type )) + if ((!isBinaryColumn && !isValidCPRange(it->second.max, it->second.min, it->second.type)) || + (isBinaryColumn && !isValidCPRange(it->second.bigMax, it->second.bigMin, it->second.type))) { break; } @@ -803,10 +841,8 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) // having all NULL values, in which case the current // min/max needs to be set instead of merged. - if (isValidCPRange( - fExtentMap[i].partition.cprange.hi_val, - fExtentMap[i].partition.cprange.lo_val, - it->second.type)) + if ((!isBinaryColumn && isValidCPRange(fExtentMap[i].partition.cprange.hiVal, fExtentMap[i].partition.cprange.loVal, it->second.type)) || + (isBinaryColumn && isValidCPRange(fExtentMap[i].partition.cprange.bigHiVal, fExtentMap[i].partition.cprange.bigLoVal, it->second.type))) { // Swap byte order to do binary string comparison if (isCharType(it->second.type)) @@ -820,55 +856,99 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) int64_t oldMinVal = static_cast( uint64ToStr( static_cast( - fExtentMap[i].partition.cprange.lo_val)) ); + fExtentMap[i].partition.cprange.loVal)) ); int64_t oldMaxVal = static_cast( uint64ToStr( static_cast( - fExtentMap[i].partition.cprange.hi_val)) ); + fExtentMap[i].partition.cprange.hiVal)) ); if (newMinVal < oldMinVal) - fExtentMap[i].partition.cprange.lo_val = + fExtentMap[i].partition.cprange.loVal = it->second.min; if (newMaxVal > oldMaxVal) - fExtentMap[i].partition.cprange.hi_val = + fExtentMap[i].partition.cprange.hiVal = it->second.max; } else if (isUnsigned(it->second.type)) { - if (static_cast(it->second.min) < - static_cast(fExtentMap[i].partition.cprange.lo_val)) + if (!isBinaryColumn) { - fExtentMap[i].partition.cprange.lo_val = - it->second.min; - } + if (static_cast(it->second.min) < + static_cast(fExtentMap[i].partition.cprange.loVal)) + { + fExtentMap[i].partition.cprange.loVal = + it->second.min; + } - if (static_cast(it->second.max) > - static_cast(fExtentMap[i].partition.cprange.hi_val)) + if (static_cast(it->second.max) > + static_cast(fExtentMap[i].partition.cprange.hiVal)) + { + fExtentMap[i].partition.cprange.hiVal = + it->second.max; + } + } + else { - fExtentMap[i].partition.cprange.hi_val = - it->second.max; + if (static_cast(it->second.bigMin) < + static_cast(fExtentMap[i].partition.cprange.bigLoVal)) + { + fExtentMap[i].partition.cprange.bigLoVal = + it->second.bigMin; + } + + if (static_cast(it->second.bigMax) > + static_cast(fExtentMap[i].partition.cprange.bigHiVal)) + { + fExtentMap[i].partition.cprange.bigHiVal = + it->second.bigMax; + } } } else { - if (it->second.min < - fExtentMap[i].partition.cprange.lo_val) - fExtentMap[i].partition.cprange.lo_val = - it->second.min; + if (!isBinaryColumn) + { + if (it->second.min < + fExtentMap[i].partition.cprange.loVal) + fExtentMap[i].partition.cprange.loVal = + it->second.min; - if (it->second.max > - fExtentMap[i].partition.cprange.hi_val) - fExtentMap[i].partition.cprange.hi_val = - it->second.max; + if (it->second.max > + fExtentMap[i].partition.cprange.hiVal) + fExtentMap[i].partition.cprange.hiVal = + it->second.max; + } + else + { + if (it->second.bigMin < + fExtentMap[i].partition.cprange.bigLoVal) + fExtentMap[i].partition.cprange.bigLoVal = + it->second.bigMin; + + if (it->second.bigMax > + fExtentMap[i].partition.cprange.bigHiVal) + fExtentMap[i].partition.cprange.bigHiVal = + it->second.bigMax; + } } } else { - fExtentMap[i].partition.cprange.lo_val = - it->second.min; - fExtentMap[i].partition.cprange.hi_val = - it->second.max; + if (!isBinaryColumn) + { + fExtentMap[i].partition.cprange.loVal = + it->second.min; + fExtentMap[i].partition.cprange.hiVal = + it->second.max; + } + else + { + fExtentMap[i].partition.cprange.bigLoVal = + it->second.bigMin; + fExtentMap[i].partition.cprange.bigHiVal = + it->second.bigMax; + } } incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); @@ -897,14 +977,23 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) if (it->second.newExtent) { - if (isValidCPRange( it->second.max, - it->second.min, - it->second.type )) + if ((!isBinaryColumn && isValidCPRange(it->second.max, it->second.min, it->second.type)) || + (isBinaryColumn && isValidCPRange(it->second.bigMax, it->second.bigMin, it->second.type))) { - fExtentMap[i].partition.cprange.lo_val = - it->second.min; - fExtentMap[i].partition.cprange.hi_val = - it->second.max; + if (!isBinaryColumn) + { + fExtentMap[i].partition.cprange.loVal = + it->second.min; + fExtentMap[i].partition.cprange.hiVal = + it->second.max; + } + else + { + fExtentMap[i].partition.cprange.bigLoVal = + it->second.bigMin; + fExtentMap[i].partition.cprange.bigHiVal = + it->second.bigMax; + } } // Even if invalid range; we set state to CP_VALID, @@ -939,22 +1028,51 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) // Range is considered invalid if min or max, are NULL (min()), or EMPTY // (min()+1). For unsigned types NULL is max() and EMPTY is max()-1. //------------------------------------------------------------------------------ -bool ExtentMap::isValidCPRange(int64_t max, int64_t min, execplan::CalpontSystemCatalog::ColDataType type) const +template +bool ExtentMap::isValidCPRange(const T& max, const T& min, execplan::CalpontSystemCatalog::ColDataType type) const { if (isUnsigned(type)) { - if ( (static_cast(min) >= (numeric_limits::max() - 1)) || - (static_cast(max) >= (numeric_limits::max() - 1)) ) + if (typeid(T) != typeid(int128_t)) { - return false; + if ( (static_cast(min) >= (numeric_limits::max() - 1)) || + (static_cast(max) >= (numeric_limits::max() - 1)) ) + { + return false; + } + } + else + { + uint128_t temp; + utils::uint128Max(temp); + + if ( (static_cast(min) >= (temp - 1)) || + (static_cast(max) >= (temp - 1)) ) + { + return false; + } } } else { - if ( (min <= (numeric_limits::min() + 1)) || - (max <= (numeric_limits::min() + 1)) ) + if (typeid(T) != typeid(int128_t)) { - return false; + if ( (min <= (numeric_limits::min() + 1)) || + (max <= (numeric_limits::min() + 1)) ) + { + return false; + } + } + else + { + int128_t temp; + utils::int128Min(temp); + + if ( (min <= (temp + 1)) || + (max <= (temp + 1)) ) + { + return false; + } } } @@ -962,16 +1080,16 @@ bool ExtentMap::isValidCPRange(int64_t max, int64_t min, execplan::CalpontSystem } /** -* @brief retrieve the hi_val and lo_val or sequenceNum of the extent containing the LBID lbid. +* @brief retrieve the hiVal and loVal or sequenceNum of the extent containing the LBID lbid. * * For the extent containing the LBID lbid, return the max/min values if the extent range values * are valid and a -1 in the seqNum parameter. If the range values are flaged as invalid * return the sequenceNum of the extent and the max/min values as -1. **/ +template int ExtentMap::getMaxMin(const LBID_t lbid, - int64_t& max, - int64_t& min, + T& max, T& min, int32_t& seqNum) { #ifdef BRM_INFO @@ -987,8 +1105,19 @@ int ExtentMap::getMaxMin(const LBID_t lbid, } #endif - max = numeric_limits::max(); - min = 0; + if (typeid(T) == typeid(int128_t)) + { + int128_t tmpMax, tmpMin; + utils::int128Min(tmpMax); + utils::int128Max(tmpMin); + max = tmpMax; + min = tmpMin; + } + else + { + max = numeric_limits::min(); + min = numeric_limits::max(); + } seqNum *= (-1); int entries; int i; @@ -1014,8 +1143,16 @@ int ExtentMap::getMaxMin(const LBID_t lbid, if (lbid >= fExtentMap[i].range.start && lbid <= lastBlock) { - max = fExtentMap[i].partition.cprange.hi_val; - min = fExtentMap[i].partition.cprange.lo_val; + if (typeid(T) == typeid(int128_t)) + { + max = fExtentMap[i].partition.cprange.bigHiVal; + min = fExtentMap[i].partition.cprange.bigLoVal; + } + else + { + max = fExtentMap[i].partition.cprange.hiVal; + min = fExtentMap[i].partition.cprange.loVal; + } seqNum = fExtentMap[i].partition.cprange.sequenceNum; isValid = fExtentMap[i].partition.cprange.isValid; releaseEMEntryTable(READ); @@ -1134,7 +1271,7 @@ void ExtentMap::reserveLBIDRange(LBID_t start, uint8_t size) */ -void ExtentMap::loadVersion4(IDBDataFile* in) +void ExtentMap::loadVersion4or5(IDBDataFile* in, bool upgradeV4ToV5) { int emNumElements = 0, flNumElements = 0; @@ -1145,8 +1282,8 @@ void ExtentMap::loadVersion4(IDBDataFile* in) if ((size_t) nbytes != sizeof(int) + sizeof(int)) { - log_errno("ExtentMap::loadVersion4(): read "); - throw runtime_error("ExtentMap::loadVersion4(): read failed. Check the error log."); + log_errno("ExtentMap::loadVersion4or5(): read "); + throw runtime_error("ExtentMap::loadVersion4or5(): read failed. Check the error log."); } void *fExtentMapPtr = static_cast(fExtentMap); @@ -1175,18 +1312,64 @@ void ExtentMap::loadVersion4(IDBDataFile* in) growEMShmseg(nrows); } - size_t progress = 0, writeSize = emNumElements * sizeof(EMEntry); int err; - char *writePos = (char *) fExtentMap; - while (progress < writeSize) + char* writePos; + size_t progress, writeSize; + + if (!upgradeV4ToV5) { - err = in->read(writePos + progress, writeSize - progress); - if (err <= 0) + progress = 0; + writeSize = emNumElements * sizeof(EMEntry); + writePos = (char *) fExtentMap; + + while (progress < writeSize) { - log_errno("ExtentMap::loadVersion4(): read "); - throw runtime_error("ExtentMap::loadVersion4(): read failed. Check the error log."); + err = in->read(writePos + progress, writeSize - progress); + if (err <= 0) + { + log_errno("ExtentMap::loadVersion4or5(): read "); + throw runtime_error("ExtentMap::loadVersion4or5(): read failed. Check the error log."); + } + progress += (uint) err; } - progress += (uint) err; + } + else + { + // We are upgrading extent map from v4 to v5. + for (int i = 0; i < emNumElements; i++) + { + EMEntry_v4 emEntryV4; + progress = 0; + writeSize = sizeof(EMEntry_v4); + writePos = (char *) &(emEntryV4); + while (progress < writeSize) + { + err = in->read(writePos + progress, writeSize - progress); + if (err <= 0) + { + log_errno("ExtentMap::loadVersion4or5(): read "); + throw runtime_error("ExtentMap::loadVersion4or5(): read failed during upgrade. Check the error log."); + } + progress += (uint) err; + } + + fExtentMap[i].range.start = emEntryV4.range.start; + fExtentMap[i].range.size = emEntryV4.range.size; + fExtentMap[i].fileID = emEntryV4.fileID; + fExtentMap[i].blockOffset = emEntryV4.blockOffset; + fExtentMap[i].HWM = emEntryV4.HWM; + fExtentMap[i].partitionNum = emEntryV4.partitionNum; + fExtentMap[i].segmentNum = emEntryV4.segmentNum; + fExtentMap[i].dbRoot = emEntryV4.dbRoot; + fExtentMap[i].colWid = emEntryV4.colWid; + fExtentMap[i].status = emEntryV4.status; + fExtentMap[i].partition.cprange.hiVal = emEntryV4.partition.cprange.hi_val; + fExtentMap[i].partition.cprange.loVal = emEntryV4.partition.cprange.lo_val; + fExtentMap[i].partition.cprange.sequenceNum = emEntryV4.partition.cprange.sequenceNum; + fExtentMap[i].partition.cprange.isValid = emEntryV4.partition.cprange.isValid; + } + + std::cout<read((char*) &emVersion, sizeof(int)); - if (bytes == (int) sizeof(int) && emVersion == EM_MAGIC_V4) - loadVersion4(in.get()); + if (bytes == (int) sizeof(int) && + (emVersion == EM_MAGIC_V4 || emVersion == EM_MAGIC_V5)) + { + loadVersion4or5(in.get(), emVersion == EM_MAGIC_V4); + } else { log("ExtentMap::load(): That file is not a valid ExtentMap image"); @@ -1344,7 +1530,7 @@ void ExtentMap::save(const string& filename) throw ios_base::failure("ExtentMap::save(): open failed. Check the error log."); } - loadSize[0] = EM_MAGIC_V4; + loadSize[0] = EM_MAGIC_V5; loadSize[1] = fEMShminfo->currentSize / sizeof(EMEntry); loadSize[2] = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); // needs to send all entries @@ -2569,13 +2755,29 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, if (isUnsigned(colDataType)) { - e->partition.cprange.lo_val = numeric_limits::max(); - e->partition.cprange.hi_val = 0; + if (colWidth != datatypes::MAXDECIMALWIDTH) + { + e->partition.cprange.loVal = numeric_limits::max(); + e->partition.cprange.hiVal = 0; + } + else + { + e->partition.cprange.bigLoVal = -1; + e->partition.cprange.bigHiVal = 0; + } } else { - e->partition.cprange.lo_val = numeric_limits::max(); - e->partition.cprange.hi_val = numeric_limits::min(); + if (colWidth != datatypes::MAXDECIMALWIDTH) + { + e->partition.cprange.loVal = numeric_limits::max(); + e->partition.cprange.hiVal = numeric_limits::min(); + } + else + { + utils::int128Max(e->partition.cprange.bigLoVal); + utils::int128Min(e->partition.cprange.bigHiVal); + } } e->partition.cprange.sequenceNum = 0; @@ -2764,13 +2966,29 @@ LBID_t ExtentMap::_createColumnExtentExactFile(uint32_t size, int OID, if (isUnsigned(colDataType)) { - e->partition.cprange.lo_val = numeric_limits::max(); - e->partition.cprange.hi_val = 0; + if (colWidth != datatypes::MAXDECIMALWIDTH) + { + e->partition.cprange.loVal = numeric_limits::max(); + e->partition.cprange.hiVal = 0; + } + else + { + e->partition.cprange.bigLoVal = -1; + e->partition.cprange.bigHiVal = 0; + } } else { - e->partition.cprange.lo_val = numeric_limits::max(); - e->partition.cprange.hi_val = numeric_limits::min(); + if (colWidth != datatypes::MAXDECIMALWIDTH) + { + e->partition.cprange.loVal = numeric_limits::max(); + e->partition.cprange.hiVal = numeric_limits::min(); + } + else + { + utils::int128Max(e->partition.cprange.bigLoVal); + utils::int128Min(e->partition.cprange.bigHiVal); + } } e->partition.cprange.sequenceNum = 0; @@ -2953,8 +3171,8 @@ LBID_t ExtentMap::_createDictStoreExtent(uint32_t size, int OID, e->range.size = size; e->fileID = OID; e->status = EXTENTUNAVAILABLE;// @bug 1911 mark extent as in process - e->partition.cprange.lo_val = numeric_limits::max(); - e->partition.cprange.hi_val = numeric_limits::min(); + utils::int128Max(e->partition.cprange.bigLoVal); + utils::int128Min(e->partition.cprange.bigHiVal); e->partition.cprange.sequenceNum = 0; e->partition.cprange.isValid = CP_INVALID; @@ -3043,8 +3261,8 @@ void ExtentMap::printEM(const EMEntry& em) const << (long) em.range.size << " OID " << (long) em.fileID << " offset " << (long) em.blockOffset - << " LV " << em.partition.cprange.lo_val - << " HV " << em.partition.cprange.hi_val; + << " LV " << em.partition.cprange.loVal + << " HV " << em.partition.cprange.hiVal; cout << endl; } @@ -4440,8 +4658,8 @@ void ExtentMap::setLocalHWM(int OID, uint32_t partitionNum, os << "ExtentMap::setLocalHWM(): firstLBID=" << fExtentMap[lastExtentIndex].range.start << " lastLBID=" << fExtentMap[lastExtentIndex].range.start + fExtentMap[lastExtentIndex].range.size * 1024 - 1 << " newHWM=" << fExtentMap[lastExtentIndex].HWM - << " min=" << fExtentMap[lastExtentIndex].partition.cprange.lo_val << " max=" << - fExtentMap[lastExtentIndex].partition.cprange.hi_val << " seq=" << + << " min=" << fExtentMap[lastExtentIndex].partition.cprange.loVal << " max=" << + fExtentMap[lastExtentIndex].partition.cprange.hiVal << " seq=" << fExtentMap[lastExtentIndex].partition.cprange.sequenceNum << " status="; switch (fExtentMap[lastExtentIndex].partition.cprange.isValid) @@ -4602,8 +4820,8 @@ void ExtentMap::getExtents_dbroot(int OID, vector& entries, cons fakeEntry.dbRoot = 1; fakeEntry.colWid = 4; fakeEntry.status = EXTENTAVAILABLE; - fakeEntry.partition.cprange.hi_val = numeric_limits::min() + 2; - fakeEntry.partition.cprange.lo_val = numeric_limits::max(); + fakeEntry.partition.cprange.hiVal = numeric_limits::min() + 2; + fakeEntry.partition.cprange.loVal = numeric_limits::max(); fakeEntry.partition.cprange.sequenceNum = 0; fakeEntry.partition.cprange.isValid = CP_INVALID; entries.push_back(fakeEntry); @@ -5247,8 +5465,8 @@ void ExtentMap::lookup(OID_t OID, LBIDRange_v& ranges) fakeEntry.dbRoot = 1; fakeEntry.colWid = 4; fakeEntry.status = EXTENTAVAILABLE; - fakeEntry.partition.cprange.hi_val = numeric_limits::min() + 2; - fakeEntry.partition.cprange.lo_val = numeric_limits::max(); + fakeEntry.partition.cprange.hiVal = numeric_limits::min() + 2; + fakeEntry.partition.cprange.loVal = numeric_limits::max(); fakeEntry.partition.cprange.sequenceNum = 0; fakeEntry.partition.cprange.isValid = CP_INVALID; #endif @@ -5783,8 +6001,8 @@ void ExtentMap::dumpTo(ostream& os) << fExtentMap[i].dbRoot << '|' << fExtentMap[i].colWid << '|' << fExtentMap[i].status << '|' - << fExtentMap[i].partition.cprange.hi_val << '|' - << fExtentMap[i].partition.cprange.lo_val << '|' + << fExtentMap[i].partition.cprange.hiVal << '|' + << fExtentMap[i].partition.cprange.loVal << '|' << fExtentMap[i].partition.cprange.sequenceNum << '|' << (int)fExtentMap[i].partition.cprange.isValid << '|' << endl; @@ -5840,6 +6058,12 @@ void ExtentMap::dumpTo(ostream& os) } */ +template +int ExtentMap::getMaxMin(const LBID_t lbidRange, int128_t& max, int128_t& min, int32_t& seqNum); + +template +int ExtentMap::getMaxMin(const LBID_t lbidRange, int64_t& max, int64_t& min, int32_t& seqNum); + } //namespace // vim:ts=4 sw=4: diff --git a/versioning/BRM/extentmap.h b/versioning/BRM/extentmap.h index 6cd628a37..8f16c3cee 100644 --- a/versioning/BRM/extentmap.h +++ b/versioning/BRM/extentmap.h @@ -100,14 +100,54 @@ const char CP_INVALID = 0; const char CP_UPDATING = 1; const char CP_VALID = 2; -struct EMCasualPartition_struct +// The _v4 structs are defined below for upgrading extent map +// from v4 to v5; see ExtentMap::loadVersion4or5 for details. +struct EMCasualPartition_struct_v4 { - RangePartitionData_t hi_val; // This needs to be reinterpreted as unsigned for uint64_t column types. + RangePartitionData_t hi_val; // This needs to be reinterpreted as unsigned for uint64_t column types. RangePartitionData_t lo_val; int32_t sequenceNum; char isValid; //CP_INVALID - No min/max and no DML in progress. CP_UPDATING - Update in progress. CP_VALID- min/max is valid +}; + +struct EMPartition_struct_v4 +{ + EMCasualPartition_struct_v4 cprange; +}; + +struct EMEntry_v4 +{ + InlineLBIDRange range; + int fileID; + uint32_t blockOffset; + HWM_t HWM; + uint32_t partitionNum; // starts at 0 + uint16_t segmentNum; // starts at 0 + uint16_t dbRoot; // starts at 1 to match Columnstore.xml + uint16_t colWid; + int16_t status; //extent avail for query or not, or out of service + EMPartition_struct_v4 partition; +}; + +// MCOL-641: v5 structs of the extent map. This version supports int128_t min +// and max values for casual partitioning. +struct EMCasualPartition_struct +{ + int32_t sequenceNum; + char isValid; //CP_INVALID - No min/max and no DML in progress. CP_UPDATING - Update in progress. CP_VALID- min/max is valid + union + { + int128_t bigLoVal; // These need to be reinterpreted as unsigned for uint64_t/uint128_t column types. + int64_t loVal; + }; + union + { + int128_t bigHiVal; + int64_t hiVal; + }; EXPORT EMCasualPartition_struct(); EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum); + EXPORT EMCasualPartition_struct(const int128_t bigLo, const int128_t bigHi, const int32_t seqNum); EXPORT EMCasualPartition_struct(const EMCasualPartition_struct& em); EXPORT EMCasualPartition_struct& operator= (const EMCasualPartition_struct& em); }; @@ -115,7 +155,7 @@ typedef EMCasualPartition_struct EMCasualPartition_t; struct EMPartition_struct { - EMCasualPartition_t cprange; + EMCasualPartition_t cprange; }; typedef EMPartition_struct EMPartition_t; @@ -854,7 +894,8 @@ public: */ void mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock = true); - EXPORT int getMaxMin(const LBID_t lbidRange, int64_t& max, int64_t& min, int32_t& seqNum); + template + EXPORT int getMaxMin(const LBID_t lbidRange, T& max, T& min, int32_t& seqNum); inline bool empty() { @@ -934,7 +975,8 @@ private: uint16_t dbRoot, uint32_t partitionNum, uint16_t segmentNum); - bool isValidCPRange(int64_t max, int64_t min, execplan::CalpontSystemCatalog::ColDataType type) const; + template + bool isValidCPRange(const T& max, const T& min, execplan::CalpontSystemCatalog::ColDataType type) const; void deleteExtent(int emIndex); LBID_t getLBIDsFromFreeList(uint32_t size); void reserveLBIDRange(LBID_t start, uint8_t size); // used by load() to allocate pre-existing LBIDs @@ -960,8 +1002,13 @@ private: int _markInvalid(const LBID_t lbid, const execplan::CalpontSystemCatalog::ColDataType colDataType); - void loadVersion4(std::ifstream& in); - void loadVersion4(idbdatafile::IDBDataFile* in); + /** @brief Loads the extent map from a file into memory. + * + * @param in (in) the file to load the extent map from. + * @param upgradeV4ToV5 (in) flag indicating whether we are upgrading + * extent map from v4 to v5. + */ + void loadVersion4or5(idbdatafile::IDBDataFile* in, bool upgradeV4ToV5); ExtentMapImpl* fPExtMapImpl; FreeListImpl* fPFreeListImpl; diff --git a/versioning/BRM/slavecomm.cpp b/versioning/BRM/slavecomm.cpp index f43a8c351..f63ffcadf 100644 --- a/versioning/BRM/slavecomm.cpp +++ b/versioning/BRM/slavecomm.cpp @@ -1329,9 +1329,12 @@ void SlaveComm::do_setExtentsMaxMin(ByteStream& msg) LBID_t lbid; uint64_t tmp64; uint32_t tmp32; + uint8_t tmp8; + uint128_t tmp128; int err; ByteStream reply; int32_t updateCount; + bool isBinaryColumn = false; #ifdef BRM_VERBOSE cerr << "WorkerComm: do_setExtentsMaxMin()" << endl; @@ -1348,14 +1351,30 @@ void SlaveComm::do_setExtentsMaxMin(ByteStream& msg) // Loop through extents and add each one to a map. for (int64_t i = 0; i < updateCount; i++) { + msg >> tmp8; + isBinaryColumn = (tmp8 != 0); + msg >> tmp64; lbid = tmp64; - msg >> tmp64; - cpMaxMin.max = tmp64; + cpMaxMin.isBinaryColumn = isBinaryColumn; - msg >> tmp64; - cpMaxMin.min = tmp64; + if (isBinaryColumn) + { + msg >> tmp128; + cpMaxMin.bigMax = tmp128; + + msg >> tmp128; + cpMaxMin.bigMin = tmp128; + } + else + { + msg >> tmp64; + cpMaxMin.max = tmp64; + + msg >> tmp64; + cpMaxMin.min = tmp64; + } msg >> tmp32; cpMaxMin.seqNum = tmp32; diff --git a/versioning/BRM/slavedbrmnode.cpp b/versioning/BRM/slavedbrmnode.cpp index 36d4b0cb7..f163ea31f 100644 --- a/versioning/BRM/slavedbrmnode.cpp +++ b/versioning/BRM/slavedbrmnode.cpp @@ -413,8 +413,17 @@ int SlaveDBRMNode::bulkSetHWMAndCP(const vector& hwmArgs, { for (i = 0; i < setCPDataArgs.size(); i++) { - setCPEntry.max = setCPDataArgs[i].max; - setCPEntry.min = setCPDataArgs[i].min; + setCPEntry.isBinaryColumn = setCPDataArgs[i].isBinaryColumn; + if (!setCPEntry.isBinaryColumn) + { + setCPEntry.max = setCPDataArgs[i].max; + setCPEntry.min = setCPDataArgs[i].min; + } + else + { + setCPEntry.bigMax = setCPDataArgs[i].bigMax; + setCPEntry.bigMin = setCPDataArgs[i].bigMin; + } setCPEntry.seqNum = setCPDataArgs[i].seqNum; bulkSetCPMap[setCPDataArgs[i].firstLbid] = setCPEntry; } @@ -428,8 +437,17 @@ int SlaveDBRMNode::bulkSetHWMAndCP(const vector& hwmArgs, for (i = 0; i < mergeCPDataArgs.size(); i++) { mergeCPEntry.type = mergeCPDataArgs[i].type; - mergeCPEntry.max = mergeCPDataArgs[i].max; - mergeCPEntry.min = mergeCPDataArgs[i].min; + mergeCPEntry.colWidth = mergeCPDataArgs[i].colWidth; + if (mergeCPDataArgs[i].colWidth <= 8) + { + mergeCPEntry.max = mergeCPDataArgs[i].max; + mergeCPEntry.min = mergeCPDataArgs[i].min; + } + else + { + mergeCPEntry.bigMax = mergeCPDataArgs[i].bigMax; + mergeCPEntry.bigMin = mergeCPDataArgs[i].bigMin; + } mergeCPEntry.newExtent = mergeCPDataArgs[i].newExtent; mergeCPEntry.seqNum = mergeCPDataArgs[i].seqNum; bulkMergeCPMap[mergeCPDataArgs[i].startLbid] = mergeCPEntry; diff --git a/writeengine/bulk/we_brmreporter.cpp b/writeengine/bulk/we_brmreporter.cpp index fd9691ef9..5517325e1 100644 --- a/writeengine/bulk/we_brmreporter.cpp +++ b/writeengine/bulk/we_brmreporter.cpp @@ -34,6 +34,8 @@ #include "we_log.h" #include "cacheutils.h" #include "IDBPolicy.h" +#include "mcs_decimal.h" +#include "dataconvert.h" namespace WriteEngine { @@ -305,12 +307,27 @@ void BRMReporter::sendCPToFile( ) for (unsigned int i = 0; i < fCPInfo.size(); i++) { - fRptFile << "CP: " << fCPInfo[i].startLbid << ' ' << - fCPInfo[i].max << ' ' << - fCPInfo[i].min << ' ' << - fCPInfo[i].seqNum << ' ' << - fCPInfo[i].type << ' ' << - fCPInfo[i].newExtent << std::endl; + if (!datatypes::isWideDecimalType(fCPInfo[i].type, fCPInfo[i].colWidth)) + { + fRptFile << "CP: " << fCPInfo[i].startLbid << ' ' << + fCPInfo[i].max << ' ' << + fCPInfo[i].min << ' ' << + fCPInfo[i].seqNum << ' ' << + fCPInfo[i].type << ' ' << + fCPInfo[i].newExtent << std::endl; + } + else + { + datatypes::TSInt128 bigMin(&fCPInfo[i].bigMin); + datatypes::TSInt128 bigMax(&fCPInfo[i].bigMax); + + fRptFile << "CP: " << fCPInfo[i].startLbid << ' ' << + bigMax << ' ' << + bigMin << ' ' << + fCPInfo[i].seqNum << ' ' << + fCPInfo[i].type << ' ' << + fCPInfo[i].newExtent << std::endl; + } } } } diff --git a/writeengine/bulk/we_bulkload.cpp b/writeengine/bulk/we_bulkload.cpp index e51b51237..0b4c4abb5 100644 --- a/writeengine/bulk/we_bulkload.cpp +++ b/writeengine/bulk/we_bulkload.cpp @@ -563,9 +563,10 @@ int BulkLoad::preProcess( Job& job, int tableNo, job.jobTableList[tableNo].colList[i].weType = curColStruct.colType; // set width to correct column width job.jobTableList[tableNo].colList[i].width = curColStruct.colWidth; - job.jobTableList[tableNo].colList[i].emptyVal = getEmptyRowValue( - job.jobTableList[tableNo].colList[i].dataType, - job.jobTableList[tableNo].colList[i].width ); + getEmptyRowValue( + job.jobTableList[tableNo].colList[i].dataType, + job.jobTableList[tableNo].colList[i].width, + (uint8_t*)&job.jobTableList[tableNo].colList[i].emptyVal); // check HWM for column file rc = BRMWrapper::getInstance()->getDbRootHWMInfo( curJobCol.mapOid, diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index 069aa0c64..e0fadf1f8 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -39,6 +39,7 @@ #include "brmtypes.h" #include "dataconvert.h" #include "exceptclasses.h" +#include "mcs_decimal.h" #include "joblisttypes.h" @@ -301,6 +302,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, int32_t iDate; char charTmpBuf[MAX_COLUMN_BOUNDARY + 1] = {0}; long long llVal = 0, llDate = 0; + int128_t bigllVal = 0; uint64_t tmp64; uint32_t tmp32; uint8_t ubiVal; @@ -938,7 +940,9 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, // BIG INT //---------------------------------------------------------------------- case WriteEngine::WR_LONGLONG: + case WriteEngine::WR_BINARY: { + // TODO MCOL-641 Add full support here. bool bSatVal = false; if ( column.dataType != CalpontSystemCatalog::DATETIME && @@ -983,9 +987,18 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, if ( (column.dataType == CalpontSystemCatalog::DECIMAL) || (column.dataType == CalpontSystemCatalog::UDECIMAL)) { - // errno is initialized and set in convertDecimalString - llVal = Convertor::convertDecimalString( - field, fieldLength, column.scale ); + if (LIKELY(width == datatypes::MAXDECIMALWIDTH)) + { + bool saturate = false; + bigllVal = dataconvert::string_to_ll(string(field), saturate); + // TODO MCOL-641 check saturate + } + else if (width <= 8) + { + // errno is initialized and set in convertDecimalString + llVal = Convertor::convertDecimalString( + field, fieldLength, column.scale ); + } } else { @@ -1016,13 +1029,26 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, bufStats.satCount++; // Update min/max range - if (llVal < bufStats.minBufferVal) - bufStats.minBufferVal = llVal; + if (width <= 8) + { + if (llVal < bufStats.minBufferVal) + bufStats.minBufferVal = llVal; - if (llVal > bufStats.maxBufferVal) - bufStats.maxBufferVal = llVal; + if (llVal > bufStats.maxBufferVal) + bufStats.maxBufferVal = llVal; - pVal = &llVal; + pVal = &llVal; + } + else + { + if (bigllVal < bufStats.bigMinBufferVal) + bufStats.bigMinBufferVal = bigllVal; + + if (bigllVal > bufStats.bigMaxBufferVal) + bufStats.bigMaxBufferVal = bigllVal; + + pVal = &bigllVal; + } } else if (column.dataType == CalpontSystemCatalog::TIME) { @@ -1651,11 +1677,24 @@ int BulkLoadBuffer::parseCol(ColumnInfo& columnInfo) // Update CP min/max if this is last row in this extent if ( (fStartRowParser + i) == lastInputRowInExtent ) { - columnInfo.updateCPInfo( lastInputRowInExtent, - bufStats.minBufferVal, - bufStats.maxBufferVal, - columnInfo.column.dataType ); + if (columnInfo.column.width <= 8) + { + columnInfo.updateCPInfo( lastInputRowInExtent, + bufStats.minBufferVal, + bufStats.maxBufferVal, + columnInfo.column.dataType, + columnInfo.column.width ); + } + else + { + columnInfo.updateCPInfo( lastInputRowInExtent, + bufStats.bigMinBufferVal, + bufStats.bigMaxBufferVal, + columnInfo.column.dataType, + columnInfo.column.width ); + } + // TODO MCOL-641 Add support here. if (fLog->isDebug( DEBUG_2 )) { ostringstream oss; @@ -1675,14 +1714,30 @@ int BulkLoadBuffer::parseCol(ColumnInfo& columnInfo) if (isUnsigned(columnInfo.column.dataType) || isCharType(columnInfo.column.dataType)) { - bufStats.minBufferVal = static_cast(MAX_UBIGINT); - bufStats.maxBufferVal = static_cast(MIN_UBIGINT); + if (columnInfo.column.width <= 8) + { + bufStats.minBufferVal = static_cast(MAX_UBIGINT); + bufStats.maxBufferVal = static_cast(MIN_UBIGINT); + } + else + { + bufStats.bigMinBufferVal = -1; + bufStats.bigMaxBufferVal = 0; + } updateCPInfoPendingFlag = false; } else { - bufStats.minBufferVal = MAX_BIGINT; - bufStats.maxBufferVal = MIN_BIGINT; + if (columnInfo.column.width <= 8) + { + bufStats.minBufferVal = MAX_BIGINT; + bufStats.maxBufferVal = MIN_BIGINT; + } + else + { + utils::int128Max(bufStats.bigMinBufferVal); + utils::int128Min(bufStats.bigMaxBufferVal); + } updateCPInfoPendingFlag = false; } } @@ -1690,10 +1745,22 @@ int BulkLoadBuffer::parseCol(ColumnInfo& columnInfo) if (updateCPInfoPendingFlag) { - columnInfo.updateCPInfo( lastInputRowInExtent, - bufStats.minBufferVal, - bufStats.maxBufferVal, - columnInfo.column.dataType ); + if (columnInfo.column.width <= 8) + { + columnInfo.updateCPInfo( lastInputRowInExtent, + bufStats.minBufferVal, + bufStats.maxBufferVal, + columnInfo.column.dataType, + columnInfo.column.width ); + } + else + { + columnInfo.updateCPInfo( lastInputRowInExtent, + bufStats.bigMinBufferVal, + bufStats.bigMaxBufferVal, + columnInfo.column.dataType, + columnInfo.column.width ); + } } if (bufStats.satCount) // @bug 3504: increment row saturation count @@ -1724,6 +1791,7 @@ int BulkLoadBuffer::parseCol(ColumnInfo& columnInfo) Stats::stopParseEvent(WE_STATS_PARSE_COL); #endif + // TODO MCOL-641 Add support here. if (fLog->isDebug( DEBUG_2 )) { ostringstream oss; diff --git a/writeengine/bulk/we_bulkloadbuffer.h b/writeengine/bulk/we_bulkloadbuffer.h index b5e817741..e953c1b82 100644 --- a/writeengine/bulk/we_bulkloadbuffer.h +++ b/writeengine/bulk/we_bulkloadbuffer.h @@ -30,6 +30,7 @@ #include "boost/ptr_container/ptr_vector.hpp" #include "we_columninfo.h" #include "calpontsystemcatalog.h" +#include "dataconvert.h" namespace WriteEngine { @@ -42,17 +43,31 @@ public: int64_t minBufferVal; int64_t maxBufferVal; int64_t satCount; + union + { + int128_t bigMinBufferVal; + int64_t minBufferVal_; + }; + union + { + int128_t bigMaxBufferVal; + int64_t maxBufferVal_; + }; BLBufferStats(ColDataType colDataType) : satCount(0) { if (isUnsigned(colDataType) || isCharType(colDataType)) { minBufferVal = static_cast(MAX_UBIGINT); maxBufferVal = static_cast(MIN_UBIGINT); + bigMinBufferVal = -1; + bigMaxBufferVal = 0; } else { minBufferVal = MAX_BIGINT; maxBufferVal = MIN_BIGINT; + utils::int128Max(bigMinBufferVal); + utils::int128Min(bigMaxBufferVal); } } }; diff --git a/writeengine/bulk/we_colbuf.cpp b/writeengine/bulk/we_colbuf.cpp index 7be21a6ec..caa4f8535 100644 --- a/writeengine/bulk/we_colbuf.cpp +++ b/writeengine/bulk/we_colbuf.cpp @@ -115,12 +115,13 @@ int ColumnBuffer::writeToFile(int startOffset, int writeSize, bool fillUpWEmptie { BlockOp blockOp; newBuf = new unsigned char[BYTE_PER_BLOCK]; - uint64_t EmptyValue = blockOp.getEmptyRowValue(fColInfo->column.dataType, - fColInfo->column.width); + uint8_t* emptyVal = (uint8_t*) alloca(fColInfo->column.width); + blockOp.getEmptyRowValue(fColInfo->column.dataType, + fColInfo->column.width, emptyVal); ::memcpy(static_cast(newBuf), static_cast(fBuffer + startOffset), writeSize); blockOp.setEmptyBuf(newBuf + writeSize, BYTE_PER_BLOCK - writeSize, - EmptyValue, fColInfo->column.width); + emptyVal, fColInfo->column.width); } #ifdef PROFILE Stats::startParseEvent(WE_STATS_WRITE_COL); diff --git a/writeengine/bulk/we_colbufcompressed.cpp b/writeengine/bulk/we_colbufcompressed.cpp index 43534dc75..0e359e885 100644 --- a/writeengine/bulk/we_colbufcompressed.cpp +++ b/writeengine/bulk/we_colbufcompressed.cpp @@ -132,7 +132,7 @@ int ColumnBufferCompressed::resetToBeCompressedColBuf( BlockOp::setEmptyBuf( fToBeCompressedBuffer, IDBCompressInterface::UNCOMPRESSED_INBUF_LEN, - fColInfo->column.emptyVal, + (uint8_t*)&fColInfo->column.emptyVal, fColInfo->column.width ); if (fLog->isDebug( DEBUG_2 )) @@ -317,7 +317,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize, // Start over again loading a new to-be-compressed buffer BlockOp::setEmptyBuf( fToBeCompressedBuffer, IDBCompressInterface::UNCOMPRESSED_INBUF_LEN, - fColInfo->column.emptyVal, + (uint8_t*)&fColInfo->column.emptyVal, fColInfo->column.width ); fToBeCompressedCapacity = @@ -628,7 +628,7 @@ int ColumnBufferCompressed::initToBeCompressedBuffer(long long& startFileOffset) new unsigned char[IDBCompressInterface::UNCOMPRESSED_INBUF_LEN]; BlockOp::setEmptyBuf( fToBeCompressedBuffer, IDBCompressInterface::UNCOMPRESSED_INBUF_LEN, - fColInfo->column.emptyVal, + (uint8_t*)&fColInfo->column.emptyVal, fColInfo->column.width ); bNewBuffer = true; } @@ -743,7 +743,7 @@ int ColumnBufferCompressed::initToBeCompressedBuffer(long long& startFileOffset) { BlockOp::setEmptyBuf( fToBeCompressedBuffer, IDBCompressInterface::UNCOMPRESSED_INBUF_LEN, - fColInfo->column.emptyVal, + (uint8_t*)&fColInfo->column.emptyVal, fColInfo->column.width ); } diff --git a/writeengine/bulk/we_colextinf.cpp b/writeengine/bulk/we_colextinf.cpp index 2e98dd63d..c65158292 100644 --- a/writeengine/bulk/we_colextinf.cpp +++ b/writeengine/bulk/we_colextinf.cpp @@ -70,10 +70,11 @@ void ColExtInf::addFirstEntry( RID lastInputRow, // buffer is flushed), so we will not have an LBID for the 1st buffer for this // extent. //------------------------------------------------------------------------------ -void ColExtInf::addOrUpdateEntry( RID lastInputRow, - int64_t minVal, - int64_t maxVal, - ColDataType colDataType ) +template +void ColExtInf::addOrUpdateEntryTemplate( RID lastInputRow, + T minVal, T maxVal, + ColDataType colDataType, + int width ) { boost::mutex::scoped_lock lock(fMapMutex); @@ -91,30 +92,65 @@ void ColExtInf::addOrUpdateEntry( RID lastInputRow, // If all rows had null value for this column, then minVal will be // MAX_INT and maxVal will be MIN_INT (see getCPInfoForBRM()). - if (iter->second.fMinVal == LLONG_MIN) // init the range + int128_t bigMinValInit; + utils::int128Max(bigMinValInit); + if ((iter->second.fMinVal == LLONG_MIN && width <= 8) || + (iter->second.fbigMinVal == bigMinValInit && width > 8)) // init the range { - iter->second.fMinVal = minVal; - iter->second.fMaxVal = maxVal; + if (width <= 8) + { + iter->second.fMinVal = minVal; + iter->second.fMaxVal = maxVal; + } + else + { + iter->second.fbigMinVal = minVal; + iter->second.fbigMaxVal = maxVal; + } } else // Update the range { if (isUnsigned(colDataType) || isCharType(colDataType)) { - if (static_cast(minVal) - < static_cast(iter->second.fMinVal)) - iter->second.fMinVal = minVal; + if (width <= 8) + { + if (static_cast(minVal) + < static_cast(iter->second.fMinVal)) + iter->second.fMinVal = minVal; - if (static_cast(maxVal) - > static_cast(iter->second.fMaxVal)) - iter->second.fMaxVal = maxVal; + if (static_cast(maxVal) + > static_cast(iter->second.fMaxVal)) + iter->second.fMaxVal = maxVal; + } + else + { + if (static_cast(minVal) + < static_cast(iter->second.fbigMinVal)) + iter->second.fbigMinVal = minVal; + + if (static_cast(maxVal) + > static_cast(iter->second.fbigMaxVal)) + iter->second.fbigMaxVal = maxVal; + } } else { - if (minVal < iter->second.fMinVal) - iter->second.fMinVal = minVal; + if (width <= 8) + { + if (minVal < iter->second.fMinVal) + iter->second.fMinVal = minVal; - if (maxVal > iter->second.fMaxVal) - iter->second.fMaxVal = maxVal; + if (maxVal > iter->second.fMaxVal) + iter->second.fMaxVal = maxVal; + } + else + { + if (minVal < iter->second.fbigMinVal) + iter->second.fbigMinVal = minVal; + + if (maxVal > iter->second.fbigMaxVal) + iter->second.fbigMaxVal = maxVal; + } } } } @@ -178,6 +214,8 @@ void ColExtInf::getCPInfoForBRM( JobColumn column, BRMReporter& brmReporter ) // if applicable (indicating an extent with no non-NULL values). int64_t minVal = iter->second.fMinVal; int64_t maxVal = iter->second.fMaxVal; + int128_t bigMinVal = iter->second.fbigMinVal; + int128_t bigMaxVal = iter->second.fbigMaxVal; if ( bIsChar ) { @@ -198,6 +236,7 @@ void ColExtInf::getCPInfoForBRM( JobColumn column, BRMReporter& brmReporter ) // Log for now; may control with debug flag later //if (fLog->isDebug( DEBUG_1 )) + // TODO MCOL-641 Add support here. { std::ostringstream oss; oss << "Saving CP update for OID-" << fColOid << @@ -232,11 +271,20 @@ void ColExtInf::getCPInfoForBRM( JobColumn column, BRMReporter& brmReporter ) BRM::CPInfoMerge cpInfoMerge; cpInfoMerge.startLbid = iter->second.fLbid; - cpInfoMerge.max = maxVal; - cpInfoMerge.min = minVal; + if (column.width <= 8) + { + cpInfoMerge.max = maxVal; + cpInfoMerge.min = minVal; + } + else + { + cpInfoMerge.bigMax = bigMaxVal; + cpInfoMerge.bigMin = bigMinVal; + } cpInfoMerge.seqNum = -1; // Not used by mergeExtentsMaxMin cpInfoMerge.type = column.dataType; cpInfoMerge.newExtent = iter->second.fNewExtent; + cpInfoMerge.colWidth = column.width; brmReporter.addToCPInfo( cpInfoMerge ); ++iter; diff --git a/writeengine/bulk/we_colextinf.h b/writeengine/bulk/we_colextinf.h index acc1a59bd..60930d8ed 100644 --- a/writeengine/bulk/we_colextinf.h +++ b/writeengine/bulk/we_colextinf.h @@ -40,6 +40,7 @@ #include "brmtypes.h" #include "we_type.h" +#include "dataconvert.h" namespace WriteEngine { @@ -62,14 +63,22 @@ public: ColExtInfEntry() : fLbid(INVALID_LBID), fMinVal(LLONG_MIN), fMaxVal(LLONG_MIN), - fNewExtent(true) { } + fNewExtent(true) + { + utils::int128Min(fbigMaxVal); + utils::int128Max(fbigMinVal); + } // Used to create entry for an existing extent we are going to add data to. ColExtInfEntry(BRM::LBID_t lbid, bool bIsNewExtent) : fLbid(lbid), fMinVal(LLONG_MIN), fMaxVal(LLONG_MIN), - fNewExtent(bIsNewExtent) { } + fNewExtent(bIsNewExtent) + { + utils::int128Min(fbigMaxVal); + utils::int128Max(fbigMinVal); + } // Used to create entry for a new extent, with LBID not yet allocated ColExtInfEntry(int64_t minVal, int64_t maxVal) : @@ -78,6 +87,13 @@ public: fMaxVal(maxVal), fNewExtent(true) { } + // Used to create entry for a new extent, with LBID not yet allocated + ColExtInfEntry(int128_t bigMinVal, int128_t bigMaxVal) : + fLbid(INVALID_LBID), + fNewExtent(true), + fbigMinVal(bigMinVal), + fbigMaxVal(bigMaxVal) { } + // Used to create entry for a new extent, with LBID not yet allocated ColExtInfEntry(uint64_t minVal, uint64_t maxVal) : fLbid(INVALID_LBID), @@ -85,10 +101,27 @@ public: fMaxVal(static_cast(maxVal)), fNewExtent(true) { } + // Used to create entry for a new extent, with LBID not yet allocated + ColExtInfEntry(uint128_t bigMinVal, uint128_t bigMaxVal) : + fLbid(INVALID_LBID), + fNewExtent(true), + fbigMinVal(static_cast(bigMinVal)), + fbigMaxVal(static_cast(bigMaxVal)) { } + BRM::LBID_t fLbid; // LBID for an extent; should be the starting LBID int64_t fMinVal; // minimum value for extent associated with LBID int64_t fMaxVal; // maximum value for extent associated with LBID bool fNewExtent;// is this a new extent + union + { + int128_t fbigMinVal; + int64_t fMinVal_; + }; + union + { + int128_t fbigMaxVal; + int64_t fMaxVal_; + }; }; //------------------------------------------------------------------------------ @@ -122,7 +155,14 @@ public: virtual void addOrUpdateEntry( RID lastInputRow, int64_t minVal, int64_t maxVal, - ColDataType colDataType ) { } + ColDataType colDataType, + int width ) { } + + virtual void addOrUpdateEntry( RID lastInputRow, + int128_t minVal, + int128_t maxVal, + ColDataType colDataType, + int width ) { } virtual void getCPInfoForBRM ( JobColumn column, BRMReporter& brmReporter) { } @@ -173,10 +213,33 @@ public: * @param minVal Minimum value for the latest buffer read * @param maxVal Maximum value for the latest buffer read */ + template + void addOrUpdateEntryTemplate( RID lastInputRow, + T minVal, T maxVal, + ColDataType colDataType, + int width ); + virtual void addOrUpdateEntry( RID lastInputRow, - int64_t minVal, - int64_t maxVal, - ColDataType colDataType ); + int64_t minVal, int64_t maxVal, + ColDataType colDataType, + int width ) + { + addOrUpdateEntryTemplate( lastInputRow, + minVal, maxVal, + colDataType, + width ); + } + + virtual void addOrUpdateEntry( RID lastInputRow, + int128_t minVal, int128_t maxVal, + ColDataType colDataType, + int width ) + { + addOrUpdateEntryTemplate( lastInputRow, + minVal, maxVal, + colDataType, + width ); + } /** @brief Send updated Casual Partition (CP) info to BRM. */ diff --git a/writeengine/bulk/we_columninfo.cpp b/writeengine/bulk/we_columninfo.cpp index 5a9e4afa0..f0e40fb9d 100644 --- a/writeengine/bulk/we_columninfo.cpp +++ b/writeengine/bulk/we_columninfo.cpp @@ -895,7 +895,7 @@ int ColumnInfo::extendColumnOldExtent( } rc = colOp->expandAbbrevColumnExtent( pFile, dbRootNext, - column.emptyVal, column.width); + (uint8_t*)&column.emptyVal, column.width); if (rc != NO_ERROR) { diff --git a/writeengine/bulk/we_columninfo.h b/writeengine/bulk/we_columninfo.h index 15ded5310..f3bb8f23e 100644 --- a/writeengine/bulk/we_columninfo.h +++ b/writeengine/bulk/we_columninfo.h @@ -294,10 +294,11 @@ public: /** @brief Update extent CP information */ + template void updateCPInfo( RID lastInputRow, - int64_t minVal, - int64_t maxVal, - ColDataType colDataType ); + T minVal, T maxVal, + ColDataType colDataType, + int width ); /** @brief Setup initial extent we will begin loading at start of import. * @param dbRoot DBRoot of starting extent @@ -567,13 +568,15 @@ inline unsigned ColumnInfo::rowsPerExtent() return fRowsPerExtent; } +template inline void ColumnInfo::updateCPInfo( RID lastInputRow, - int64_t minVal, - int64_t maxVal, - ColDataType colDataType ) + T minVal, + T maxVal, + ColDataType colDataType, + int width ) { - fColExtInf->addOrUpdateEntry( lastInputRow, minVal, maxVal, colDataType ); + fColExtInf->addOrUpdateEntry( lastInputRow, minVal, maxVal, colDataType, width ); } } // end of namespace diff --git a/writeengine/bulk/we_columninfocompressed.cpp b/writeengine/bulk/we_columninfocompressed.cpp index 25af8bc22..4fdd7f7be 100644 --- a/writeengine/bulk/we_columninfocompressed.cpp +++ b/writeengine/bulk/we_columninfocompressed.cpp @@ -540,7 +540,7 @@ int ColumnInfoCompressed::extendColumnOldExtent( int rc = colOp->fillCompColumnExtentEmptyChunks( curCol.dataFile.fid, curCol.colWidth, - column.emptyVal, + (uint8_t*)&column.emptyVal, curCol.dataFile.fDbRoot, curCol.dataFile.fPartition, curCol.dataFile.fSegment, diff --git a/writeengine/bulk/we_tableinfo.cpp b/writeengine/bulk/we_tableinfo.cpp index cb1a25d9a..06d96dafe 100644 --- a/writeengine/bulk/we_tableinfo.cpp +++ b/writeengine/bulk/we_tableinfo.cpp @@ -69,6 +69,31 @@ const std::string BOLD_STOP = "\033[0;39m"; namespace WriteEngine { +// Helpers +int TableInfo::compareHWMs(const int smallestColumnId, + const int widerColumnId, + const size_t widerColumnWidth, + const std::vector& segFileInfo, + int& colIdx) +{ + int rc = NO_ERROR; + if (widerColumnId < 0) + { + return rc; + } + HWM hwmLo = segFileInfo[smallestColumnId].fLocalHwm * widerColumnWidth; + HWM hwmHi = hwmLo + widerColumnWidth - 1; + + if ((segFileInfo[widerColumnId].fLocalHwm < hwmLo) || + (segFileInfo[widerColumnId].fLocalHwm > hwmHi)) + { + colIdx = widerColumnId; + rc = ERR_BRM_HWMS_OUT_OF_SYNC; + } + return rc; +} + + //------------------------------------------------------------------------------ // Puts the current thread to sleep for the specified number of milliseconds. // (Ex: used to wait for a Read buffer to become available.) @@ -2064,6 +2089,7 @@ int TableInfo::validateColumnHWMs( int byte2First = -1; int byte4First = -1; int byte8First = -1; + int byte16First = -1; // Make sure the HWMs for all 1-byte columns match; same for all 2-byte, // 4-byte, and 8-byte columns as well. @@ -2107,7 +2133,6 @@ int TableInfo::validateColumnHWMs( } case 8: - default: { if (byte8First == -1) byte8First = k; @@ -2115,16 +2140,34 @@ int TableInfo::validateColumnHWMs( k1 = byte8First; break; } - } // end of switch based on column width (1,2,4, or 8) + case 16: + { + if (byte16First == -1) + byte16First = k; + + k1 = byte16First; + break; + } + default: + { + ostringstream oss; + oss << stage << " Unsupported width for" + " OID-" << jobColK.mapOid << + "; column-" << jobColK.colName << + "; width-" << jobColK.width; + fLog->logMsg( oss.str(), ERR_BRM_UNSUPP_WIDTH, MSGLVL_ERROR ); + return ERR_BRM_UNSUPP_WIDTH; + } + } // end of switch based on column width. // Validate HWMs in jobTable if we have it, else use fColumns. const JobColumn& jobColK1 = ( (jobTable != 0) ? jobTable->colList[k1] : fColumns[k1].column ); -//std::cout << "dbg: comparing0 " << stage << " refcol-" << k1 << -// "; wid-" << jobColK1.width << "; hwm-" << segFileInfo[k1].fLocalHwm << -// " col-" << k << -// "; wid-" << jobColK.width << " ; hwm-"< col-" << k << + // "; wid-" << jobColK.width << " ; hwm-"<= 0) -// std::cout << "dbg: cross compare1 " << stage << " col-" << byte1First << -// "; wid-" << ( (jobTable != 0) ? jobTable->colList[byte1First].width : -// fColumns[byte1First].column.width ) << -// "; hwm-" << segFileInfo[byte1First].fLocalHwm << std::endl; + //if (byte1First >= 0) + // std::cout << "dbg: cross compare1 " << stage << " col-" << byte1First << + // "; wid-" << ( (jobTable != 0) ? jobTable->colList[byte1First].width : + // fColumns[byte1First].column.width ) << + // "; hwm-" << segFileInfo[byte1First].fLocalHwm << std::endl; -//if (byte2First >= 0) -// std::cout << "dbg: cross compare2 " << stage << " col-" << byte2First << -// "; wid-" << ( (jobTable != 0) ? jobTable->colList[byte2First].width : -// fColumns[byte2First].column.width ) << -// "; hwm-" << segFileInfo[byte2First].fLocalHwm << std::endl; + //if (byte2First >= 0) + // std::cout << "dbg: cross compare2 " << stage << " col-" << byte2First << + // "; wid-" << ( (jobTable != 0) ? jobTable->colList[byte2First].width : + // fColumns[byte2First].column.width ) << + // "; hwm-" << segFileInfo[byte2First].fLocalHwm << std::endl; -//if (byte4First >= 0) -// std::cout << "dbg: cross compare4 " << stage << " col-" << byte4First << -// "; wid-" << ( (jobTable != 0) ? jobTable->colList[byte4First].width : -// fColumns[byte4First].column.width ) << -// "; hwm-" << segFileInfo[byte4First].fLocalHwm << std::endl; + //if (byte4First >= 0) + // std::cout << "dbg: cross compare4 " << stage << " col-" << byte4First << + // "; wid-" << ( (jobTable != 0) ? jobTable->colList[byte4First].width : + // fColumns[byte4First].column.width ) << + // "; hwm-" << segFileInfo[byte4First].fLocalHwm << std::endl; -//if (byte8First >= 0) -// std::cout << "dbg: cross compare8 " << stage << " col-" << byte8First << -// "; wid-" << ( (jobTable != 0) ? jobTable->colList[byte8First].width : -// fColumns[byte8First].column.width ) << -// "; hwm-" << segFileInfo[byte8First].fLocalHwm << std::endl; + //if (byte8First >= 0) + // std::cout << "dbg: cross compare8 " << stage << " col-" << byte8First << + // "; wid-" << ( (jobTable != 0) ? jobTable->colList[byte8First].width : + // fColumns[byte8First].column.width ) << + // "; hwm-" << segFileInfo[byte8First].fLocalHwm << std::endl; // Validate/compare HWMs given a 1-byte column as a starting point if (byte1First >= 0) @@ -2259,6 +2302,11 @@ int TableInfo::validateColumnHWMs( goto errorCheck; } } + if ((rc = compareHWMs(byte1First, byte16First, 16, + segFileInfo, colIdx) < 0)) + { + goto errorCheck; + } } // Validate/compare HWMs given a 2-byte column as a starting point @@ -2293,6 +2341,13 @@ int TableInfo::validateColumnHWMs( goto errorCheck; } } + if ((rc = compareHWMs(byte2First, byte16First, 16, + segFileInfo, colIdx) < 0)) + { + goto errorCheck; + } + + } // Validate/compare HWMs given a 4-byte column as a starting point @@ -2313,6 +2368,22 @@ int TableInfo::validateColumnHWMs( goto errorCheck; } } + if ((rc = compareHWMs(byte4First, byte16First, 16, + segFileInfo, colIdx) < 0)) + { + goto errorCheck; + } + + } + if (byte8First >= 0) + { + refCol = byte8First; + if ((rc = compareHWMs(byte8First, byte16First, 16, + segFileInfo, colIdx) < 0)) + { + goto errorCheck; + } + } // To avoid repeating this message 6 times in the preceding source code, we diff --git a/writeengine/bulk/we_tableinfo.h b/writeengine/bulk/we_tableinfo.h index eddc552a2..b0c635461 100644 --- a/writeengine/bulk/we_tableinfo.h +++ b/writeengine/bulk/we_tableinfo.h @@ -187,6 +187,13 @@ private: int openTableFile(); // Open data file and set the buffer void reportTotals(double elapsedSec);//Report summary totals void sleepMS(long int ms); // Sleep method + // Compare column HWM with the examplar HWM. + int compareHWMs(const int smallestColumnId, + const int widerColumnId, + const size_t widerColumnWidth, + const std::vector& segFileInfo, + int& colIdx); + int synchronizeAutoInc(); // Sychronize AutoInc in BRM with syscat // Write the list of errors for this table diff --git a/writeengine/server/CMakeLists.txt b/writeengine/server/CMakeLists.txt index c4128df7e..1165dde47 100644 --- a/writeengine/server/CMakeLists.txt +++ b/writeengine/server/CMakeLists.txt @@ -15,7 +15,8 @@ set(WriteEngineServer_SRCS we_cleartablelockcmd.cpp we_cpifeederthread.cpp we_getfilesizes.cpp - ../../utils/common/crashtrace.cpp) + ../../utils/common/crashtrace.cpp + ../../datatypes/mcs_datatype.cpp) add_executable(WriteEngineServer ${WriteEngineServer_SRCS}) diff --git a/writeengine/server/we_ddlcommandproc.cpp b/writeengine/server/we_ddlcommandproc.cpp index 0b557a2c4..d60ff033f 100644 --- a/writeengine/server/we_ddlcommandproc.cpp +++ b/writeengine/server/we_ddlcommandproc.cpp @@ -140,6 +140,7 @@ uint8_t WE_DDLCommandProc::writeSystable(ByteStream& bs, std::string& err) WriteEngine::ColTuple colTuple; WriteEngine::ColStruct colStruct; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; WriteEngine::ColTupleList colTuples; WriteEngine::dictStr dctColTuples; WriteEngine::DctnryStruct dctnryStruct; @@ -212,12 +213,12 @@ uint8_t WE_DDLCommandProc::writeSystable(ByteStream& bs, std::string& err) else if (INIT_COL == column.tableColName.column) { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } else if (NEXT_COL == column.tableColName.column) { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } else if (AUTOINC_COL == column.tableColName.column) { @@ -225,7 +226,7 @@ uint8_t WE_DDLCommandProc::writeSystable(ByteStream& bs, std::string& err) } else { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } colStruct.dataOid = column.oid; @@ -255,6 +256,7 @@ uint8_t WE_DDLCommandProc::writeSystable(ByteStream& bs, std::string& err) } colStructs.push_back(colStruct); + cscColTypeList.push_back(column.colType); oids[colStruct.dataOid] = colStruct.dataOid; //oidsToFlush.push_back(colStruct.dataOid); @@ -293,7 +295,7 @@ uint8_t WE_DDLCommandProc::writeSystable(ByteStream& bs, std::string& err) // TODO: This may be redundant static boost::mutex dbrmMutex; boost::mutex::scoped_lock lk(dbrmMutex); - error = fWEWrapper.insertColumnRec_SYS(txnID, colStructs, colValuesList, + error = fWEWrapper.insertColumnRec_SYS(txnID, cscColTypeList, colStructs, colValuesList, dctnryStructList, dctnryValueList, SYSCOLUMN_BASE); if (error != WriteEngine::NO_ERROR) @@ -392,6 +394,7 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err WriteEngine::ColTuple colTuple; WriteEngine::ColStruct colStruct; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; WriteEngine::ColTupleList colTuples; WriteEngine::dictStr dctColTuples; WriteEngine::DctnryStruct dctnryStruct; @@ -456,10 +459,11 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err if (dataType == CalpontSystemCatalog::DECIMAL || dataType == CalpontSystemCatalog::UDECIMAL) { - if (colDefPtr->fType->fPrecision > 18) //@Bug 5717 precision cannot be over 18. + // WIP MCOL-641 + if (colDefPtr->fType->fPrecision > 38) // precision cannot be over 38. { ostringstream os; - os << "Syntax error: The maximum precision (total number of digits) that can be specified is 18"; + os << "Syntax error: The maximum precision (total number of digits) that can be specified is 38"; throw std::runtime_error(os.str()); } else if (colDefPtr->fType->fPrecision < colDefPtr->fType->fScale) @@ -506,6 +510,16 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err throw std::runtime_error(os.str()); } + else if (dataType == CalpontSystemCatalog::BINARY + && ! (colDefPtr->fType->fLength == 16 + || colDefPtr->fType->fLength == 32)) + { + ostringstream os; + os << "binary length may not be other than 16 or 32"; + throw std::runtime_error(os.str()); + } + + unsigned int i = 0; column_iterator = columns.begin(); @@ -574,7 +588,7 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err else { tmpStr = ""; - //colTuple.data = getNullValueForType(column.colType); + //colTuple.data = column.colType.getNullValueForType(); } } @@ -615,16 +629,16 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err } else { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } } else if (LISTOBJID_COL == column.tableColName.column) { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } else if (TREEOBJID_COL == column.tableColName.column) { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } else if (MINVAL_COL == column.tableColName.column) { @@ -650,7 +664,7 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err } else { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } colStruct.dataOid = column.oid; @@ -691,6 +705,7 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err { colStructs.push_back(colStruct); dctnryStructList.push_back (dctnryStruct); + cscColTypeList.push_back(column.colType); } colList[i].push_back(colTuple); @@ -722,7 +737,7 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err } //fWEWrapper.setDebugLevel(WriteEngine::DEBUG_3); - error = fWEWrapper.insertColumnRec_SYS(txnID, colStructs, colValuesList, + error = fWEWrapper.insertColumnRec_SYS(txnID, cscColTypeList, colStructs, colValuesList, dctnryStructList, dctnryValueList, SYSCOLUMN_BASE); if (idbdatafile::IDBPolicy::useHdfs()) @@ -808,6 +823,7 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string& err) WriteEngine::ColStruct colStruct; WriteEngine::ColTuple colTuple; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; WriteEngine::ColTupleList colTuples; WriteEngine::DctColTupleList dctColTuples; WriteEngine::ColValueList colValuesList; @@ -966,7 +982,7 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string& err) else { tmpStr = ""; - //colTuple.data = getNullValueForType(column.colType); + //colTuple.data = column.colType.getNullValueForType(); } } @@ -1007,16 +1023,16 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string& err) } else { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } } else if (LISTOBJID_COL == column.tableColName.column) { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } else if (TREEOBJID_COL == column.tableColName.column) { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } else if (MINVAL_COL == column.tableColName.column) { @@ -1042,7 +1058,7 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string& err) } else { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } colStruct.dataOid = column.oid; @@ -1080,6 +1096,7 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string& err) colStructs.push_back(colStruct); dctnryStructList.push_back (dctnryStruct); + cscColTypeList.push_back(column.colType); colList[i].push_back(colTuple); //colList.push_back(WriteEngine::ColTupleList()); //colList.back().push_back(colTuple); @@ -1107,7 +1124,7 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string& err) fWEWrapper.startTransaction(txnID); int rc1 = 0; - error = fWEWrapper.insertColumnRec_SYS(txnID, colStructs, colValuesList, + error = fWEWrapper.insertColumnRec_SYS(txnID, cscColTypeList, colStructs, colValuesList, dctnryStructList, dctnryValueList, SYSCOLUMN_BASE); if (idbdatafile::IDBPolicy::useHdfs()) @@ -1357,7 +1374,9 @@ uint8_t WE_DDLCommandProc::deleteSyscolumn(ByteStream& bs, std::string& err) WriteEngine::ColStruct colStruct; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colExtentsStruct; + std::vector colExtentsColType; std::vector colValuesList; WriteEngine::RIDList ridList; std::vector ridLists; @@ -1392,17 +1411,19 @@ uint8_t WE_DDLCommandProc::deleteSyscolumn(ByteStream& bs, std::string& err) oids[colStruct.dataOid] = colStruct.dataOid; //oidsToFlush.push_back(colStruct.dataOid); colStructs.push_back(colStruct); + cscColTypeList.push_back(column.colType); ++column_iterator; } colExtentsStruct.push_back(colStructs); + colExtentsColType.push_back(cscColTypeList); ridLists.push_back(ridList); if (0 != colStructs.size() && 0 != ridLists[0].size()) { - int error = fWEWrapper.deleteRow(txnID, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); + int error = fWEWrapper.deleteRow(txnID, colExtentsColType, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); int rc1 = 0; @@ -1497,7 +1518,9 @@ uint8_t WE_DDLCommandProc::deleteSyscolumnRow(ByteStream& bs, std::string& err) WriteEngine::ColStruct colStruct; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colExtentsStruct; + std::vector colExtentsColType; std::vector colValuesList; WriteEngine::RIDList ridList; std::vector ridLists; @@ -1526,17 +1549,20 @@ uint8_t WE_DDLCommandProc::deleteSyscolumnRow(ByteStream& bs, std::string& err) oids[colStruct.dataOid] = colStruct.dataOid; //oidsToFlush.push_back(colStruct.dataOid); colStructs.push_back(colStruct); + cscColTypeList.push_back(column.colType); ++column_iterator; } colExtentsStruct.push_back(colStructs); + colExtentsColType.push_back(cscColTypeList); ridLists.push_back(ridList); if (0 != colStructs.size() && 0 != ridLists[0].size()) { - int error = fWEWrapper.deleteRow(txnID, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); + int error = fWEWrapper.deleteRow(txnID, colExtentsColType, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); + int rc1 = 0; if (idbdatafile::IDBPolicy::useHdfs()) @@ -1633,7 +1659,9 @@ uint8_t WE_DDLCommandProc::deleteSystable(ByteStream& bs, std::string& err) WriteEngine::ColStruct colStruct; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colExtentsStruct; + std::vector colExtentsColType; std::vector colValuesList; WriteEngine::RIDList ridList; std::vector ridLists; @@ -1661,17 +1689,20 @@ uint8_t WE_DDLCommandProc::deleteSystable(ByteStream& bs, std::string& err) oids[colStruct.dataOid] = colStruct.dataOid; //oidsToFlush.push_back(colStruct.dataOid); colStructs.push_back(colStruct); + cscColTypeList.push_back(column.colType); ++column_iterator; } colExtentsStruct.push_back(colStructs); + colExtentsColType.push_back(cscColTypeList); ridLists.push_back(ridList); if (0 != colStructs.size() && 0 != ridLists[0].size()) { - int error = fWEWrapper.deleteRow(txnID, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); + int error = fWEWrapper.deleteRow(txnID, colExtentsColType, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); + int rc1 = 0; if (idbdatafile::IDBPolicy::useHdfs()) @@ -1744,7 +1775,9 @@ uint8_t WE_DDLCommandProc::deleteSystables(ByteStream& bs, std::string& err) systemCatalogPtr->identity(CalpontSystemCatalog::EC); WriteEngine::ColStruct colStruct; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colExtentsStruct; + std::vector colExtentsColType; std::vector colValuesList; WriteEngine::RIDList ridList; std::vector ridLists; @@ -1793,16 +1826,19 @@ uint8_t WE_DDLCommandProc::deleteSystables(ByteStream& bs, std::string& err) oids[colStruct.dataOid] = colStruct.dataOid; //oidsToFlush.push_back(colStruct.dataOid); colStructs.push_back(colStruct); + cscColTypeList.push_back(column.colType); ++column_iterator; } colExtentsStruct.push_back(colStructs); + colExtentsColType.push_back(cscColTypeList); ridLists.push_back(ridList); { - int error = fWEWrapper.deleteRow(txnID, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); + int error = fWEWrapper.deleteRow(txnID, colExtentsColType, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); + int rc1 = 0; if (idbdatafile::IDBPolicy::useHdfs()) @@ -1856,7 +1892,9 @@ uint8_t WE_DDLCommandProc::deleteSystables(ByteStream& bs, std::string& err) CalpontSystemCatalog::RIDList colRidList = systemCatalogPtr->columnRIDs(userTableName); colStructs.clear(); + cscColTypeList.clear(); colExtentsStruct.clear(); + colExtentsColType.clear(); colValuesList.clear(); ridList.clear(); ridLists.clear(); @@ -1891,17 +1929,20 @@ uint8_t WE_DDLCommandProc::deleteSystables(ByteStream& bs, std::string& err) colStructs.push_back(colStruct); oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column.colType); //oidsToFlush.push_back(colStruct.dataOid); ++column_iterator; } colExtentsStruct.push_back(colStructs); + colExtentsColType.push_back(cscColTypeList); ridLists.push_back(ridList); if (0 != colStructs.size() && 0 != ridLists[0].size()) { - int error = fWEWrapper.deleteRow(txnID, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); + int error = fWEWrapper.deleteRow(txnID, colExtentsColType, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); + int rc1 = 0; if (idbdatafile::IDBPolicy::useHdfs()) @@ -2024,6 +2065,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnAuto(ByteStream& bs, std::string& err) WriteEngine::ColValueList colValuesList; WriteEngine::ColTupleList aColList; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colOldValuesList; std::map oids; //std::vector oidsToFlush; @@ -2064,6 +2106,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnAuto(ByteStream& bs, std::string& err) oids[colStruct.dataOid] = colStruct.dataOid; //oidsToFlush.push_back(colStruct.dataOid); dctnryStructList.push_back(dctnryStruct); + cscColTypeList.push_back(column.colType); for (unsigned int i = 0; i < roList.size(); i++) { @@ -2072,6 +2115,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnAuto(ByteStream& bs, std::string& err) colValuesList.push_back(aColList); std::vector colExtentsStruct; + std::vector colExtentsColType; std::vector dctnryExtentsStruct; std::vector extentsinfo; extentInfo aExtentinfo; @@ -2114,13 +2158,14 @@ uint8_t WE_DDLCommandProc::updateSyscolumnAuto(ByteStream& bs, std::string& err) colExtentsStruct.push_back(colStructs); dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); } // call the write engine to update the row if (idbdatafile::IDBPolicy::useHdfs()) fWEWrapper.startTransaction(txnID); - rc = fWEWrapper.updateColumnRec(txnID, colExtentsStruct, colValuesList, colOldValuesList, + rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); if (rc != NO_ERROR) @@ -2215,6 +2260,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnNextvalCol(ByteStream& bs, std::string WriteEngine::ColValueList colValuesList; WriteEngine::ColTupleList aColList; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colOldValuesList; std::map oids; //std::vector oidsToFlush; @@ -2248,6 +2294,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnNextvalCol(ByteStream& bs, std::string //oidsToFlush.push_back(colStruct.dataOid); colStructs.push_back(colStruct); dctnryStructList.push_back(dctnryStruct); + cscColTypeList.push_back(column.colType); for (unsigned int i = 0; i < roList.size(); i++) { @@ -2285,6 +2332,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnNextvalCol(ByteStream& bs, std::string std::vector ridLists; std::vector colExtentsStruct; + std::vector colExtentsColType; std::vector dctnryExtentsStruct; ridLists.push_back(ridList); @@ -2303,13 +2351,14 @@ uint8_t WE_DDLCommandProc::updateSyscolumnNextvalCol(ByteStream& bs, std::string colExtentsStruct.push_back(colStructs); dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); } // call the write engine to update the row fWEWrapper.setTransId(txnID); fWEWrapper.startTransaction(txnID); - rc = fWEWrapper.updateColumnRec(txnID, colExtentsStruct, colValuesList, colOldValuesList, + rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); if (rc != NO_ERROR) @@ -2385,6 +2434,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnTablename(ByteStream& bs, std::string& WriteEngine::ColValueList colValuesList; WriteEngine::ColTupleList aColList; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colOldValuesList; tableName.schema = CALPONT_SCHEMA; tableName.table = SYSCOLUMN_TABLE; @@ -2452,6 +2502,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnTablename(ByteStream& bs, std::string& colStructs.push_back(colStruct); dctnryStructList.push_back(dctnryStruct); + cscColTypeList.push_back(column.colType); for (unsigned int i = 0; i < roList.size(); i++) { @@ -2474,6 +2525,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnTablename(ByteStream& bs, std::string& std::vector extentsinfo; extentInfo aExtentinfo; std::vector colExtentsStruct; + std::vector colExtentsColType; std::vector dctnryExtentsStruct; for (unsigned int i = 0; i < roList.size(); i++) @@ -2510,6 +2562,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnTablename(ByteStream& bs, std::string& colExtentsStruct.push_back(colStructs); dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); } // call the write engine to update the row @@ -2518,7 +2571,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnTablename(ByteStream& bs, std::string& fWEWrapper.setBulkFlag(false); fWEWrapper.startTransaction(txnID); - rc = fWEWrapper.updateColumnRec(txnID, colExtentsStruct, colValuesList, colOldValuesList, + rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); if (rc != NO_ERROR) @@ -2614,6 +2667,7 @@ uint8_t WE_DDLCommandProc::updateSystableAuto(ByteStream& bs, std::string& err) WriteEngine::ColValueList colValuesList; WriteEngine::ColTupleList aColList; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colOldValuesList; std::map oids; //std::vector oidsToFlush; @@ -2647,12 +2701,14 @@ uint8_t WE_DDLCommandProc::updateSystableAuto(ByteStream& bs, std::string& err) } colStructs.push_back(colStruct); + cscColTypeList.push_back(column.colType); oids[colStruct.dataOid] = colStruct.dataOid; //oidsToFlush.push_back(colStruct.dataOid); dctnryStructList.push_back(dctnryStruct); aColList.push_back(colTuple); colValuesList.push_back(aColList); std::vector colExtentsStruct; + std::vector colExtentsColType; std::vector dctnryExtentsStruct; @@ -2690,6 +2746,7 @@ uint8_t WE_DDLCommandProc::updateSystableAuto(ByteStream& bs, std::string& err) } colExtentsStruct.push_back(colStructs); + colExtentsColType.push_back(cscColTypeList); dctnryExtentsStruct.push_back(dctnryStructList); } @@ -2699,7 +2756,7 @@ uint8_t WE_DDLCommandProc::updateSystableAuto(ByteStream& bs, std::string& err) fWEWrapper.setBulkFlag(false); fWEWrapper.startTransaction(txnID); - rc = fWEWrapper.updateColumnRec(txnID, colExtentsStruct, colValuesList, colOldValuesList, + rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); if (rc != NO_ERROR) @@ -2793,6 +2850,7 @@ uint8_t WE_DDLCommandProc::updateSystableTablename(ByteStream& bs, std::string& WriteEngine::ColValueList colValuesList; WriteEngine::ColTupleList aColList; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colOldValuesList; std::map oids; //std::vector oidsToFlush; @@ -2842,6 +2900,7 @@ uint8_t WE_DDLCommandProc::updateSystableTablename(ByteStream& bs, std::string& colStructs.push_back(colStruct); dctnryStructList.push_back(dctnryStruct); oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column.colType); //oidsToFlush.push_back(colStruct.dataOid); if (dctnryStruct.dctnryOid > 0) @@ -2854,6 +2913,7 @@ uint8_t WE_DDLCommandProc::updateSystableTablename(ByteStream& bs, std::string& colValuesList.push_back(aColList); std::vector colExtentsStruct; std::vector dctnryExtentsStruct; + std::vector colExtentsColType; dctColList = dictTuple; dctRowList.push_back(dctColList); @@ -2889,6 +2949,7 @@ uint8_t WE_DDLCommandProc::updateSystableTablename(ByteStream& bs, std::string& colExtentsStruct.push_back(colStructs); dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); } // call the write engine to update the row @@ -2897,7 +2958,7 @@ uint8_t WE_DDLCommandProc::updateSystableTablename(ByteStream& bs, std::string& fWEWrapper.setBulkFlag(false); fWEWrapper.startTransaction(txnID); - rc = fWEWrapper.updateColumnRec(txnID, colExtentsStruct, colValuesList, colOldValuesList, + rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); if (rc != NO_ERROR) @@ -3021,6 +3082,7 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& WriteEngine::ColValueList colValuesList; WriteEngine::ColTupleList aColList; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colOldValuesList; std::map oids; //std::vector oidsToFlush; @@ -3079,6 +3141,7 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& colStructs.push_back(colStruct); dctnryStructList.push_back(dctnryStruct); oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column.colType); //oidsToFlush.push_back(colStruct.dataOid); if (dctnryStruct.dctnryOid > 0) @@ -3091,6 +3154,7 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& colValuesList.push_back(aColList); std::vector colExtentsStruct; std::vector dctnryExtentsStruct; + std::vector colExtentsColType; dctColList = dictTuple; dctRowList.push_back(dctColList); @@ -3126,6 +3190,7 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& colExtentsStruct.push_back(colStructs); dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); } // call the write engine to update the row @@ -3134,7 +3199,7 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& fWEWrapper.setBulkFlag(false); fWEWrapper.startTransaction(txnID); - rc = fWEWrapper.updateColumnRec(txnID, colExtentsStruct, colValuesList, colOldValuesList, + rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); if (rc != NO_ERROR) @@ -3196,6 +3261,7 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& colValuesList.clear(); aColList.clear(); colStructs.clear(); + cscColTypeList.clear(); colOldValuesList.clear(); oids.clear(); tableName.schema = CALPONT_SCHEMA; @@ -3276,6 +3342,7 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& colStructs.push_back(colStruct); dctnryStructList.push_back(dctnryStruct); + cscColTypeList.push_back(column.colType); for (unsigned int i = 0; i < roList.size(); i++) { @@ -3296,6 +3363,7 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& dctnryValueList.push_back(dctRowList); extentsinfo.clear(); colExtentsStruct.clear(); + colExtentsColType.clear(); dctnryExtentsStruct.clear(); oid = 1021; @@ -3333,10 +3401,11 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& colExtentsStruct.push_back(colStructs); dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); } // call the write engine to update the row - rc = fWEWrapper.updateColumnRec(txnID, colExtentsStruct, colValuesList, colOldValuesList, + rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); if (rc != NO_ERROR) @@ -3470,11 +3539,13 @@ uint8_t WE_DDLCommandProc::updateSyscolumnColumnposCol(messageqcpp::ByteStream& WriteEngine::ColStruct colStruct; WriteEngine::DctnryStructList dctnryStructList; WriteEngine::DctnryValueList dctnryValueList; + WriteEngine::CSCTypesList cscColTypeList; + CalpontSystemCatalog::ColType colType; //Build column structure for COLUMNPOS_COL - colStruct.dataOid = OID_SYSCOLUMN_COLUMNPOS; - colStruct.colWidth = 4; + colType.columnOID = colStruct.dataOid = OID_SYSCOLUMN_COLUMNPOS; + colType.colWidth = colStruct.colWidth = 4; colStruct.tokenFlag = false; - colStruct.colDataType = CalpontSystemCatalog::INT; + colType.colDataType = colStruct.colDataType = CalpontSystemCatalog::INT; colStruct.fColDbRoot = dbRoot; if (idbdatafile::IDBPolicy::useHdfs()) @@ -3483,9 +3554,10 @@ uint8_t WE_DDLCommandProc::updateSyscolumnColumnposCol(messageqcpp::ByteStream& } colStructs.push_back(colStruct); + cscColTypeList.push_back(colType); oids[colStruct.dataOid] = colStruct.dataOid; //oidsToFlush.push_back(colStruct.dataOid); - rc = fWEWrapper.updateColumnRecs( txnID, colStructs, colValuesList, ridList, SYSCOLUMN_BASE ); + rc = fWEWrapper.updateColumnRecs( txnID, cscColTypeList, colStructs, colValuesList, ridList, SYSCOLUMN_BASE ); } int rc1 = 0; @@ -3574,14 +3646,14 @@ uint8_t WE_DDLCommandProc::fillNewColumn(ByteStream& bs, std::string& err) colType.scale = scale; colType.precision = precision; bool pushWarning = false; - defaultVal.data = DataConvert::convertColumnData(colType, defaultValStr, pushWarning, timeZone, isNULL, false, false); + defaultVal.data = colType.convertColumnData(defaultValStr, pushWarning, timeZone, isNULL, false, false); fWEWrapper.setTransId(txnID); fWEWrapper.setIsInsert(true); fWEWrapper.setBulkFlag(true); std::map oids; oids[dataOid] = dataOid; oids[refColOID] = refColOID; - rc = fWEWrapper.fillColumn(txnID, dataOid, dataType, dataWidth, defaultVal, refColOID, refColDataType, + rc = fWEWrapper.fillColumn(txnID, dataOid, colType, defaultVal, refColOID, refColDataType, refColWidth, refCompressionType, isNULL, compressionType, defaultValStr, dictOid, autoincrement); if ( rc != 0 ) @@ -4152,6 +4224,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnSetDefault(messageqcpp::ByteStream& bs WriteEngine::ColValueList colValuesList; WriteEngine::ColTupleList aColList1; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colOldValuesList; WriteEngine::DctnryStructList dctnryStructList; WriteEngine::DctnryValueList dctnryValueList; @@ -4257,6 +4330,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnSetDefault(messageqcpp::ByteStream& bs colStructs.push_back(colStruct); oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column.colType); //oidsToFlush.push_back(colStruct.dataOid); if (dctnryStruct.dctnryOid > 0) @@ -4288,6 +4362,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnSetDefault(messageqcpp::ByteStream& bs std::vector colExtentsStruct; std::vector dctnryExtentsStruct; + std::vector colExtentsColType; std::vector ridLists; ridLists.push_back(ridList); @@ -4318,11 +4393,12 @@ uint8_t WE_DDLCommandProc::updateSyscolumnSetDefault(messageqcpp::ByteStream& bs colExtentsStruct.push_back(colStructs); dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); } // call the write engine to update the row - if (NO_ERROR != fWEWrapper.updateColumnRec(txnID, colExtentsStruct, colValuesList, colOldValuesList, + if (NO_ERROR != fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE)) { err = "WE: Update failed on: " + atableName.table; @@ -4428,6 +4504,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& WriteEngine::ColValueList colValuesList; WriteEngine::ColTupleList aColList1; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colOldValuesList; std::map oids; //std::vector oidsToFlush; @@ -4546,6 +4623,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& colStructs.push_back(colStruct); oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column1.colType); //oidsToFlush.push_back(colStruct.dataOid); if (dctnryStruct.dctnryOid > 0) @@ -4583,6 +4661,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& colStructs.push_back(colStruct); oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column2.colType); //oidsToFlush.push_back(colStruct.dataOid); if (dctnryStruct.dctnryOid > 0) @@ -4616,6 +4695,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& colStructs.push_back(colStruct); oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column3.colType); //oidsToFlush.push_back(colStruct.dataOid); if (dctnryStruct.dctnryOid > 0) @@ -4650,6 +4730,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& colStructs.push_back(colStruct); oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column4.colType); //oidsToFlush.push_back(colStruct.dataOid); if (dctnryStruct.dctnryOid > 0) @@ -4759,6 +4840,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& colStructs.push_back(colStruct); dctnryStructList.push_back(dctnryStruct); oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column5.colType); //oidsToFlush.push_back(colStruct.dataOid); if (dctnryStruct.dctnryOid > 0) @@ -4786,6 +4868,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& dctRowList.push_back(dctColList); dctnryValueList.push_back(dctRowList); std::vector colExtentsStruct; + std::vector colExtentsColType; std::vector dctnryExtentsStruct; std::vector ridLists; ridLists.push_back(ridList); @@ -4817,10 +4900,11 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& colExtentsStruct.push_back(colStructs); dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); } // call the write engine to update the row - if (NO_ERROR != fWEWrapper.updateColumnRec(txnID, colExtentsStruct, colValuesList, colOldValuesList, + if (NO_ERROR != fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE)) { err = "WE: Update failed on: " + atableName.table; diff --git a/writeengine/server/we_ddlcommon.h b/writeengine/server/we_ddlcommon.h index 9fcc06f7b..b1741a440 100644 --- a/writeengine/server/we_ddlcommon.h +++ b/writeengine/server/we_ddlcommon.h @@ -143,239 +143,6 @@ inline void getColumnsForTable(uint32_t sessionID, std::string schema, std::str } -inline boost::any getNullValueForType(const execplan::CalpontSystemCatalog::ColType& colType) -{ - boost::any value; - - switch (colType.colDataType) - { - case execplan::CalpontSystemCatalog::BIT: - break; - - case execplan::CalpontSystemCatalog::TINYINT: - { - char tinyintvalue = joblist::TINYINTNULL; - value = tinyintvalue; - - } - break; - - case execplan::CalpontSystemCatalog::UTINYINT: - { - uint8_t tinyintvalue = joblist::UTINYINTNULL; - value = tinyintvalue; - - } - break; - - case execplan::CalpontSystemCatalog::SMALLINT: - { - short smallintvalue = joblist::SMALLINTNULL; - value = smallintvalue; - } - break; - - case execplan::CalpontSystemCatalog::USMALLINT: - { - uint16_t smallintvalue = joblist::USMALLINTNULL; - value = smallintvalue; - } - break; - - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - { - int intvalue = joblist::INTNULL; - value = intvalue; - } - break; - - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - { - uint32_t intvalue = joblist::UINTNULL; - value = intvalue; - } - break; - - case execplan::CalpontSystemCatalog::BIGINT: - { - long long bigint = joblist::BIGINTNULL; - value = bigint; - } - break; - - case execplan::CalpontSystemCatalog::UBIGINT: - { - uint64_t bigint = joblist::UBIGINTNULL; - value = bigint; - } - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - if (colType.colWidth <= execplan::CalpontSystemCatalog::FOUR_BYTE) - { - short smallintvalue = joblist::SMALLINTNULL; - value = smallintvalue; - } - else if (colType.colWidth <= 9) - { - int intvalue = joblist::INTNULL; - value = intvalue; - } - else if (colType.colWidth <= 18) - { - long long eightbyte = joblist::BIGINTNULL; - value = eightbyte; - } - else - { - WriteEngine::Token nullToken; - value = nullToken; - } - } - break; - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - uint32_t jlfloatnull = joblist::FLOATNULL; - float* fp = reinterpret_cast(&jlfloatnull); - value = *fp; - } - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - uint64_t jldoublenull = joblist::DOUBLENULL; - double* dp = reinterpret_cast(&jldoublenull); - value = *dp; - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int d = joblist::DATENULL; - value = d; - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - long long d = joblist::DATETIMENULL; - value = d; - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - long long d = joblist::TIMENULL; - value = d; - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - long long d = joblist::TIMESTAMPNULL; - value = d; - } - break; - - case execplan::CalpontSystemCatalog::CHAR: - { - std::string charnull; - - if (colType.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - { - //charnull = joblist::CHAR1NULL; - charnull = "\376"; - value = charnull; - } - else if (colType.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) - { - //charnull = joblist::CHAR2NULL; - charnull = "\377\376"; - value = charnull; - } - else if (colType.colWidth <= execplan::CalpontSystemCatalog::FOUR_BYTE) - { - //charnull = joblist::CHAR4NULL; - charnull = "\377\377\377\376"; - value = charnull; - } - else - { - WriteEngine::Token nullToken; - value = nullToken; - } - - } - break; - - case execplan::CalpontSystemCatalog::VARCHAR: - { - std::string charnull; - - if (colType.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - { - //charnull = joblist::CHAR2NULL; - charnull = "\377\376"; - value = charnull; - } - else if (colType.colWidth < execplan::CalpontSystemCatalog::FOUR_BYTE) - { - //charnull = joblist::CHAR4NULL; - charnull = "\377\377\377\376"; - value = charnull; - } - else - { - WriteEngine::Token nullToken; - value = nullToken; - } - - } - break; - - case execplan::CalpontSystemCatalog::BLOB: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::VARBINARY: - { - std::string charnull; - - if (colType.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - { - //charnull = joblist::CHAR2NULL; - charnull = "\377\376"; - value = charnull; - } - else if (colType.colWidth < execplan::CalpontSystemCatalog::FOUR_BYTE) - { - //charnull = joblist::CHAR4NULL; - charnull = "\377\377\377\376"; - value = charnull; - } - else - { - WriteEngine::Token nullToken; - value = nullToken; - } - - } - break; - - - default: - throw std::runtime_error("getNullValueForType: unkown column data type"); - break; - - } - - return value; -} inline int convertDataType(int dataType) { @@ -495,6 +262,10 @@ inline int convertDataType(int dataType) case ddlpackage::DDL_UNSIGNED_DOUBLE: calpontDataType = execplan::CalpontSystemCatalog::UDOUBLE; break; + + case ddlpackage::DDL_BINARY: + calpontDataType = execplan::CalpontSystemCatalog::BINARY; + break; default: throw runtime_error("Unsupported datatype!"); diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 2592e58c9..0753ac7b8 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -1,5 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2016 MariaDB Corporation + Copyright (C) 2016-2019 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -51,8 +51,8 @@ using namespace BRM; #include "cacheutils.h" #include "IDBDataFile.h" #include "IDBPolicy.h" - #include "checks.h" +#include "columnwidth.h" namespace WriteEngine { @@ -116,6 +116,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: RowList rows = tablePtr->get_RowList(); WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; WriteEngine::DctnryStructList dctnryStructList; WriteEngine::DctnryValueList dctnryValueList; WriteEngine::ColValueList colValuesList; @@ -139,12 +140,18 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: { Row* rowPtr = rows.at(0); ColumnList columns = rowPtr->get_ColumnList(); + unsigned int numcols = rowPtr->get_NumberOfColumns(); + cscColTypeList.reserve(numcols); + // WIP + // We presume that DictCols number is low + colStructs.reserve(numcols); ColumnList::const_iterator column_iterator = columns.begin(); while (column_iterator != columns.end()) { DMLColumn* columnPtr = *column_iterator; tableColName.column = columnPtr->get_Name(); + // WIP MCOL-641 replace with getColRidsOidsTypes() CalpontSystemCatalog::ROPair roPair = systemCatalogPtr->columnRID(tableColName); CalpontSystemCatalog::OID oid = systemCatalogPtr->lookupOID(tableColName); @@ -161,8 +168,9 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: colStruct.fCompressionType = colType.compressionType; // Token - if ( isDictCol(colType) ) + if (isDictCol(colType) ) { + // WIP Hardcoded value colStruct.colWidth = 8; colStruct.tokenFlag = true; } @@ -190,11 +198,11 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: colStructs.push_back(colStruct); dctnryStructList.push_back(dctnryStruct); + cscColTypeList.push_back(colType); ++column_iterator; } - unsigned int numcols = rowPtr->get_NumberOfColumns(); std::string tmpStr(""); for (unsigned int i = 0; i < numcols; i++) @@ -210,6 +218,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: const DMLColumn* columnPtr = rowPtr->get_ColumnAt(i); tableColName.column = columnPtr->get_Name(); + // WIP MCOL-641 remove these calls CalpontSystemCatalog::OID oid = systemCatalogPtr->lookupOID(tableColName); CalpontSystemCatalog::ColType colType; @@ -303,6 +312,8 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: { try { + // WIP What if we combine this and previous loop and fail + // after get nextAIValue ? nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); fDbrm.startAISequence(oid, nextVal, colType.colWidth, colType.colDataType); } @@ -359,7 +370,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: try { - datavalue = DataConvert::convertColumnData(colType, indata, pushWarning, insertPkg.get_TimeZone(), isNULL, false, false); + datavalue = colType.convertColumnData(indata, pushWarning, insertPkg.get_TimeZone(), isNULL, false, false); } catch (exception&) { @@ -375,7 +386,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: return rc; } - if ( pushWarning) + if (pushWarning) { if (!isWarningSet) isWarningSet = true; @@ -412,14 +423,16 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: // call the write engine to write the rows int error = NO_ERROR; - //fWriteEngine.setDebugLevel(WriteEngine::DEBUG_3); - //cout << "inserting a row with transaction id " << txnid.id << endl; + // MCOL-641 WIP + fWEWrapper.setDebugLevel(WriteEngine::DEBUG_3); + cout << "inserting a row with transaction id " << txnid.id << endl; fWEWrapper.setIsInsert(true); fWEWrapper.setBulkFlag(true); fWEWrapper.setTransId(txnid.id); //For hdfs use only uint32_t tblOid = tableRoPair.objnum; + // WIP are we saving HDFS? if (idbdatafile::IDBPolicy::useHdfs()) { @@ -523,7 +536,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: if (colValuesList[0].size() > 0) { if (NO_ERROR != - (error = fWEWrapper.insertColumnRec_Single(txnid.id, colStructs, colValuesList, dctnryStructList, dicStringList, tableRoPair.objnum))) + (error = fWEWrapper.insertColumnRec_Single(txnid.id, cscColTypeList, colStructs, colValuesList, dctnryStructList, dicStringList, tableRoPair.objnum))) { if (error == ERR_BRM_DEAD_LOCK) { @@ -531,7 +544,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: WErrorCodes ec; err = ec.errorString(error); } - else if ( error == ERR_BRM_VB_OVERFLOW ) + else if (error == ERR_BRM_VB_OVERFLOW) { rc = dmlpackageprocessor::DMLPackageProcessor::VB_OVERFLOW_ERROR; err = IDBErrorInfo::instance()->errorMsg(ERR_VERSIONBUFFER_OVERFLOW); @@ -546,9 +559,9 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: } std::map oids; - std::vector oidsToFlush; + std::vector oidsToFlush; - for ( unsigned i = 0; i < colStructs.size(); i++) + for (unsigned i = 0; i < colStructs.size(); i++) { oids[colStructs[i].dataOid] = colStructs[i].dataOid; oidsToFlush.push_back(colStructs[i].dataOid); @@ -802,24 +815,19 @@ uint8_t WE_DMLCommandProc::rollbackVersion(ByteStream& bs, std::string& err) uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId) { int rc = 0; - //cout << "processBatchInsert received bytestream length " << bs.length() << endl; InsertDMLPackage insertPkg; ByteStream::quadbyte tmp32; bs >> tmp32; - //cout << "processBatchInsert got transaction id " << tmp32 << endl; bs >> PMId; - //cout << "processBatchInsert gor PMId " << PMId << endl; insertPkg.read( bs); uint32_t sessionId = insertPkg.get_SessionID(); - //cout << " processBatchInsert for session " << sessionId << endl; DMLTable* tablePtr = insertPkg.get_Table(); bool isAutocommitOn = insertPkg.get_isAutocommitOn(); if (idbdatafile::IDBPolicy::useHdfs()) isAutocommitOn = true; - //cout << "This session isAutocommitOn is " << isAutocommitOn << endl; BRM::TxnID txnid; txnid.id = tmp32; txnid.valid = true; @@ -827,6 +835,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std:: bool isInsertSelect = insertPkg.get_isInsertSelect(); WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; WriteEngine::DctnryStructList dctnryStructList; WriteEngine::DctnryValueList dctnryValueList; WriteEngine::ColValueList colValuesList; @@ -844,7 +853,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std:: try { ridList = systemCatalogPtr->columnRIDs(tableName, true); - roPair = systemCatalogPtr->tableRID( tableName); + roPair = systemCatalogPtr->tableRID(tableName); } catch (std::exception& ex) { @@ -853,7 +862,6 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std:: return rc; } - std::vector dctnryStoreOids(ridList.size()) ; std::vector columns; DctnryStructList dctnryList; @@ -905,14 +913,10 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std:: if (i == 0) { rc = pDBRootExtentTracker->selectFirstSegFile(dbRootExtent, bFirstExtentOnThisPM, bEmptyPM, trkErrMsg); - /* cout << "bEmptyPM = " << (int) bEmptyPM << " bFirstExtentOnThisPM= " << (int)bFirstExtentOnThisPM << - " oid:dbroot:hwm = " << ridList[i].objnum << ":"< 0) { if (colValuesList[0].size() > 0) @@ -1309,7 +1311,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std:: */ if (NO_ERROR != - (error = fWEWrapper.insertColumnRecs(txnid.id, colStructs, colValuesList, dctnryStructList, dicStringList, + (error = fWEWrapper.insertColumnRecs(txnid.id, cscColTypeList, colStructs, colValuesList, dctnryStructList, dicStringList, dbRootExtTrackerVec, 0, bFirstExtentOnThisPM, isInsertSelect, isAutocommitOn, roPair.objnum, fIsFirstBatchPm))) { if (error == ERR_BRM_DEAD_LOCK) @@ -1346,7 +1348,6 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std:: if ( isWarningSet && ( rc == NO_ERROR ) ) { rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; - //cout << "Got warning" << endl; Message::Args args; string cols = "'" + colNames[0] + "'"; @@ -2614,13 +2615,13 @@ uint8_t WE_DMLCommandProc::rollbackBatchAutoOff(messageqcpp::ByteStream& bs, std //Rollbacked all versioned blocks return rc; } + uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId, uint64_t& blocksChanged) { uint8_t rc = 0; - //cout << " In processUpdate" << endl; uint32_t tmp32, sessionID; TxnID txnId; bs >> PMId; @@ -2637,7 +2638,6 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, uint8_t pkgType; bs >> pkgType; cpackages[txnId].read(bs); - //cout << "Processed meta data in update" << endl; rc = fWEWrapper.startTransaction(txnId); @@ -2676,6 +2676,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, WriteEngine::ColStruct colStruct; WriteEngine::ColValueList colValueList; WriteEngine::RIDList rowIDLists; + WriteEngine::CSCTypesList cscColTypeList; WriteEngine::DctnryStructList dctnryStructList; WriteEngine::DctnryStruct dctnryStruct; @@ -2786,54 +2787,6 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, for (unsigned int j = 0; j < columnsUpdated.size(); j++) { - /* WriteEngine::ColTupleList colTupleList; - //timer.start("lookupsyscat"); - tableColName.column = columnsUpdated[j]->get_Name(); - try - { - oid = systemCatalogPtr->lookupOID(tableColName); - } - catch (std::exception& ex) - { - rc = 1; - ostringstream oss; - oss << "lookupOID got exception " << ex.what() << " with column " << tableColName.schema << "." << tableColName.table << "." << tableColName.column; - err = oss.str(); - } - catch ( ... ) - { - rc = 1; - ostringstream oss; - oss << "lookupOID got unknown exception with column " << tableColName.schema << "." << tableColName.table << "." << tableColName.column; - err = oss.str(); - } - - if (rc != 0) - return rc; - - CalpontSystemCatalog::ColType colType; - try - { - colType = systemCatalogPtr->colType(oid); - } - catch (std::exception& ex) - { - rc = 1; - ostringstream oss; - oss << "colType got exception " << ex.what() << " with column oid " << oid; - err = oss.str(); - } - catch ( ... ) - { - rc = 1; - ostringstream oss; - oss << "colType got unknown exception with column oid " << oid; - err = oss.str(); - } - - if (rc !=0) - return rc; - */ WriteEngine::ColTupleList colTupleList; CalpontSystemCatalog::ColType colType = colTypes[j]; oid = oids[j]; @@ -2843,7 +2796,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, colStruct.fCompressionType = colType.compressionType; tableColName.column = columnsUpdated[j]->get_Name(); - if ( !ridsFetched) + if (!ridsFetched) { // querystats uint64_t relativeRID = 0; @@ -2856,13 +2809,26 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, rid = relativeRID; convertToRelativeRid (rid, extentNum, blockNum); rowIDLists.push_back(rid); - uint32_t colWidth = (colTypes[j].colWidth > 8 ? 8 : colTypes[j].colWidth); + + uint32_t colWidth = colTypes[j].colWidth; + + if (colWidth > 8 && + !(colTypes[j].colDataType == CalpontSystemCatalog::DECIMAL || + colTypes[j].colDataType == CalpontSystemCatalog::UDECIMAL)) + { + colWidth = 8; + } + else if (colWidth >= datatypes::MAXDECIMALWIDTH) + { + colWidth = datatypes::MAXDECIMALWIDTH; + } + int rrid = (int) relativeRID / (BYTE_PER_BLOCK / colWidth); // populate stats.blocksChanged if (rrid > preBlkNums[j]) { - preBlkNums[j] = rrid ; - blocksChanged++; + preBlkNums[j] = rrid ; + blocksChanged++; } } @@ -3026,17 +2992,15 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - // decimal width > 8 cannot be stored in an integer - if (fetchColColwidths[fetchColPos] > 8) + if (fetchColColwidths[fetchColPos] == datatypes::MAXDECIMALWIDTH) { - value = row.getStringField(fetchColPos); - unsigned i = strlen(value.c_str()); - value = value.substr(0, i); + datatypes::VDecimal dec(0, + fetchColScales[fetchColPos], + rowGroups[txnId]->getPrecision()[fetchColPos], + row.getBinaryField(fetchColPos)); + value = dec.toString(true); break; } - - // else - // fall through to integer cases } /* fall through */ @@ -3073,12 +3037,10 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, } else { - const int ctmp_size = 65 + 1 + 1 + 1; - char ctmp[ctmp_size] = {0}; - DataConvert::decimalToString( - intColVal, fetchColScales[fetchColPos], - ctmp, ctmp_size, fetchColTypes[fetchColPos]); - value = ctmp; // null termination by decimalToString + datatypes::VDecimal dec(intColVal, + fetchColScales[fetchColPos], + rowGroups[txnId]->getPrecision()[fetchColPos]); + value = dec.toString(); } } break; @@ -3385,17 +3347,15 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - // decimal width > 8 cannot be stored in an integer - if (fetchColColwidths[fetchColPos] > 8) + if (fetchColColwidths[fetchColPos] == datatypes::MAXDECIMALWIDTH) { - value = row.getStringField(fetchColPos); - unsigned i = strlen(value.c_str()); - value = value.substr(0, i); + datatypes::VDecimal dec(0, + fetchColScales[fetchColPos], + rowGroups[txnId]->getPrecision()[fetchColPos], + row.getBinaryField(fetchColPos)); + value = dec.toString(true); break; } - - // else - // fall through to integer cases } /* fall through */ @@ -3433,12 +3393,10 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, } else { - const int ctmp_size = 65 + 1 + 1 + 1; - char ctmp[ctmp_size] = {0}; - DataConvert::decimalToString( - intColVal, fetchColScales[fetchColPos], - ctmp, ctmp_size, fetchColTypes[fetchColPos]); - value = ctmp; // null termination by decimalToString + datatypes::VDecimal dec(intColVal, + fetchColScales[fetchColPos], + rowGroups[txnId]->getPrecision()[fetchColPos]); + value = dec.toString(); } } break; @@ -3547,7 +3505,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, try { - datavalue = DataConvert::convertColumnData(colType, colType.defaultValue, pushWarn, timeZone, isNull, false, false); + datavalue = colType.convertColumnData(colType.defaultValue, pushWarn, timeZone, isNull, false, false); } catch (exception&) { @@ -3579,7 +3537,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, { try { - datavalue = DataConvert::convertColumnData(colType, value, pushWarn, timeZone, isNull, false, false); + datavalue = colType.convertColumnData(value, pushWarn, timeZone, isNull, false, false); } catch (exception&) { @@ -3676,7 +3634,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, try { - datavalue = DataConvert::convertColumnData(colType, inData, pushWarn, timeZone, isNull, false, false); + datavalue = colType.convertColumnData(inData, pushWarn, timeZone, isNull, false, false); } catch (exception&) { @@ -3722,7 +3680,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, { try { - datavalue = DataConvert::convertColumnData(colType, colType.defaultValue, pushWarn, timeZone, isNull, false, false); + datavalue = colType.convertColumnData(colType.defaultValue, pushWarn, timeZone, isNull, false, false); } catch (exception&) { @@ -3755,7 +3713,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, { try { - datavalue = DataConvert::convertColumnData(colType, inData, pushWarn, timeZone, isNull, false, true); + datavalue = colType.convertColumnData(inData, pushWarn, timeZone, isNull, false, true); } catch (exception& ex) { @@ -3791,12 +3749,13 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, colStructList.push_back(colStruct); colValueList.push_back (colTupleList); + cscColTypeList.push_back(colType); } //end of bulding values and column structure. //timer.stop("fetch values"); if (rowIDLists.size() > 0) { - error = fWEWrapper.updateColumnRecs(txnId, colStructList, colValueList, rowIDLists, tableRO.objnum); + error = fWEWrapper.updateColumnRecs(txnId, cscColTypeList, colStructList, colValueList, rowIDLists, tableRO.objnum); } if (error != NO_ERROR) @@ -4026,7 +3985,6 @@ uint8_t WE_DMLCommandProc::processDelete(messageqcpp::ByteStream& bs, uint64_t& blocksChanged) { uint8_t rc = 0; - //cout << " In processDelete" << endl; uint32_t tmp32, sessionID; TxnID txnId; bs >> PMId; @@ -4101,13 +4059,25 @@ uint8_t WE_DMLCommandProc::processDelete(messageqcpp::ByteStream& bs, for (uint32_t j = 0; j < row.getColumnCount(); j++) { preBlkNums[j] = -1; - colWidth[j] = (row.getColumnWidth(j) >= 8 ? 8 : row.getColumnWidth(j)); + colWidth[j] = row.getColumnWidth(j); + execplan::CalpontSystemCatalog::ColDataType colDataType = row.getColType(j); + if (colWidth[j] >= 8 && + !(colDataType == execplan::CalpontSystemCatalog::DECIMAL || + colDataType == execplan::CalpontSystemCatalog::UDECIMAL)) + { + colWidth[j] = 8; + } + else if (colWidth[j] >= datatypes::MAXDECIMALWIDTH) + { + colWidth[j] = datatypes::MAXDECIMALWIDTH; + } } //Get the file information from rowgroup dbRoot = rowGroups[txnId]->getDBRoot(); rowGroups[txnId]->getLocation(&partition, &segment, &extentNum, &blockNum); WriteEngine::ColStructList colStructList; + WriteEngine::CSCTypesList cscColTypeList; WriteEngine::ColStruct colStruct; colStruct.fColPartition = partition; colStruct.fColSegment = segment; @@ -4143,7 +4113,9 @@ uint8_t WE_DMLCommandProc::processDelete(messageqcpp::ByteStream& bs, colStruct.tokenFlag = false; colStruct.fCompressionType = colType.compressionType; - if (colType.colWidth > 8) //token + if (colType.colWidth > 8 && + !(colType.colDataType == CalpontSystemCatalog::DECIMAL || + colType.colDataType == CalpontSystemCatalog::UDECIMAL)) //token { colStruct.colWidth = 8; colStruct.tokenFlag = true; @@ -4155,7 +4127,8 @@ uint8_t WE_DMLCommandProc::processDelete(messageqcpp::ByteStream& bs, colStruct.colDataType = colType.colDataType; - colStructList.push_back( colStruct ); + colStructList.push_back(colStruct); + cscColTypeList.push_back(colType); } } catch (exception& ex) @@ -4166,13 +4139,15 @@ uint8_t WE_DMLCommandProc::processDelete(messageqcpp::ByteStream& bs, } std::vector colExtentsStruct; + std::vector colExtentsColType; std::vector colOldValueList; std::vector ridLists; colExtentsStruct.push_back(colStructList); + colExtentsColType.push_back(cscColTypeList); ridLists.push_back(rowIDList); int error = 0; - error = fWEWrapper.deleteRow( txnId, colExtentsStruct, colOldValueList, ridLists, roPair.objnum ); + error = fWEWrapper.deleteRow(txnId, colExtentsColType, colExtentsStruct, colOldValueList, ridLists, roPair.objnum); if (error != NO_ERROR) { diff --git a/writeengine/server/we_dmlcommandproc.h b/writeengine/server/we_dmlcommandproc.h index 8eb7d2fe2..e80597d68 100644 --- a/writeengine/server/we_dmlcommandproc.h +++ b/writeengine/server/we_dmlcommandproc.h @@ -106,12 +106,12 @@ private: WriteEngineWrapper fWEWrapper; boost::scoped_ptr fRBMetaWriter; std::vector > dbRootExtTrackerVec; - inline bool isDictCol ( execplan::CalpontSystemCatalog::ColType colType ) + inline bool isDictCol ( execplan::CalpontSystemCatalog::ColType &colType ) { if (((colType.colDataType == execplan::CalpontSystemCatalog::CHAR) && (colType.colWidth > 8)) || ((colType.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7)) - || ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 18)) - || ((colType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && (colType.precision > 18)) + || ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 38)) + || ((colType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && (colType.precision > 38)) || (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY) || (colType.colDataType == execplan::CalpontSystemCatalog::BLOB) || (colType.colDataType == execplan::CalpontSystemCatalog::TEXT)) diff --git a/writeengine/server/we_readthread.cpp b/writeengine/server/we_readthread.cpp index f27ed9795..f262817f5 100644 --- a/writeengine/server/we_readthread.cpp +++ b/writeengine/server/we_readthread.cpp @@ -154,10 +154,7 @@ void DmlReadThread::operator()() case WE_SVR_BATCH_INSERT: { - //timer.start("processBatchInsert"); rc = fWeDMLprocessor->processBatchInsert(ibs, errMsg, PMId); - //timer.stop("processBatchInsert"); - //cout << "fWeDMLprocessor " << fWeDMLprocessor << " is processing batchinsert ..." << endl; break; } diff --git a/writeengine/shared/shared_components_tests.cpp b/writeengine/shared/shared_components_tests.cpp index 682bf1251..db2f9d48c 100644 --- a/writeengine/shared/shared_components_tests.cpp +++ b/writeengine/shared/shared_components_tests.cpp @@ -91,7 +91,8 @@ CPPUNIT_TEST(setUp); // Extent & dict related testing CPPUNIT_TEST( testExtensionWOPrealloc ); CPPUNIT_TEST( testDictExtensionWOPrealloc ); -// Semaphore related testing + CPPUNIT_TEST( testExtentCrWOPreallocBin ); + // Semaphore related testing // CPPUNIT_TEST( testSem ); // Log related testing @@ -1542,7 +1543,180 @@ public: } */ + template struct binary; + typedef binary<16> binary16; + typedef binary<32> binary32; + template + struct binary { + unsigned char data[W]; // May be ok for empty value ? + void operator=(uint64_t v) {*((uint64_t *) data) = v; memset(data + 8, 0, W - 8);} + inline uint8_t& operator[](const int index) {return *((uint8_t*) (data + index));} + inline uint64_t& uint64(const int index) {return *((uint64_t*) (data + (index << 3)));} + }; + + void testExtentCrWOPreallocBin() { + IDBDataFile* pFile = NULL; + ColumnOpCompress1 fileOp; + BlockOp blockOp; + char fileName[20]; + int rc; + char hdrs[ IDBCompressInterface::HDR_BUF_LEN * 2 ]; + int dbRoot = 1; + idbdatafile::IDBPolicy::init(true, false, "", 0); + // Set to versionbuffer to satisfy IDBPolicy::getType + strcpy(fileName, "versionbuffer"); + fileOp.compressionType(1); + + fileOp.deleteFile(fileName); + CPPUNIT_ASSERT(fileOp.exists(fileName) == false); + + //binary16 emptyVal = blockOp.getEmptyBinRowValue( execplan::CalpontSystemCatalog::BINARY, 16 ); + uint64_t emptyVal = blockOp.getEmptyRowValue(execplan::CalpontSystemCatalog::BIGINT, 8); + int width = blockOp.getCorrectRowWidth(execplan::CalpontSystemCatalog::BINARY, sizeof (binary16)); + int nBlocks = INITIAL_EXTENT_ROWS_TO_DISK / BYTE_PER_BLOCK * width; + + // createFile runs IDBDataFile::open + initAbrevCompColumnExtent + // under the hood + // bigint column file + rc = fileOp.createFile(fileName, + nBlocks, // number of blocks + emptyVal, // NULL value + width, // width + dbRoot); // dbroot + CPPUNIT_ASSERT(rc == NO_ERROR); + + fileOp.closeFile(pFile); + + // open created compressed file and check its header + pFile = IDBDataFile::open(IDBPolicy::getType(fileName, + IDBPolicy::WRITEENG), fileName, "rb", dbRoot); + + rc = pFile->seek(0, 0); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = fileOp.readHeaders(pFile, hdrs); + CPPUNIT_ASSERT(rc == NO_ERROR); + // Couldn't use IDBDataFile->close() here w/o excplicit cast + fileOp.closeFile(pFile); + + // Extend the extent up to 64MB + pFile = IDBDataFile::open(IDBPolicy::getType(fileName, + IDBPolicy::WRITEENG), fileName, "rb", dbRoot); + + // disable disk space preallocation + idbdatafile::IDBPolicy::setPreallocSpace(dbRoot); + rc = fileOp.initColumnExtent(pFile, + dbRoot, + BYTE_PER_BLOCK - nBlocks, // number of blocks + emptyVal, + width, + false, // use existing file + false, // don't expand; new extent + false, // add full (not abbreviated) extent + true); // optimize extention + + CPPUNIT_ASSERT(rc == NO_ERROR); + fileOp.closeFile(pFile); + // file has been extended + cout << endl << "file has been extended"; + + // write up to INITIAL_EXTENT_ROWS_TO_DISK + 1 rows into the file + + Column curCol; + binary16 valArray[INITIAL_EXTENT_ROWS_TO_DISK + 1]; + RID rowIdArray[INITIAL_EXTENT_ROWS_TO_DISK + 1]; + // This is the magic for the stub in FileOp::oid2FileName + int fid = 42; + + for (uint64_t it = 0; it <= INITIAL_EXTENT_ROWS_TO_DISK; it++) { + rowIdArray[it] = it; + valArray[it].uint64(0) = it + 3; + valArray[it].uint64(1) = it + 5; + } + + fileOp.initColumn(curCol); + fileOp.setColParam(curCol, + 1, // column number + width, + execplan::CalpontSystemCatalog::BINARY, + WriteEngine::WR_BINARY, + fid, + 1); //compression type + + string segFile; + // openColumnFile uses DBRM's oid server but we + // have to get the chunks' pointers from the header. + curCol.dataFile.pFile = fileOp.openFile( + curCol, + dbRoot, + 0, + 0, + segFile, + false, + "r+b", + BYTE_PER_BLOCK * BYTE_PER_BLOCK); // buffer size is 64MB + + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = fileOp.writeRow(curCol, INITIAL_EXTENT_ROWS_TO_DISK + 1, + (RID*) rowIdArray, valArray); + CPPUNIT_ASSERT_EQUAL(NO_ERROR, rc); // I prefer this way as it prints values + + // flush and close the file used for reading + fileOp.clearColumn(curCol); + + std::map oids; + oids[fid] = fid; + + // flush changed chunks from the Manager + int rtn1 = fileOp.chunkManager()->flushChunks(rc, oids); + + // read back the file + cout << endl << "Read file "; + DataBlock block; + binary16* bin16 = (binary16*) block.data; + + fileOp.initColumn(curCol); + fileOp.setColParam(curCol, + 1, // column number + width, + execplan::CalpontSystemCatalog::BINARY, + WriteEngine::WR_BINARY, + fid, + 1); //compression type + + curCol.dataFile.pFile = fileOp.openFile( + curCol, + dbRoot, + 0, + 0, + segFile, + false, + "r+b", + BYTE_PER_BLOCK * BYTE_PER_BLOCK); // buffer size is 64MB + + CPPUNIT_ASSERT_EQUAL(NO_ERROR, rc); + + int blocks = fileOp.blocksInFile(curCol.dataFile.pFile); + + for (int b = 0; b < blocks; b++) { + rc = fileOp.chunkManager()->readBlock(curCol.dataFile.pFile, block.data, b); // ColumnOpCompress1.readBlock() is protected so ... + CPPUNIT_ASSERT_EQUAL(NO_ERROR, rc); + //cout << endl << bin16[0].uint64(0); + CPPUNIT_ASSERT_EQUAL(b * 512UL + 3, bin16[0].uint64(0)); // Checking just first value of each block as it was written before + CPPUNIT_ASSERT_EQUAL(b * 512UL + 5, bin16[0].uint64(1)); + } + + fileOp.clearColumn(curCol); + fileOp.closeFile(curCol.dataFile.pFile); // Seems done by clearColumn, but anyways... + + cout << endl << "Delete file "; + + fileOp.deleteFile(fileName); + CPPUNIT_ASSERT(fileOp.exists(fileName) == false); + cout << endl << "End of test"; + } + void testCleanup() { // shutdown diff --git a/writeengine/shared/we_blockop.cpp b/writeengine/shared/we_blockop.cpp index fb8dfe89b..9cb1700eb 100644 --- a/writeengine/shared/we_blockop.cpp +++ b/writeengine/shared/we_blockop.cpp @@ -34,6 +34,8 @@ using namespace execplan; +#include "emptyvaluemanip.h" + namespace WriteEngine { @@ -82,104 +84,11 @@ bool BlockOp::calculateRowId( * RETURN: * emptyVal - the value of empty row ***********************************************************/ -uint64_t BlockOp::getEmptyRowValue( - const CalpontSystemCatalog::ColDataType colDataType, const int width ) const +// TODO MCOL-641 Add support here +void BlockOp::getEmptyRowValue( + const CalpontSystemCatalog::ColDataType colDataType, const int width, uint8_t* emptyVal ) const { - uint64_t emptyVal = 0; - int offset = 0; - - switch ( colDataType ) - { - case CalpontSystemCatalog::TINYINT : - emptyVal = joblist::TINYINTEMPTYROW; - break; - - case CalpontSystemCatalog::SMALLINT: - emptyVal = joblist::SMALLINTEMPTYROW; - break; - - case CalpontSystemCatalog::MEDINT : - case CalpontSystemCatalog::INT : - emptyVal = joblist::INTEMPTYROW; - break; - - case CalpontSystemCatalog::BIGINT : - emptyVal = joblist::BIGINTEMPTYROW; - break; - - case CalpontSystemCatalog::FLOAT : - case CalpontSystemCatalog::UFLOAT : - emptyVal = joblist::FLOATEMPTYROW; - break; - - case CalpontSystemCatalog::DOUBLE : - case CalpontSystemCatalog::UDOUBLE : - emptyVal = joblist::DOUBLEEMPTYROW; - break; - - case CalpontSystemCatalog::DECIMAL : - case CalpontSystemCatalog::UDECIMAL : - - /* if( width <= 4 ) - emptyVal = joblist::SMALLINTEMPTYROW; - else - if( width <= 9 ) - emptyVal = 0x80000001; - else - if( width <= 18 ) - emptyVal = 0x8000000000000001LL; - else - emptyVal = 0xFFFFFFFFFFFFFFFFLL; - */ - // @bug 194 use the correct logic in handling empty value for decimal - if (width <= 1) - emptyVal = joblist::TINYINTEMPTYROW; - else if ( width <= 2 ) - emptyVal = joblist::SMALLINTEMPTYROW; - else if ( width <= 4 ) - emptyVal = joblist::INTEMPTYROW; - else - emptyVal = joblist::BIGINTEMPTYROW; - - break; - - case CalpontSystemCatalog::UTINYINT : - emptyVal = joblist::UTINYINTEMPTYROW; - break; - - case CalpontSystemCatalog::USMALLINT: - emptyVal = joblist::USMALLINTEMPTYROW; - break; - - case CalpontSystemCatalog::UMEDINT : - case CalpontSystemCatalog::UINT : - emptyVal = joblist::UINTEMPTYROW; - break; - - case CalpontSystemCatalog::UBIGINT : - emptyVal = joblist::UBIGINTEMPTYROW; - break; - - case CalpontSystemCatalog::CHAR : - case CalpontSystemCatalog::VARCHAR : - case CalpontSystemCatalog::DATE : - case CalpontSystemCatalog::DATETIME : - case CalpontSystemCatalog::TIMESTAMP : - default: - offset = ( colDataType == CalpontSystemCatalog::VARCHAR ) ? -1 : 0; - emptyVal = joblist::CHAR1EMPTYROW; - - if ( width == (2 + offset) ) - emptyVal = joblist::CHAR2EMPTYROW; - else if ( width >= (3 + offset) && width <= ( 4 + offset ) ) - emptyVal = joblist::CHAR4EMPTYROW; - else if ( width >= (5 + offset) ) - emptyVal = joblist::CHAR8EMPTYROW; - - break; - } - - return emptyVal; + utils::getEmptyRowValue(colDataType, width, emptyVal); } /*********************************************************** @@ -257,7 +166,7 @@ void BlockOp::resetBuf( unsigned char* buf, const int bufSize ) const ***********************************************************/ /* static */ void BlockOp::setEmptyBuf( - unsigned char* buf, const int bufSize, uint64_t emptyVal, const int width ) + unsigned char* buf, const int bufSize, uint8_t* emptyVal, const int width ) { const int ARRAY_COUNT = 128; const int NBYTES_IN_ARRAY = width * ARRAY_COUNT; @@ -267,9 +176,10 @@ void BlockOp::setEmptyBuf( // Optimize buffer initialization by constructing and copying in an array // instead of individual values. This reduces the number of calls to // memcpy(). - for (int j = 0; j < ARRAY_COUNT; j++) + + for(uint8_t* pos = emptyValArray, * end = pos + NBYTES_IN_ARRAY; pos < end; pos += width) //FIXME for no loop { - memcpy(emptyValArray + (j * width), &emptyVal, width); + memcpy(pos, emptyVal, width); } int countFull128 = (bufSize / width) / ARRAY_COUNT; diff --git a/writeengine/shared/we_blockop.h b/writeengine/shared/we_blockop.h index eb88c5e38..10df5a8bb 100644 --- a/writeengine/shared/we_blockop.h +++ b/writeengine/shared/we_blockop.h @@ -89,8 +89,9 @@ public: /** * @brief Get an empty row value */ - EXPORT uint64_t getEmptyRowValue(const execplan::CalpontSystemCatalog::ColDataType colDataType, - const int width ) const; + EXPORT void getEmptyRowValue(const execplan::CalpontSystemCatalog::ColDataType colDataType, + const int width, + uint8_t* emptyVal ) const; /** * @brief Calculate row id @@ -116,7 +117,7 @@ public: */ EXPORT void static setEmptyBuf( unsigned char* buf, const int bufSize, - uint64_t emptyVal, const int width ); + uint8_t* emptyVal, const int width ); /** * @brief Set a value in a buffer diff --git a/writeengine/shared/we_brm.h b/writeengine/shared/we_brm.h index 23dfe16ce..897c961c4 100644 --- a/writeengine/shared/we_brm.h +++ b/writeengine/shared/we_brm.h @@ -649,7 +649,7 @@ inline int BRMWrapper::bulkSetHWMAndCP( inline int BRMWrapper::setExtentsMaxMin(const BRM::CPInfoList_t& cpinfoList) { - int rc = blockRsltnMgrPtr->setExtentsMaxMin( cpinfoList ); + int rc = blockRsltnMgrPtr->setExtentsMaxMin(cpinfoList); return getRC( rc, ERR_BRM_SET_EXTENTS_CP ); } diff --git a/writeengine/shared/we_bulkrollbackfile.cpp b/writeengine/shared/we_bulkrollbackfile.cpp index fbf4dfba7..ac4cdb4ee 100644 --- a/writeengine/shared/we_bulkrollbackfile.cpp +++ b/writeengine/shared/we_bulkrollbackfile.cpp @@ -306,7 +306,8 @@ void BulkRollbackFile::reInitTruncColumnExtent( } // Initialize the remainder of the extent after the HWM block - uint64_t emptyVal = fDbFile.getEmptyRowValue( colType, colWidth ); + uint8_t* emptyVal = (uint8_t*) alloca(colWidth); + fDbFile.getEmptyRowValue( colType, colWidth, emptyVal ); int rc = fDbFile.reInitPartialColumnExtent( pFile, startOffset, diff --git a/writeengine/shared/we_bulkrollbackfilecompressed.cpp b/writeengine/shared/we_bulkrollbackfilecompressed.cpp index 08954411d..4954930af 100644 --- a/writeengine/shared/we_bulkrollbackfilecompressed.cpp +++ b/writeengine/shared/we_bulkrollbackfilecompressed.cpp @@ -374,7 +374,8 @@ void BulkRollbackFileCompressed::reInitTruncColumnExtent( if (nBlocksToInit > 0) { - uint64_t emptyVal = fDbFile.getEmptyRowValue( colType, colWidth ); + uint8_t* emptyVal = (uint8_t*) alloca(colWidth); + fDbFile.getEmptyRowValue( colType, colWidth, emptyVal ); rc = fDbFile.reInitPartialColumnExtent( pFile, (chunkPtrs[chunkIndex].first + restoredChunkLen), nBlocksToInit, diff --git a/writeengine/shared/we_chunkmanager.cpp b/writeengine/shared/we_chunkmanager.cpp index 75a8feff8..a435ce1d4 100644 --- a/writeengine/shared/we_chunkmanager.cpp +++ b/writeengine/shared/we_chunkmanager.cpp @@ -821,7 +821,8 @@ int ChunkManager::fetchChunkFromFile(IDBDataFile* pFile, int64_t id, ChunkData*& void ChunkManager::initializeColumnChunk(char* buf, CompFileData* fileData) { int size = UNCOMPRESSED_CHUNK_SIZE; - uint64_t emptyVal = fFileOp->getEmptyRowValue(fileData->fColDataType, fileData->fColWidth); + uint8_t* emptyVal = (uint8_t*) alloca(fileData->fColWidth); + fFileOp->getEmptyRowValue(fileData->fColDataType, fileData->fColWidth, emptyVal); fFileOp->setEmptyBuf((unsigned char*)buf, size, emptyVal, fileData->fColWidth); } @@ -1342,7 +1343,7 @@ inline int ChunkManager::writeHeader_(CompFileData* fileData, int ptrSecSize) // For the specified segment file (pFile), read in an abbreviated/compressed // chunk extent, uncompress, and expand to a full chunk for a full extent. //------------------------------------------------------------------------------ -int ChunkManager::expandAbbrevColumnExtent(IDBDataFile* pFile, uint64_t emptyVal, int width) +int ChunkManager::expandAbbrevColumnExtent(IDBDataFile* pFile, uint8_t* emptyVal, int width) { map::iterator i = fFilePtrMap.find(pFile); diff --git a/writeengine/shared/we_chunkmanager.h b/writeengine/shared/we_chunkmanager.h index edf9b232f..6122537e3 100644 --- a/writeengine/shared/we_chunkmanager.h +++ b/writeengine/shared/we_chunkmanager.h @@ -214,7 +214,7 @@ public: void cleanUp(const std::map& columOids); // @brief Expand an initial column, not dictionary, extent to a full extent. - int expandAbbrevColumnExtent(IDBDataFile* pFile, uint64_t emptyVal, int width); + int expandAbbrevColumnExtent(IDBDataFile* pFile, uint8_t* emptyVal, int width); // @brief Update column extent int updateColumnExtent(IDBDataFile* pFile, int addBlockCount); diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index c550c1050..77fa8128d 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -328,6 +328,7 @@ void Convertor::mapErrnoToString(int errNum, std::string& errString) * none ******************************************************************************/ /* static */ +// TODO MCOL-641 void Convertor::convertColType(CalpontSystemCatalog::ColDataType dataType, ColType& internalType, bool isToken) { @@ -434,6 +435,11 @@ void Convertor::convertColType(CalpontSystemCatalog::ColDataType dataType, case CalpontSystemCatalog::UBIGINT: internalType = WriteEngine::WR_ULONGLONG; break; + + // Map BINARY to WR_BINARY + case CalpontSystemCatalog::BINARY: + internalType = WriteEngine::WR_BINARY; + break; default: internalType = WriteEngine::WR_CHAR; @@ -628,9 +634,13 @@ void Convertor::convertColType(ColStruct* curStruct) *internalType = WriteEngine::WR_INT; break; - default: + case 8: *internalType = WriteEngine::WR_LONGLONG; break; + + default: + *internalType = WriteEngine::WR_BINARY; + break; } break; @@ -682,6 +692,11 @@ void Convertor::convertColType(ColStruct* curStruct) case CalpontSystemCatalog::UBIGINT: *internalType = WriteEngine::WR_ULONGLONG; break; + + // Map BINARY to WR_BINARY + case CalpontSystemCatalog::BINARY: + *internalType = WriteEngine::WR_BINARY; + break; default: *internalType = WriteEngine::WR_CHAR; @@ -693,14 +708,8 @@ void Convertor::convertColType(ColStruct* curStruct) // check whether width is in sync with the requirement *width = getCorrectRowWidth(dataType, *width); - - // This is the patch for the decimal thing, override -// if (dataType == CalpontSystemCatalog::DECIMAL) -// { -// *internalType = *width <= 4 ? -// WriteEngine::WR_INT : WriteEngine::WR_LONGLONG; -// } } + /******************************************************************************* * DESCRIPTION: @@ -758,9 +767,10 @@ int Convertor::getCorrectRowWidth(CalpontSystemCatalog::ColDataType dataType, in newWidth = 2; else if (width <= 4) newWidth = 4; - else + else if (width <= 8) newWidth = 8; - + else + newWidth = 16; break; case CalpontSystemCatalog::DATE: @@ -772,7 +782,11 @@ int Convertor::getCorrectRowWidth(CalpontSystemCatalog::ColDataType dataType, in case CalpontSystemCatalog::TIMESTAMP: newWidth = 8; break; - + + case CalpontSystemCatalog::BINARY: + newWidth = width; + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::VARBINARY: // treat same as varchar for now diff --git a/writeengine/shared/we_define.cpp b/writeengine/shared/we_define.cpp index 6ec008826..704600f60 100644 --- a/writeengine/shared/we_define.cpp +++ b/writeengine/shared/we_define.cpp @@ -205,6 +205,7 @@ WErrorCodes::WErrorCodes() : fErrorCodes() fErrorCodes[ERR_BRM_SUSPEND] = " The system is in write suspended mode"; fErrorCodes[ERR_BRM_GET_SUSPEND] = " BRM error get the system suspend flag "; fErrorCodes[ERR_BRM_BAD_STRIPE_CNT] = " Incorrect number of column extents allocated in stripe"; + fErrorCodes[ERR_BRM_UNSUPP_WIDTH] = " Unsupported non-dictionary column width"; // DM error fErrorCodes[ERR_DM_CONVERT_OID] = " a DM Conversion error"; diff --git a/writeengine/shared/we_define.h b/writeengine/shared/we_define.h index da6337ca4..03c7783ff 100644 --- a/writeengine/shared/we_define.h +++ b/writeengine/shared/we_define.h @@ -36,6 +36,8 @@ /** Namespace WriteEngine */ namespace WriteEngine { +// Max column size is 16 bytes since MCOL-641. However left this value +// for backward compatibility const short MAX_COLUMN_BOUNDARY = 8; // Max bytes for one column const int MAX_SIGNATURE_SIZE = 8000; // Max len of dict sig val const int MAX_FIELD_SIZE = 1000; // Max len non-dict fld val @@ -308,6 +310,7 @@ const int ERR_BRM_GET_SHUTDOWN = ERR_BRMBASE + 43;// error getting BRM Shut const int ERR_BRM_SUSPEND = ERR_BRMBASE + 44;// BRM is set to Suspend writes const int ERR_BRM_GET_SUSPEND = ERR_BRMBASE + 45;// error getting BRM Suspend flag const int ERR_BRM_BAD_STRIPE_CNT = ERR_BRMBASE + 46;// Incorrect num of cols allocated in stripe +const int ERR_BRM_UNSUPP_WIDTH = ERR_BRMBASE + 47;// Non-dict column Width > allowed MAX. //-------------------------------------------------------------------------- // DM error diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index fe2eb2710..23e285e47 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -161,7 +161,7 @@ int FileOp::createDir( const char* dirName, mode_t mode ) const * ERR_FILE_CREATE if can not create the file ***********************************************************/ int FileOp::createFile( const char* fileName, int numOfBlock, - uint64_t emptyVal, int width, + uint8_t* emptyVal, int width, uint16_t dbRoot ) { IDBDataFile* pFile = @@ -228,7 +228,7 @@ int FileOp::createFile(FID fid, uint16_t dbRoot, uint32_t partition, execplan::CalpontSystemCatalog::ColDataType colDataType, - uint64_t emptyVal, + uint8_t* emptyVal, int width) { //std::cout << "Creating file oid: " << fid << @@ -569,7 +569,7 @@ bool FileOp::existsOIDDir( FID fid ) const ***********************************************************/ int FileOp::extendFile( OID oid, - uint64_t emptyVal, + uint8_t* emptyVal, int width, HWM hwm, BRM::LBID_t startLbid, @@ -875,7 +875,7 @@ int FileOp::extendFile( ***********************************************************/ int FileOp::addExtentExactFile( OID oid, - uint64_t emptyVal, + uint8_t* emptyVal, int width, int& allocSize, uint16_t dbRoot, @@ -1045,7 +1045,7 @@ int FileOp::initColumnExtent( IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, - uint64_t emptyVal, + uint8_t* emptyVal, int width, bool bNewFile, bool bExpandExtent, @@ -1225,7 +1225,7 @@ int FileOp::initAbbrevCompColumnExtent( IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, - uint64_t emptyVal, + uint8_t* emptyVal, int width) { // Reserve disk space for optimized abbreviated extent @@ -1285,7 +1285,7 @@ int FileOp::writeInitialCompColumnChunk( IDBDataFile* pFile, int nBlocksAllocated, int nRows, - uint64_t emptyVal, + uint8_t* emptyVal, int width, char* hdrs) { @@ -1366,7 +1366,7 @@ int FileOp::writeInitialCompColumnChunk( ***********************************************************/ int FileOp::fillCompColumnExtentEmptyChunks(OID oid, int colWidth, - uint64_t emptyVal, + uint8_t* emptyVal, uint16_t dbRoot, uint32_t partition, uint16_t segment, @@ -1671,7 +1671,7 @@ int FileOp::fillCompColumnExtentEmptyChunks(OID oid, ***********************************************************/ int FileOp::expandAbbrevColumnChunk( IDBDataFile* pFile, - uint64_t emptyVal, + uint8_t* emptyVal, int colWidth, const CompChunkPtr& chunkInPtr, CompChunkPtr& chunkOutPtr ) @@ -2036,7 +2036,7 @@ int FileOp::reInitPartialColumnExtent( IDBDataFile* pFile, long long startOffset, int nBlocks, - uint64_t emptyVal, + uint8_t* emptyVal, int width ) { int rc = setFileOffset( pFile, startOffset, SEEK_SET ); @@ -2845,7 +2845,7 @@ bool FileOp::isDiskSpaceAvail(const std::string& fileName, int nBlocks) const int FileOp::expandAbbrevColumnExtent( IDBDataFile* pFile, // FILE ptr to file where abbrev extent is to be expanded uint16_t dbRoot, // The DBRoot of the file with the abbreviated extent - uint64_t emptyVal,// Empty value to be used in expanding the extent + uint8_t* emptyVal,// Empty value to be used in expanding the extent int width ) // Width of the column (in bytes) { // Based on extent size, see how many blocks to add to fill the extent diff --git a/writeengine/shared/we_fileop.h b/writeengine/shared/we_fileop.h index 0b7d094e4..0615eaef8 100644 --- a/writeengine/shared/we_fileop.h +++ b/writeengine/shared/we_fileop.h @@ -92,7 +92,7 @@ public: int& allocSize, uint16_t dbRoot, uint32_t partition, execplan::CalpontSystemCatalog::ColDataType colDataType, - uint64_t emptyVal = 0, int width = 1 ) ; + uint8_t* emptyVal, int width = 1 ) ; /** @@ -100,7 +100,7 @@ public: * Changed to public for UT. */ int createFile( const char* fileName, int fileSize, - uint64_t emptyVal, int width, + uint8_t* emptyVal, int width, uint16_t dbRoot ); /** @@ -163,7 +163,7 @@ public: EXPORT virtual int expandAbbrevColumnExtent( IDBDataFile* pFile, uint16_t dbRoot, - uint64_t emptyVal, + uint8_t* emptyVal, int width ); /** @@ -198,7 +198,7 @@ public: * @param hdrs (in/out) Contents of headers, if file is compressed. * @return returns NO_ERROR if success. */ - EXPORT int extendFile(OID oid, uint64_t emptyVal, + EXPORT int extendFile(OID oid, uint8_t* emptyVal, int width, HWM hwm, BRM::LBID_t startLbid, @@ -226,7 +226,7 @@ public: * @param newFile (out) Indicates if a new file was created for the extent * @param hdrs (in/out) Contents of headers, if file is compressed. */ - EXPORT int addExtentExactFile(OID oid, uint64_t emptyVal, + EXPORT int addExtentExactFile(OID oid, uint8_t* emptyVal, int width, int& allocSize, uint16_t dbRoot, @@ -253,7 +253,7 @@ public: */ EXPORT int fillCompColumnExtentEmptyChunks(OID oid, int colWidth, - uint64_t emptyVal, + uint8_t* emptyVal, uint16_t dbRoot, uint32_t partition, uint16_t segment, @@ -433,7 +433,7 @@ public: EXPORT int reInitPartialColumnExtent( IDBDataFile* pFile, long long startOffset, int nBlocks, - uint64_t emptyVal, + uint8_t* emptyVal, int width ); /** @@ -497,7 +497,7 @@ public: int initColumnExtent( IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, - uint64_t emptyVal, + uint8_t* emptyVal, int width, bool bNewFile, bool bExpandExtent, @@ -519,7 +519,7 @@ private: FileOp& operator=(const FileOp& rhs); int expandAbbrevColumnChunk( IDBDataFile* pFile, - uint64_t emptyVal, + uint8_t* emptyVal, int colWidth, const compress::CompChunkPtr& chunkInPtr, compress::CompChunkPtr& chunkOutPt); @@ -527,7 +527,7 @@ private: int initAbbrevCompColumnExtent( IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, - uint64_t emptyVal, + uint8_t* emptyVal, int width); static void initDbRootExtentMutexes(); @@ -536,7 +536,7 @@ private: int writeInitialCompColumnChunk( IDBDataFile* pFile, int nBlocksAllocated, int nRows, - uint64_t emptyVal, + uint8_t* emptyVal, int width, char* hdrs); diff --git a/writeengine/shared/we_type.h b/writeengine/shared/we_type.h index 71581b44b..824058d7f 100644 --- a/writeengine/shared/we_type.h +++ b/writeengine/shared/we_type.h @@ -108,9 +108,12 @@ enum ColType /** @brief Column type enumeration*/ WR_USHORT = 14, /** @brief Unsigned Short */ WR_UINT = 15, /** @brief Unsigned Int */ WR_ULONGLONG = 16, /** @brief Unsigned Long long*/ - WR_TEXT = 17, /** @brief TEXT */ + WR_TEXT = 17, /** @brief TEXT */ WR_MEDINT = 18, /** @brief Medium Int */ - WR_UMEDINT = 19 /** @brief Unsigned Medium Int */ + WR_UMEDINT = 19, /** @brief Unsigned Medium Int */ + WR_BINARY = 20 /** @brief BINARY */ + // WIP + //WR_INT128 }; // Describes relation of field to column for a bulk load @@ -299,6 +302,7 @@ struct ColStruct /** @brief Column Interface Struct*/ typedef std::vector ColStructList; /** @brief column struct list */ typedef std::vector ColValueList; /** @brief column value list */ typedef std::vector RIDList; /** @brief RID list */ +typedef std::vector CSCTypesList; /** @brief CSC column types list */ typedef std::vector dictStr; typedef std::vector DictStrList; @@ -343,7 +347,7 @@ struct JobColumn /** @brief Job Column Structure */ execplan::CalpontSystemCatalog::ColDataType dataType; /** @brief column data type */ ColType weType; /** @brief write engine data type */ std::string typeName; /** @brief data type name */ - uint64_t emptyVal; /** @brief default empty value */ + uint128_t emptyVal; /** @brief default empty value */ int width; /** @brief column width; for a dictionary column, this is "eventually" the token width */ int definedWidth; /** @brief column width as defined in the table, used for non-dictionary strings */ int dctnryWidth; /** @brief dictionary width */ diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index 69e2d6f58..a7f41e5e5 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -39,11 +39,14 @@ using namespace std; using namespace execplan; +#include "dataconvert.h" #include "IDBDataFile.h" #include "IDBPolicy.h" using namespace idbdatafile; +#include "emptyvaluemanip.h" +#include "mcs_decimal.h" namespace WriteEngine { @@ -53,6 +56,7 @@ struct RefcolInfo unsigned numExtents; }; + /** * Constructor */ @@ -84,6 +88,7 @@ ColumnOp::~ColumnOp() * NO_ERROR if success * rowIdArray - allocation of the row id left here ***********************************************************/ +// TODO MCOL-641 add support here int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, Column& column, uint64_t totalRow, RID* rowIdArray, HWM& hwm, bool& newExtent, uint64_t& rowsLeft, HWM& newHwm, bool& newFile, ColStructList& newColStructList, DctnryStructList& newDctnryStructList, std::vector >& dbRootExtentTrackers, @@ -123,7 +128,10 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, newFile = false; Column newCol; unsigned char buf[BYTE_PER_BLOCK]; - + unsigned char* curVal; + uint8_t* emptyVal = (uint8_t*) alloca(column.colWidth); + getEmptyRowValue(column.colDataType, column.colWidth, emptyVal); + if (useStartingExtent) { // ZZ. For insert select, skip the hwm block and start inserting from the next block @@ -137,10 +145,10 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, if ( rc != NO_ERROR) return rc; - - for (j = 0; j < totalRowPerBlock; j++) + + for (j = 0, curVal = buf; j < totalRowPerBlock; j++, curVal += column.colWidth) { - if (isEmptyRow(buf, j, column)) + if (isEmptyRow((uint64_t*)curVal, emptyVal, column.colWidth)) { rowIdArray[counter] = getRowId(hwm, column.colWidth, j); rowsallocated++; @@ -183,18 +191,20 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, { if (rc == ERR_FILE_EOF) { - uint64_t emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); + uint8_t* emptyVal = (uint8_t*) alloca(column.colWidth); + getEmptyRowValue(column.colDataType, column.colWidth, emptyVal); setEmptyBuf(buf, BYTE_PER_BLOCK, emptyVal, column.colWidth); RETURN_ON_ERROR(saveBlock(column.dataFile.pFile, buf, hwm)); - } else + } + else { return rc; } } - for (j = 0; j < totalRowPerBlock; j++) + for (j = 0, curVal = buf; j < totalRowPerBlock; j++, curVal += column.colWidth) { - if (isEmptyRow(buf, j, column)) + if (isEmptyRow((uint64_t*)curVal, emptyVal, column.colWidth)) { rowIdArray[counter] = getRowId(hwm, column.colWidth, j); rowsallocated++; @@ -280,7 +290,8 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, if (newColStructList[i].fCompressionType > 0) { - uint64_t emptyVal = getEmptyRowValue(newColStructList[i].colDataType, newColStructList[i].colWidth); + uint8_t* emptyVal = (uint8_t*) alloca(newColStructList[i].colWidth); + getEmptyRowValue(newColStructList[i].colDataType, newColStructList[i].colWidth, emptyVal); string errorInfo; rc = fileOp.fillCompColumnExtentEmptyChunks(newColStructList[i].dataOid, newColStructList[i].colWidth, emptyVal, dbRoot, partition, segment, newHwm, segFile, errorInfo); @@ -306,7 +317,8 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, return rc; } - uint64_t emptyVal = getEmptyRowValue(newColStructList[i].colDataType, newColStructList[i].colWidth); + uint8_t* emptyVal = (uint8_t*) alloca(newColStructList[i].colWidth); + getEmptyRowValue(newColStructList[i].colDataType, newColStructList[i].colWidth, emptyVal); rc = fileOp.expandAbbrevColumnExtent( pFile, dbRoot, emptyVal, newColStructList[i].colWidth); //set hwm for this extent. fileOp.closeFile(pFile); @@ -378,7 +390,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, return rc; //create corresponding dictionary files - if (newFile ) + if (newFile) { boost::scoped_ptr we (new WriteEngineWrapper()); we->setTransId(txnid); @@ -399,7 +411,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, } } - we->flushDataFiles(rc, txnid, columnOids ); + we->flushDataFiles(rc, txnid, columnOids); } } @@ -483,18 +495,20 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, { if (rc == ERR_FILE_EOF) { - uint64_t emptyVal = getEmptyRowValue(newCol.colDataType, newCol.colWidth); + uint8_t* emptyVal = (uint8_t*) alloca(newCol.colWidth); + getEmptyRowValue(newCol.colDataType, newCol.colWidth, emptyVal); setEmptyBuf(buf, BYTE_PER_BLOCK, emptyVal, newCol.colWidth); RETURN_ON_ERROR(saveBlock(newCol.dataFile.pFile, buf, newHwm)); - } else + } + else { return rc; } } - for (j = 0; j < totalRowPerBlock; j++) + for (j = 0, curVal = buf; j < totalRowPerBlock; j++, curVal += column.colWidth) { - if (isEmptyRow(buf, j, column)) + if (isEmptyRow((uint64_t*)curVal, emptyVal, column.colWidth)) // Why to check it if beacause line 483 is always true ? { rowIdArray[counter] = getRowId(newHwm, column.colWidth, j); rowsallocated++; @@ -524,18 +538,20 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, { if (rc == ERR_FILE_EOF) { - uint64_t emptyVal = getEmptyRowValue(newCol.colDataType, newCol.colWidth); + uint8_t* emptyVal = (uint8_t*) alloca(newCol.colWidth); + getEmptyRowValue(newCol.colDataType, newCol.colWidth, emptyVal); setEmptyBuf(buf, BYTE_PER_BLOCK, emptyVal, newCol.colWidth); RETURN_ON_ERROR(saveBlock(newCol.dataFile.pFile, buf, newHwm)); - } else + } + else { return rc; } } - for (j = 0; j < totalRowPerBlock; j++) + for (j = 0, curVal = buf; j < totalRowPerBlock; j++, curVal += column.colWidth) { - if (isEmptyRow(buf, j, column)) + if (isEmptyRow((uint64_t*)curVal, emptyVal, column.colWidth)) { rowIdArray[counter] = getRowId(newHwm, newCol.colWidth, j); rowsallocated++; @@ -634,10 +650,10 @@ int ColumnOp::createColumn(Column& column, uint32_t partition) { int rc, newWidth, allocSize; - uint64_t emptyVal = 0; int compressionType = column.compressionType; setColParam(column, colNo, colWidth, colDataType, colType); - emptyVal = getEmptyRowValue(colDataType, colWidth); + uint8_t* emptyVal = (uint8_t*) alloca(colWidth); + getEmptyRowValue(colDataType, colWidth, emptyVal); newWidth = getCorrectRowWidth(colDataType, colWidth); column.dataFile.fid = dataFid; column.dataFile.fDbRoot = dbRoot; @@ -670,8 +686,6 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi HWM colHwm = 0; RID maxRowId = 0; int size = sizeof(Token); - uint64_t emptyVal; - uint64_t refEmptyVal; long long startColFbo = 0; long long startRefColFbo = 0; @@ -706,8 +720,10 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi config.initConfigCache(); std::vector rootList; config.getRootIdList( rootList ); - emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); - refEmptyVal = getEmptyRowValue(refCol.colDataType, refCol.colWidth); + uint8_t* emptyVal = (uint8_t*) alloca(column.colWidth); + uint8_t* refEmptyVal = (uint8_t*) alloca(refCol.colWidth); + getEmptyRowValue(column.colDataType, column.colWidth, emptyVal); + getEmptyRowValue(refCol.colDataType, refCol.colWidth, refEmptyVal); //find the dbroots which have rows for refrence column unsigned int i = 0, k = 0; @@ -931,7 +947,7 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi while (refBufOffset > 0) { - if (memcmp(&refColBuf[refBufOffset], &refEmptyVal, refCol.colWidth) != 0) + if (memcmp(&refColBuf[refBufOffset], refEmptyVal, refCol.colWidth) != 0) { maxRowId = maxRowId + (refBufOffset / refCol.colWidth); break; @@ -992,7 +1008,7 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi while ((tmpBufOffset + refCol.colWidth) <= BYTE_PER_BLOCK) { - if (memcmp(refColBuf + tmpBufOffset, &refEmptyVal, refCol.colWidth) != 0) //Find the number of nextVal needed. + if (memcmp(refColBuf + tmpBufOffset, refEmptyVal, refCol.colWidth) != 0) //Find the number of nextVal needed. { nexValNeeded++; //memcpy(colBuf + colBufOffset, defaultVal, column.colWidth); @@ -1021,7 +1037,7 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi while (((refBufOffset + refCol.colWidth) <= BYTE_PER_BLOCK) && ((colBufOffset + column.colWidth) <= BYTE_PER_BLOCK)) { - if (memcmp(refColBuf + refBufOffset, &refEmptyVal, refCol.colWidth) != 0) //Find the number of nextVal needed. + if (memcmp(refColBuf + refBufOffset, refEmptyVal, refCol.colWidth) != 0) //Find the number of nextVal needed. { memcpy(defaultVal, &nextVal, 8); nextVal++; @@ -1034,8 +1050,18 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi } } - cpInfo.max = nextValStart + nexValNeeded - 1; - cpInfo.min = nextValStart; + cpInfo.isBinaryColumn = column.colWidth > 8; + + if (!cpInfo.isBinaryColumn) + { + cpInfo.max = nextValStart + nexValNeeded - 1; + cpInfo.min = nextValStart; + } + else + { + cpInfo.bigMax = nextValStart + nexValNeeded - 1; + cpInfo.bigMin = nextValStart; + } cpInfo.seqNum = 0; } @@ -1064,11 +1090,11 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi startColFbo++; colBufOffset = 0; } - + while (((refBufOffset + refCol.colWidth) <= BYTE_PER_BLOCK) && ((colBufOffset + column.colWidth) <= BYTE_PER_BLOCK)) { - if (memcmp(refColBuf + refBufOffset, &refEmptyVal, refCol.colWidth) != 0) + if (memcmp(refColBuf + refBufOffset, refEmptyVal, refCol.colWidth) != 0) { /*if (autoincrement) { @@ -1080,7 +1106,7 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi } else if (column.compressionType != 0) //@Bug 3866, fill the empty row value for compressed chunk { - memcpy(colBuf + colBufOffset, &emptyVal, column.colWidth); + memcpy(colBuf + colBufOffset, emptyVal, column.colWidth); dirty = true; } @@ -1089,15 +1115,33 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi } } - if (isUnsigned(column.colDataType)) + cpInfo.isBinaryColumn = column.colWidth > 8; + + if (!cpInfo.isBinaryColumn) { - cpInfo.max = 0; - cpInfo.min = static_cast(numeric_limits::max()); + if (isUnsigned(column.colDataType)) + { + cpInfo.max = 0; + cpInfo.min = static_cast(numeric_limits::max()); + } + else + { + cpInfo.max = numeric_limits::min(); + cpInfo.min = numeric_limits::max(); + } } else { - cpInfo.max = numeric_limits::min(); - cpInfo.min = numeric_limits::max(); + if (isUnsigned(column.colDataType)) + { + cpInfo.bigMax = 0; + cpInfo.min = -1; + } + else + { + utils::int128Min(cpInfo.bigMax); + utils::int128Max(cpInfo.bigMin); + } } cpInfo.seqNum = -1; @@ -1124,16 +1168,33 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi if (autoincrement) //@Bug 4074. Mark it invalid first to set later { BRM::CPInfo cpInfo1; + cpInfo1.isBinaryColumn = column.colWidth > 8; - if (isUnsigned(column.colDataType)) + if (!cpInfo1.isBinaryColumn) { - cpInfo1.max = 0; - cpInfo1.min = static_cast(numeric_limits::max()); + if (isUnsigned(column.colDataType)) + { + cpInfo1.max = 0; + cpInfo1.min = static_cast(numeric_limits::max()); + } + else + { + cpInfo1.max = numeric_limits::min(); + cpInfo1.min = numeric_limits::max(); + } } else { - cpInfo1.max = numeric_limits::min(); - cpInfo1.min = numeric_limits::max(); + if (isUnsigned(column.colDataType)) + { + cpInfo1.bigMax = 0; + cpInfo1.bigMin = -1; + } + else + { + utils::int128Min(cpInfo1.bigMax); + utils::int128Max(cpInfo1.bigMin); + } } cpInfo1.seqNum = -1; @@ -1149,13 +1210,11 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi BRM::CPInfoList_t cpinfoList; cpInfo.firstLbid = startLbid; cpinfoList.push_back(cpInfo); - //cout << "calling setExtentsMaxMin for startLbid = " << startLbid << endl; rc = BRMWrapper::getInstance()->setExtentsMaxMin(cpinfoList); if ( rc != NO_ERROR) return rc; - //cout << "calling setLocalHWM for oid:hwm = " << column.dataFile.fid <<":"<setLocalHWM((OID)column.dataFile.fid, column.dataFile.fPartition, column.dataFile.fSegment, colHwm); @@ -1276,9 +1335,8 @@ int ColumnOp::extendColumn( bool& newFile, char* hdrs) { - uint64_t emptyVal = 0; - - emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); + uint8_t* emptyVal = (uint8_t*) alloca(column.colWidth); + getEmptyRowValue(column.colDataType, column.colWidth, emptyVal); int rc = extendFile(column.dataFile.fid, emptyVal, column.colWidth, @@ -1319,9 +1377,8 @@ int ColumnOp::addExtent( int& allocSize, char* hdrs) { - uint64_t emptyVal = 0; - - emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); + uint8_t* emptyVal = (uint8_t*) alloca(column.colWidth); + getEmptyRowValue(column.colDataType, column.colWidth, emptyVal); int rc = addExtentExactFile(column.dataFile.fid, emptyVal, column.colWidth, @@ -1349,7 +1406,8 @@ int ColumnOp::addExtent( ***********************************************************/ int ColumnOp::expandAbbrevExtent(const Column& column) { - uint64_t emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); + uint8_t* emptyVal = (uint8_t*) alloca(column.colWidth); + getEmptyRowValue(column.colDataType, column.colWidth, emptyVal); int rc = expandAbbrevColumnExtent(column.dataFile.pFile, column.dataFile.fDbRoot, emptyVal, @@ -1405,18 +1463,31 @@ void ColumnOp::initColumn(Column& column) const * RETURN: * true if success, false otherwise ***********************************************************/ -bool ColumnOp::isEmptyRow(unsigned char* buf, int offset, const Column& column) +inline bool ColumnOp::isEmptyRow(uint64_t* curVal, uint8_t* emptyVal, const int colWidth) { - bool emptyFlag = true; - uint64_t curVal, emptyVal; + // colWidth is either 1, 2, 4, 8, or 16 (Convertor::getCorrectRowWidth) + switch(colWidth){ + case 1: + return *(uint8_t*)curVal == *(uint8_t*)emptyVal; - memcpy(&curVal, buf + offset * column.colWidth, column.colWidth); - emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); + case 2: + return *(uint16_t*)curVal == *(uint16_t*)emptyVal; + + case 4: + return *(uint32_t*)curVal == *(uint32_t*)emptyVal; + + case 8: + return *(uint64_t*)curVal == *(uint64_t*)emptyVal; + + case 16: + return *(uint128_t*)curVal == *(uint128_t*)emptyVal; + + //case 32: + // return ((curVal[0] == emptyVal) && (curVal[1] == emptyVal) + // && (curVal[2] == emptyVal) && (curVal[3] == emptyVal)); + } - if (/*curVal != emptyVal*/memcmp(&curVal, &emptyVal, column.colWidth)) - emptyFlag = false; - - return emptyFlag; + return false; } /*********************************************************** @@ -1534,7 +1605,7 @@ void ColumnOp::setColParam(Column& column, column.colWidth = colWidth; column.colType = colType; column.colDataType = colDataType; - + column.dataFile.fid = dataFid; column.dataFile.fDbRoot = dbRoot; column.dataFile.fPartition = partition; @@ -1543,7 +1614,6 @@ void ColumnOp::setColParam(Column& column, column.compressionType = compressionType; } - /*********************************************************** * DESCRIPTION: * Write row(s) @@ -1566,7 +1636,7 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, bool bExit = false, bDataDirty = false; void* pVal = 0; char charTmpBuf[8]; - uint64_t emptyVal; + uint8_t* emptyVal = (uint8_t*) alloca(curCol.colWidth); int rc = NO_ERROR; uint16_t rowsInBlock = BYTE_PER_BLOCK / curCol.colWidth; @@ -1598,8 +1668,6 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, bDataDirty = true; } - // This is a awkward way to convert void* and get its element, I just don't have a good solution for that - // How about pVal = valArray + i*curCol.colWidth? switch (curCol.colType) { case WriteEngine::WR_FLOAT : @@ -1662,6 +1730,14 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, if (!bDelete) pVal = &((uint64_t*) valArray)[i]; break; + case WriteEngine::WR_BINARY: + if (!bDelete) + { + if (curCol.colWidth == datatypes::MAXDECIMALWIDTH) + pVal = &((int128_t*) valArray)[i]; + } + break; + default : if (!bDelete) pVal = &((int*) valArray)[i]; break; @@ -1669,8 +1745,8 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, if (bDelete) { - emptyVal = getEmptyRowValue(curCol.colDataType, curCol.colWidth); - pVal = &emptyVal; + utils::getEmptyRowValue(curCol.colDataType, curCol.colWidth, emptyVal); + pVal = emptyVal; } // This is the write stuff @@ -1715,7 +1791,7 @@ int ColumnOp::writeRows(Column& curCol, uint64_t totalRow, const RIDList& ridLis void* pVal = 0; //void* pOldVal; char charTmpBuf[8]; - uint64_t emptyVal; + uint8_t* emptyVal; int rc = NO_ERROR; while (!bExit) @@ -1750,108 +1826,79 @@ int ColumnOp::writeRows(Column& curCol, uint64_t totalRow, const RIDList& ridLis // This is a awkward way to convert void* and get ith element, I just don't have a good solution for that // How about pVal = valArray? You're always getting the 0'th element here anyways. - switch (curCol.colType) + // TODO MCOL-641 add support here + // This branch does not seem to be called from anywhere + if (!bDelete) { -// case WriteEngine::WR_LONG : pVal = &((long *) valArray)[i]; break; - case WriteEngine::WR_FLOAT : - if (!bDelete) pVal = &((float*) valArray)[0]; - - //pOldVal = &((float *) oldValArray)[i]; - break; - - case WriteEngine::WR_DOUBLE : - if (!bDelete) pVal = &((double*) valArray)[0]; - - //pOldVal = &((double *) oldValArray)[i]; - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_BLOB : - case WriteEngine::WR_TEXT : - case WriteEngine::WR_CHAR : - if (!bDelete) - { + switch (curCol.colType) + { + case WriteEngine::WR_FLOAT : + pVal = &((float*) valArray)[0]; + break; + + case WriteEngine::WR_DOUBLE : + pVal = &((double*) valArray)[0]; + break; + + case WriteEngine::WR_VARBINARY : // treat same as char for now + case WriteEngine::WR_BLOB : + case WriteEngine::WR_TEXT : + case WriteEngine::WR_CHAR : memcpy(charTmpBuf, (char*)valArray, 8); pVal = charTmpBuf; - } - - //pOldVal = (char*)oldValArray + i*8; - break; - -// case WriteEngine::WR_BIT : pVal = &((bool *) valArray)[i]; break; - case WriteEngine::WR_SHORT : - if (!bDelete) pVal = &((short*) valArray)[0]; - - //pOldVal = &((short *) oldValArray)[i]; - break; - - case WriteEngine::WR_BYTE : - if (!bDelete) pVal = &((char*) valArray)[0]; - - //pOldVal = &((char *) oldValArray)[i]; - break; - - case WriteEngine::WR_LONGLONG: - if (!bDelete) pVal = &((long long*) valArray)[0]; - - //pOldVal = &((long long *) oldValArray)[i]; - break; - - case WriteEngine::WR_TOKEN: - if (!bDelete) pVal = &((Token*) valArray)[0]; - - //pOldVal = &((Token *) oldValArray)[i]; - break; - - case WriteEngine::WR_INT : - case WriteEngine::WR_MEDINT : - if (!bDelete) pVal = &((int*) valArray)[0]; - - //pOldVal = &((int *) oldValArray)[i]; - break; - - case WriteEngine::WR_USHORT : - if (!bDelete) pVal = &((uint16_t*) valArray)[0]; - - //pOldVal = &((uint16_t *) oldValArray)[i]; - break; - - case WriteEngine::WR_UBYTE : - if (!bDelete) pVal = &((uint8_t*) valArray)[0]; - - //pOldVal = &((uint8_t *) oldValArray)[i]; - break; - - case WriteEngine::WR_ULONGLONG: - if (!bDelete) pVal = &((uint64_t*) valArray)[0]; - - //pOldVal = &((uint64_t *) oldValArray)[i]; - break; - - case WriteEngine::WR_UINT : - case WriteEngine::WR_UMEDINT : - if (!bDelete) pVal = &((uint32_t*) valArray)[0]; - - //pOldVal = &((uint32_t *) oldValArray)[i]; - break; - - default : - if (!bDelete) pVal = &((int*) valArray)[0]; - - //pOldVal = &((int *) oldValArray)[i]; - break; + break; + + // case WriteEngine::WR_BIT : pVal = &((bool *) valArray)[i]; break; + case WriteEngine::WR_SHORT : + pVal = &((short*) valArray)[0]; + break; + + case WriteEngine::WR_BYTE : + pVal = &((char*) valArray)[0]; + break; + + case WriteEngine::WR_LONGLONG: + pVal = &((long long*) valArray)[0]; + break; + + case WriteEngine::WR_TOKEN: + pVal = &((Token*) valArray)[0]; + break; + + case WriteEngine::WR_INT : + case WriteEngine::WR_MEDINT : + pVal = &((int*) valArray)[0]; + break; + + case WriteEngine::WR_USHORT : + pVal = &((uint16_t*) valArray)[0]; + break; + + case WriteEngine::WR_UBYTE : + pVal = &((uint8_t*) valArray)[0]; + break; + + case WriteEngine::WR_ULONGLONG: + pVal = &((uint64_t*) valArray)[0]; + break; + + case WriteEngine::WR_UINT : + case WriteEngine::WR_UMEDINT : + pVal = &((uint32_t*) valArray)[0]; + break; + + default : + pVal = &((int*) valArray)[0]; + break; + } } - - // This is the stuff to retrieve old value - //memcpy(pOldVal, dataBuf + dataBio, curCol.colWidth); - - if (bDelete) + else { - emptyVal = getEmptyRowValue(curCol.colDataType, curCol.colWidth); - pVal = &emptyVal; + emptyVal = (uint8_t*) alloca(curCol.colWidth); + getEmptyRowValue(curCol.colDataType, curCol.colWidth, emptyVal); + pVal = emptyVal; } - // This is the write stuff writeBufValue(dataBuf + dataBio, pVal, curCol.colWidth); i++; @@ -1982,6 +2029,10 @@ int ColumnOp::writeRowsValues(Column& curCol, uint64_t totalRow, const RIDList& pVal = &((uint32_t*) valArray)[i]; break; + case WriteEngine::WR_BINARY: + pVal = &((int128_t*) valArray)[i]; + break; + default : pVal = &((int*) valArray)[i]; break; diff --git a/writeengine/wrapper/we_colop.h b/writeengine/wrapper/we_colop.h index d1947d752..1b2d5dcc7 100644 --- a/writeengine/wrapper/we_colop.h +++ b/writeengine/wrapper/we_colop.h @@ -220,7 +220,7 @@ public: /** * @brief Check whether it is an empty row */ - EXPORT virtual bool isEmptyRow(unsigned char* buf, int offset, const Column& column); + EXPORT virtual bool isEmptyRow(uint64_t* curVal, uint8_t* emptyVal, const int colWidth); /** * @brief Check whether it is a valid column diff --git a/writeengine/wrapper/we_colopcompress.cpp b/writeengine/wrapper/we_colopcompress.cpp index cb4fb05dd..4b06c429d 100644 --- a/writeengine/wrapper/we_colopcompress.cpp +++ b/writeengine/wrapper/we_colopcompress.cpp @@ -191,7 +191,7 @@ int ColumnOpCompress1::flushFile(int rc, std::map& columnOids) int ColumnOpCompress1::expandAbbrevColumnExtent( - IDBDataFile* pFile, uint16_t dbRoot, uint64_t emptyVal, int width) + IDBDataFile* pFile, uint16_t dbRoot, uint8_t* emptyVal, int width) { // update the uncompressed initial chunk to full chunk int rc = m_chunkManager->expandAbbrevColumnExtent(pFile, emptyVal, width); diff --git a/writeengine/wrapper/we_colopcompress.h b/writeengine/wrapper/we_colopcompress.h index 0203af320..5b6635dac 100644 --- a/writeengine/wrapper/we_colopcompress.h +++ b/writeengine/wrapper/we_colopcompress.h @@ -111,7 +111,7 @@ public: /** * @brief virtual method in FileOp */ - int expandAbbrevColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, uint64_t emptyVal, int width); + int expandAbbrevColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, uint8_t* emptyVal, int width); /** * @brief virtual method in ColumnOp diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 62842da0d..de9853f5d 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -1,5 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2016 MariaDB Corporation + Copyright (C) 2016-2019 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -57,6 +57,7 @@ using namespace execplan; #include "IDBPolicy.h" #include "MonitorProcMem.h" using namespace idbdatafile; +#include "dataconvert.h" #ifdef _MSC_VER #define isnan _isnan @@ -166,7 +167,6 @@ int WriteEngineWrapper::checkValid(const TxnID& txnid, const ColStructList& colS structListSize = colStructList.size() ; valListSize = colValueList.size(); -// if (colStructList.size() != colValueList.size()) if (structListSize != valListSize) return ERR_STRUCT_VALUE_NOT_MATCH; @@ -218,12 +218,13 @@ void WriteEngineWrapper::findSmallestColumn(uint32_t& colId, ColStructList colSt } } -/*@convertValArray - Convert interface values to internal values +/*@convertValArray - Convert interface values to internal values */ /*********************************************************** * DESCRIPTION: * Convert interface values to internal values * PARAMETERS: + * cscColType - CSC ColType struct list * colStructList - column struct list * colValueList - column value list * RETURN: @@ -231,29 +232,33 @@ void WriteEngineWrapper::findSmallestColumn(uint32_t& colId, ColStructList colSt * valArray - output value array * nullArray - output null flag array ***********************************************************/ -void WriteEngineWrapper::convertValArray(const size_t totalRow, const ColType colType, ColTupleList& curTupleList, void* valArray, bool bFromList) +void WriteEngineWrapper::convertValArray(const size_t totalRow, const CalpontSystemCatalog::ColType& cscColType, const ColType colType, ColTupleList& curTupleList, void* valArray, bool bFromList) { ColTuple curTuple; ColTupleList::size_type i; if (bFromList) + { for (i = 0; i < curTupleList.size(); i++) { curTuple = curTupleList[i]; - convertValue(colType, valArray, i, curTuple.data); - } // end of for (int i = 0 + convertValue(cscColType, colType, valArray, i, curTuple.data); + } + } else + { for (i = 0; i < totalRow; i++) { - convertValue(colType, valArray, i, curTuple.data, false); + convertValue(cscColType, colType, valArray, i, curTuple.data, false); curTupleList.push_back(curTuple); } + } } /* * @brief Convert column value to its internal representation */ -void WriteEngineWrapper::convertValue(const ColType colType, void* value, boost::any& data) +void WriteEngineWrapper::convertValue(const execplan::CalpontSystemCatalog::ColType& cscColType, ColType colType, void* value, boost::any& data) { string curStr; int size; @@ -388,10 +393,28 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* value, boost: memcpy(value, &val, size); } break; + + case WriteEngine::WR_BINARY: + { + size = cscColType.colWidth; + if (cscColType.colDataType == CalpontSystemCatalog::DECIMAL || + cscColType.colDataType == CalpontSystemCatalog::UDECIMAL) + { + int128_t val = boost::any_cast(data); + memcpy(value, &val, size); + } + else // for CalpontSystemCatalog::BINARY + { + char val = boost::any_cast(data); + memcpy(value, &val, size); + } + } + break; } // end of switch (colType) } /*@convertValue - The base for converting values */ + /*********************************************************** * DESCRIPTION: * The base for converting values @@ -402,10 +425,9 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* value, boost: * RETURN: * none ***********************************************************/ -void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, const size_t pos, boost::any& data, bool fromList) +void WriteEngineWrapper::convertValue(const CalpontSystemCatalog::ColType& cscColType, const ColType colType, void* valArray, const size_t pos, boost::any& data, bool fromList) { string curStr; -// ColTuple curTuple; if (fromList) { @@ -492,6 +514,23 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, con case WriteEngine::WR_TOKEN: ((Token*)valArray)[pos] = boost::any_cast(data); break; + + // WIP MCOL-641 + case WriteEngine::WR_BINARY: + size_t size = cscColType.colWidth; + if (cscColType.colDataType == CalpontSystemCatalog::DECIMAL || + cscColType.colDataType == CalpontSystemCatalog::UDECIMAL) + { + int128_t val = boost::any_cast(data); + memcpy((uint8_t*)valArray + pos * size, &val, size); + } + else // for CalpontSystemCatalog::BINARY + { + char val = boost::any_cast(data); + memcpy((uint8_t*)valArray + pos * size, &val, size); + } + + break; } // end of switch (colType) } else @@ -557,6 +596,23 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, con case WriteEngine::WR_TOKEN: data = ((Token*)valArray)[pos]; break; + // WIP MCOL-641 + case WriteEngine::WR_BINARY : + if (cscColType.colDataType == CalpontSystemCatalog::DECIMAL || + cscColType.colDataType == CalpontSystemCatalog::UDECIMAL) + { + data = ((int128_t*)valArray)[pos]; + } + else // for CalpontSystemCatalog::BINARY + { + // WIP do we need tmp here? + size_t size = cscColType.colWidth; + char *tmp = (char*) alloca (sizeof(char) * size); + memcpy(tmp, (uint8_t*)valArray + pos * size, size); + curStr = tmp; + data = curStr; + } + break; } // end of switch (colType) } // end of if } @@ -619,7 +675,7 @@ int WriteEngineWrapper::createColumn( * @brief Fill column with default values */ int WriteEngineWrapper::fillColumn(const TxnID& txnid, const OID& dataOid, - const CalpontSystemCatalog::ColDataType dataType, int dataWidth, + const CalpontSystemCatalog::ColType& colType, ColTuple defaultVal, const OID& refColOID, const CalpontSystemCatalog::ColDataType refColDataType, int refColWidth, int refCompressionType, @@ -638,39 +694,37 @@ int WriteEngineWrapper::fillColumn(const TxnID& txnid, const OID& dataOid, Dctnry* dctnry = m_dctnry[op(compressionType)]; colOpNewCol->initColumn(newCol); refColOp->initColumn(refCol); - //boost::shared_ptr dctnry; - // boost::shared_ptr refColOp; - // refColOp.reset(colOpRefCol); - // dctnry.reset(dctOp); uint16_t dbRoot = 1; //not to be used - int newDataWidth = dataWidth; + int newDataWidth = colType.colWidth; //Convert HWM of the reference column for the new column //Bug 1703,1705 bool isToken = false; - if (((dataType == CalpontSystemCatalog::VARCHAR) && (dataWidth > 7)) || - ((dataType == CalpontSystemCatalog::CHAR) && (dataWidth > 8)) || - (dataType == CalpontSystemCatalog::VARBINARY) || - (dataType == CalpontSystemCatalog::BLOB) || - (dataType == CalpontSystemCatalog::TEXT)) + if (((colType.colDataType == CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7)) || + ((colType.colDataType == CalpontSystemCatalog::CHAR) && (colType.colWidth > 8)) || + (colType.colDataType == CalpontSystemCatalog::VARBINARY) || + (colType.colDataType == CalpontSystemCatalog::BLOB) || + (colType.colDataType == CalpontSystemCatalog::TEXT)) { isToken = true; } - Convertor::convertColType(dataType, newColType, isToken); + Convertor::convertColType(colType.colDataType, newColType, isToken); + // WIP + // replace with isDictCol if (((refColDataType == CalpontSystemCatalog::VARCHAR) && (refColWidth > 7)) || ((refColDataType == CalpontSystemCatalog::CHAR) && (refColWidth > 8)) || (refColDataType == CalpontSystemCatalog::VARBINARY) || - (dataType == CalpontSystemCatalog::BLOB) || - (dataType == CalpontSystemCatalog::TEXT)) + (colType.colDataType == CalpontSystemCatalog::BLOB) || + (colType.colDataType == CalpontSystemCatalog::TEXT)) { isToken = true; } - newDataWidth = colOpNewCol->getCorrectRowWidth(dataType, dataWidth); + newDataWidth = colOpNewCol->getCorrectRowWidth(colType.colDataType, colType.colWidth); // MCOL-1347 CS doubles the width for ALTER TABLE..ADD COLUMN - if ( dataWidth < 4 && dataType == CalpontSystemCatalog::VARCHAR ) + if ( colType.colWidth < 4 && colType.colDataType == CalpontSystemCatalog::VARCHAR ) { newDataWidth >>= 1; } @@ -679,7 +733,7 @@ int WriteEngineWrapper::fillColumn(const TxnID& txnid, const OID& dataOid, refColOp->setColParam(refCol, 0, refColOp->getCorrectRowWidth(refColDataType, refColWidth), refColDataType, refColType, (FID)refColOID, refCompressionType, dbRoot); colOpNewCol->setColParam(newCol, 0, newDataWidth, - dataType, newColType, (FID)dataOid, compressionType, dbRoot); + colType.colDataType, newColType, (FID)dataOid, compressionType, dbRoot); int size = sizeof(Token); @@ -690,39 +744,36 @@ int WriteEngineWrapper::fillColumn(const TxnID& txnid, const OID& dataOid, Token nullToken; memcpy(defVal.get(), &nullToken, size); } - //Tokenization is done when we create dictionary file } else - convertValue(newColType, defVal.get(), defaultVal.data); + { + // WIP + convertValue(colType, newColType, defVal.get(), defaultVal.data); + } if (rc == NO_ERROR) - rc = colOpNewCol->fillColumn(txnid, newCol, refCol, defVal.get(), dctnry, refColOp, dictOid, dataWidth, defaultValStr, autoincrement); - -// colOpNewCol->clearColumn(newCol); -// colOpRefCol->clearColumn(refCol); - -// free(defVal); + rc = colOpNewCol->fillColumn(txnid, newCol, refCol, defVal.get(), dctnry, refColOp, dictOid, colType.colWidth, defaultValStr, autoincrement); // flushing files is in colOp->fillColumn() -// if (rc == NO_ERROR) -// rc = flushDataFiles(); return rc; } -int WriteEngineWrapper::deleteRow(const TxnID& txnid, vector& colExtentsStruct, vector& colOldValueList, +int WriteEngineWrapper::deleteRow(const TxnID& txnid, const vector& colExtentsColType, + vector& colExtentsStruct, vector& colOldValueList, vector& ridLists, const int32_t tableOid) { ColTuple curTuple; ColStruct curColStruct; + CalpontSystemCatalog::ColType cscColType; DctnryStruct dctnryStruct; ColValueList colValueList; ColTupleList curTupleList; DctnryStructList dctnryStructList; DctnryValueList dctnryValueList; ColStructList colStructList; - uint64_t emptyVal; + CSCTypesList cscColTypeList; int rc; string tmpStr(""); vector dctnryExtentsStruct; @@ -734,20 +785,24 @@ int WriteEngineWrapper::deleteRow(const TxnID& txnid, vector& col setTransId(txnid); unsigned numExtents = colExtentsStruct.size(); + uint128_t emptyVal; + for (unsigned extent = 0; extent < numExtents; extent++) { colStructList = colExtentsStruct[extent]; + cscColTypeList = colExtentsColType[extent]; for (ColStructList::size_type i = 0; i < colStructList.size(); i++) { curTupleList.clear(); curColStruct = colStructList[i]; - emptyVal = m_colOp[op(curColStruct.fCompressionType)]-> - getEmptyRowValue(curColStruct.colDataType, curColStruct.colWidth); + cscColType = cscColTypeList[i]; + Convertor::convertColType(&curColStruct); + m_colOp[op(curColStruct.fCompressionType)]-> + getEmptyRowValue(curColStruct.colDataType, curColStruct.colWidth, (uint8_t*)&emptyVal); curTuple.data = emptyVal; - //for (RIDList::size_type j = 0; j < ridLists[extent].size(); j++) - // curTupleList.push_back(curTuple); + curTupleList.push_back(curTuple); colValueList.push_back(curTupleList); @@ -772,12 +827,33 @@ int WriteEngineWrapper::deleteRow(const TxnID& txnid, vector& col // unfortunately I don't have a better way to instruct without passing too many parameters m_opType = DELETE; - rc = updateColumnRec(txnid, colExtentsStruct, colValueList, colOldValueList, ridLists, dctnryExtentsStruct, dctnryValueList, tableOid); + rc = updateColumnRec(txnid, colExtentsColType, colExtentsStruct, colValueList, colOldValueList, ridLists, dctnryExtentsStruct, dctnryValueList, tableOid); m_opType = NOOP; return rc; } +inline void allocateValArray(void*& valArray, ColTupleList::size_type totalRow, + ColType colType, int colWidth) +{ + // MCS allocates 8 bytes even for CHARs smaller then 8 bytes. + switch (colType) + { + case WriteEngine::WR_VARBINARY : // treat same as char for now + case WriteEngine::WR_CHAR: + case WriteEngine::WR_BLOB: + case WriteEngine::WR_TEXT: + valArray = calloc(sizeof(char), totalRow * MAX_COLUMN_BOUNDARY); + break; + case WriteEngine::WR_TOKEN: + valArray = calloc(sizeof(Token), totalRow); + break; + default: + valArray = calloc(totalRow, colWidth); + break; + } +} + int WriteEngineWrapper::deleteBadRows(const TxnID& txnid, ColStructList& colStructs, RIDList& ridList, DctnryStructList& dctnryStructList) { @@ -817,61 +893,7 @@ int WriteEngineWrapper::deleteBadRows(const TxnID& txnid, ColStructList& colStru throw std::runtime_error(oss.str()); } - switch (colStructs[i].colType) - { - case WriteEngine::WR_INT: - case WriteEngine::WR_MEDINT: - valArray = (int*) calloc(sizeof(int), 1); - break; - - case WriteEngine::WR_UINT: - case WriteEngine::WR_UMEDINT: - valArray = (uint32_t*) calloc(sizeof(uint32_t), 1); - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR: - case WriteEngine::WR_BLOB: - case WriteEngine::WR_TEXT: - valArray = (char*) calloc(sizeof(char), 1 * MAX_COLUMN_BOUNDARY); - break; - - case WriteEngine::WR_FLOAT: - valArray = (float*) calloc(sizeof(float), 1); - break; - - case WriteEngine::WR_DOUBLE: - valArray = (double*) calloc(sizeof(double), 1); - break; - - case WriteEngine::WR_BYTE: - valArray = (char*) calloc(sizeof(char), 1); - break; - - case WriteEngine::WR_UBYTE: - valArray = (uint8_t*) calloc(sizeof(uint8_t), 1); - break; - - case WriteEngine::WR_SHORT: - valArray = (short*) calloc(sizeof(short), 1); - break; - - case WriteEngine::WR_USHORT: - valArray = (uint16_t*) calloc(sizeof(uint16_t), 1); - break; - - case WriteEngine::WR_LONGLONG: - valArray = (long long*) calloc(sizeof(long long), 1); - break; - - case WriteEngine::WR_ULONGLONG: - valArray = (uint64_t*) calloc(sizeof(uint64_t), 1); - break; - - case WriteEngine::WR_TOKEN: - valArray = (Token*) calloc(sizeof(Token), 1); - break; - } + allocateValArray(valArray, 1, colStructs[i].colType, colStructs[i].colWidth); rc = colOp->writeRows(curCol, ridList.size(), ridList, valArray, 0, true); @@ -942,6 +964,7 @@ void WriteEngineWrapper::flushVMCache() const ***********************************************************/ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, + const CSCTypesList& cscColTypeList, ColStructList& colStructList, ColValueList& colValueList, DctnryStructList& dctnryStructList, @@ -1031,8 +1054,8 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, //-------------------------------------------------------------------------- if (isFirstBatchPm) { - currentDBrootIdx = dbRootExtentTrackers[colId]->getCurrentDBRootIdx(); - extentInfo = dbRootExtentTrackers[colId]->getDBRootExtentList(); + currentDBrootIdx = dbRootExtentTrackers[colId]->getCurrentDBRootIdx(); + extentInfo = dbRootExtentTrackers[colId]->getDBRootExtentList(); dbRoot = extentInfo[currentDBrootIdx].fDbRoot; partitionNum = extentInfo[currentDBrootIdx].fPartition; @@ -1041,7 +1064,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, //---------------------------------------------------------------------- if (bFirstExtentOnThisPM) { - //cout << "bFirstExtentOnThisPM is " << bFirstExtentOnThisPM << endl; std::vector cols; BRM::CreateStripeColumnExtentsArgIn createStripeColumnExtentsArgIn; @@ -1062,24 +1084,11 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, BRM::CPInfoList_t cpinfoList; BRM::CPInfo cpInfo; - if (isUnsigned(colStructList[i].colDataType)) - { - cpInfo.max = 0; - cpInfo.min = static_cast(numeric_limits::max()); - } - else - { - cpInfo.max = numeric_limits::min(); - cpInfo.min = numeric_limits::max(); - } - - cpInfo.seqNum = -1; - for ( i = 0; i < extents.size(); i++) { colOp = m_colOp[op(colStructList[i].fCompressionType)]; colOp->initColumn(curCol); - colOp->setColParam(curCol, colId, colStructList[i].colWidth, colStructList[i].colDataType, + colOp->setColParam(curCol, colId, colStructList[i].colWidth, colStructList[i].colDataType, colStructList[i].colType, colStructList[i].dataOid, colStructList[i].fCompressionType, dbRoot, partitionNum, segmentNum); rc = colOp->extendColumn(curCol, false, extents[i].startBlkOffset, extents[i].startLbid, extents[i].allocSize, dbRoot, @@ -1088,6 +1097,37 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, if (rc != NO_ERROR) return rc; + cpInfo.isBinaryColumn = colStructList[i].colWidth > 8; + + if (!cpInfo.isBinaryColumn) + { + if (isUnsigned(colStructList[i].colDataType)) + { + cpInfo.max = 0; + cpInfo.min = static_cast(numeric_limits::max()); + } + else + { + cpInfo.max = numeric_limits::min(); + cpInfo.min = numeric_limits::max(); + } + } + else + { + if (isUnsigned(colStructList[i].colDataType)) + { + cpInfo.bigMax = 0; + cpInfo.bigMin = -1; + } + else + { + utils::int128Min(cpInfo.bigMax); + utils::int128Max(cpInfo.bigMin); + } + } + + cpInfo.seqNum = -1; + //mark the extents to invalid cpInfo.firstLbid = extents[i].startLbid; cpinfoList.push_back(cpInfo); @@ -1128,8 +1168,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, colStructList[i].fColPartition = tmpExtentInfo[currentDBrootIdx].fPartition; colStructList[i].fColSegment = tmpExtentInfo[currentDBrootIdx].fSegment; colStructList[i].fColDbRoot = tmpExtentInfo[currentDBrootIdx].fDbRoot; - //cout << "Load from dbrootExtenttracker oid:dbroot:part:seg = " <getDBRootExtentList(); aExt.isNewExt = false; aExt.hwm = tmpExtentInfo[currentDBrootIdx].fLocalHwm; - //cout << "oid " << colStructList[i].dataOid << " gets hwm " << aExt.hwm << endl; } aExt.current = true; aColExtsInfo.push_back(aExt); - //cout << "get from extentinfo oid:hwm = " << colStructList[i].dataOid << ":" << aExt.hwm << endl; } tableMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); @@ -1253,7 +1288,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, //-------------------------------------------------------------------------- // allocate row id(s) //-------------------------------------------------------------------------- - curColStruct = colStructList[colId]; + curColStruct = colStructList[colId]; colOp = m_colOp[op(curColStruct.fCompressionType)]; colOp->initColumn(curCol); @@ -1264,12 +1299,12 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, vector fileInfo; dbRoot = curColStruct.fColDbRoot; //use the first column to calculate row id - ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[colId].dataOid); + ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[colId].dataOid); ColExtsInfo::iterator it = aColExtsInfo.begin(); while (it != aColExtsInfo.end()) { - if ((it->dbRoot == colStructList[colId].fColDbRoot) && + if ((it->dbRoot == colStructList[colId].fColDbRoot) && (it->partNum == colStructList[colId].fColPartition) && (it->segNum == colStructList[colId].fColSegment) && it->current ) { @@ -1281,7 +1316,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, if (it != aColExtsInfo.end()) { hwm = it->hwm; - //cout << "Got from colextinfo hwm for oid " << colStructList[colId].dataOid << " is " << hwm << " and seg is " << colStructList[0].fColSegment << endl; } oldHwm = hwm; //Save this info for rollback @@ -1316,8 +1350,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, curCol, (uint64_t)totalRow, rowIdArray, hwm, newExtent, rowsLeft, newHwm, newFile, newColStructList, newDctnryStructList, dbRootExtentTrackers, insertSelect, true, tableOid, isFirstBatchPm); - //cout << "after allocrowid, total row = " < 0) && (rowIdArray[totalRow - rowsLeft - 1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK)) { - for (unsigned k=0; klength() == 0) { @@ -1454,7 +1486,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, if (rc != NO_ERROR) return rc; - for (uint32_t rows = 0; rows < rowsLeft; rows++) + for (uint32_t rows = 0; rows < rowsLeft; rows++) { if (dctStr_iter->length() == 0) { @@ -1521,13 +1553,11 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, lastRidNew = rowIdArray[totalRow - 1]; } - //cout << "rowid allocated is " << lastRid << endl; //if a new extent is created, all the columns in this table should have their own new extent //First column already processed //@Bug 1701. Close the file (if uncompressed) m_colOp[op(curCol.compressionType)]->clearColumn(curCol); - //cout << "Saving hwm info for new ext batch" << endl; //Update hwm to set them in the end bool succFlag = false; unsigned colWidth = 0; @@ -1557,7 +1587,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, colWidth = colStructList[i].colWidth; succFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); - //cout << "insertcolumnrec oid:rid:fbo:oldhwm = " << colStructList[i].dataOid << ":" << lastRid << ":" << curFbo << ":" << oldHwm << endl; if (succFlag) { if ((HWM)curFbo >= oldHwm) @@ -1571,8 +1600,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, it->current = false; } - //cout << "updated old ext info for oid " << colStructList[i].dataOid << " dbroot:part:seg:hwm:current = " - //<< it->dbRoot<<":"<partNum<<":"<segNum<<":"<hwm<<":"<< it->current<< " and newExtent is " << newExtent << endl; } else return ERR_INVALID_PARAM; @@ -1628,7 +1655,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, colNewValueList.push_back(newColTupleList); newColTupleList.clear(); - //upate the oldvalue list for the old extent + //update the oldvalue list for the old extent for (uint64_t j = 0; j < (totalRow - rowsLeft); j++) { firstPartTupleList.push_back(colTupleList[j]); @@ -1643,7 +1670,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, #ifdef PROFILE timer.start("writeColumnRec"); #endif -//cout << "Writing column record" << endl; if (rc == NO_ERROR) { @@ -1686,7 +1712,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, //---------------------------------------------------------------------- // Write row(s) to database file(s) //---------------------------------------------------------------------- - rc = writeColumnRec(txnid, colStructList, colOldValueList, rowIdArray, newColStructList, colNewValueList, tableOid, useTmpSuffix); // @bug 5572 HDFS tmp file + rc = writeColumnRec(txnid, cscColTypeList, colStructList, colOldValueList, rowIdArray, newColStructList, colNewValueList, tableOid, useTmpSuffix); // @bug 5572 HDFS tmp file } return rc; @@ -1804,19 +1830,6 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, BRM::CPInfoList_t cpinfoList; BRM::CPInfo cpInfo; - if (isUnsigned(colStructList[i].colDataType)) - { - cpInfo.max = 0; - cpInfo.min = static_cast(numeric_limits::max()); - } - else - { - cpInfo.max = numeric_limits::min(); - cpInfo.min = numeric_limits::max(); - } - - cpInfo.seqNum = -1; - for ( i = 0; i < extents.size(); i++) { colOp = m_colOp[op(colStructList[i].fCompressionType)]; @@ -1830,6 +1843,37 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, if (rc != NO_ERROR) return rc; + cpInfo.isBinaryColumn = colStructList[i].colWidth > 8; + + if (!cpInfo.isBinaryColumn) + { + if (isUnsigned(colStructList[i].colDataType)) + { + cpInfo.max = 0; + cpInfo.min = static_cast(numeric_limits::max()); + } + else + { + cpInfo.max = numeric_limits::min(); + cpInfo.min = numeric_limits::max(); + } + } + else + { + if (isUnsigned(colStructList[i].colDataType)) + { + cpInfo.bigMax = 0; + cpInfo.bigMin = -1; + } + else + { + utils::int128Min(cpInfo.bigMax); + utils::int128Max(cpInfo.bigMin); + } + } + + cpInfo.seqNum = -1; + //mark the extents to invalid cpInfo.firstLbid = extents[i].startLbid; cpinfoList.push_back(cpInfo); @@ -2452,6 +2496,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, int WriteEngineWrapper::insertColumnRec_SYS(const TxnID& txnid, + const CSCTypesList& cscColTypeList, ColStructList& colStructList, ColValueList& colValueList, DctnryStructList& dctnryStructList, @@ -3033,11 +3078,11 @@ int WriteEngineWrapper::insertColumnRec_SYS(const TxnID& txnid, if (newExtent) { - rc = writeColumnRec(txnid, colStructList, colOldValueList, rowIdArray, newColStructList, colNewValueList, tableOid, false); // @bug 5572 HDFS tmp file + rc = writeColumnRec(txnid, cscColTypeList, colStructList, colOldValueList, rowIdArray, newColStructList, colNewValueList, tableOid, false); // @bug 5572 HDFS tmp file } else { - rc = writeColumnRec(txnid, colStructList, colValueList, rowIdArray, newColStructList, colNewValueList, tableOid, false); // @bug 5572 HDFS tmp file + rc = writeColumnRec(txnid, cscColTypeList, colStructList, colValueList, rowIdArray, newColStructList, colNewValueList, tableOid, false); // @bug 5572 HDFS tmp file } } @@ -3114,6 +3159,7 @@ int WriteEngineWrapper::insertColumnRec_SYS(const TxnID& txnid, } int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, + const CSCTypesList& cscColTypeList, ColStructList& colStructList, ColValueList& colValueList, DctnryStructList& dctnryStructList, @@ -3151,8 +3197,6 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, printInputValue(colStructList, colValueList, ridList); } - // end - //Convert data type and column width to write engine specific for (i = 0; i < colStructList.size(); i++) Convertor::convertColType(&colStructList[i]); @@ -3180,7 +3224,7 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, //-------------------------------------------------------------------------- // allocate row id(s) //-------------------------------------------------------------------------- - curColStruct = colStructList[colId]; + curColStruct = colStructList[colId]; colOp = m_colOp[op(curColStruct.fCompressionType)]; colOp->initColumn(curCol); @@ -3217,7 +3261,7 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, oldHwm = hwm; //Save this info for rollback //need to pass real dbRoot, partition, and segment to setColParam - colOp->setColParam(curCol, colId, curColStruct.colWidth, curColStruct.colDataType, + colOp->setColParam(curCol, colId, curColStruct.colWidth, curColStruct.colDataType, curColStruct.colType, curColStruct.dataOid, curColStruct.fCompressionType, dbRoot, partitionNum, segmentNum); @@ -3234,7 +3278,8 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, } bool newFile; - + // WIP + cout << "Datafile " << curCol.dataFile.fSegFileName << endl; #ifdef PROFILE timer.start("allocRowId"); #endif @@ -3334,13 +3379,13 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, // Expand initial abbreviated extent if any RID in 1st extent is > 256K. // if totalRow == rowsLeft, then not adding rows to 1st extent, so skip it. //-------------------------------------------------------------------------- -// DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? - if ((colStructList[colId].fColPartition == 0) && - (colStructList[colId].fColSegment == 0) && - ((totalRow - rowsLeft) > 0) && - (rowIdArray[totalRow - rowsLeft - 1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK)) + // DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? + if ((colStructList[colId].fColPartition == 0) && + (colStructList[colId].fColSegment == 0) && + ((totalRow - rowsLeft) > 0) && + (rowIdArray[totalRow - rowsLeft - 1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK)) { - for (unsigned k=0; kcalculateRowId(lastRid, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); - //cout << "insertcolumnrec oid:rid:fbo:hwm = " << - //colStructList[i].dataOid << ":" << lastRid << ":" << - //curFbo << ":" << hwm << endl; if (succFlag) { if ((HWM)curFbo > oldHwm) @@ -3661,33 +3702,13 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, //-------------------------------------------------------------------------- //Mark extents invalid //-------------------------------------------------------------------------- + // WIP + // Set min/max in dmlprocprocessor if aplicable vector lbids; vector colDataTypes; bool successFlag = true; unsigned width = 0; - //BRM::LBID_t lbid; int curFbo = 0, curBio, lastFbo = -1; - /* if (totalRow-rowsLeft > 0) - { - for (unsigned i = 0; i < colStructList.size(); i++) - { - colOp = m_colOp[op(colStructList[i].fCompressionType)]; - width = colStructList[i].colWidth; - successFlag = colOp->calculateRowId(lastRid , - BYTE_PER_BLOCK/width, width, curFbo, curBio); - if (successFlag) { - if (curFbo != lastFbo) { - RETURN_ON_ERROR(BRMWrapper::getInstance()->getBrmInfo( - colStructList[i].dataOid, - colStructList[i].fColPartition, - colStructList[i].fColSegment, curFbo, lbid)); - lbids.push_back((BRM::LBID_t)lbid); - colDataTypes.push_back(colStructList[i].colDataType); - } - } - } - } - */ lastRid = rowIdArray[totalRow - 1]; for (unsigned i = 0; i < colStructList.size(); i++) @@ -3711,7 +3732,6 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, } } - //cout << "lbids size = " << lbids.size()<< endl; if (lbids.size() > 0) rc = BRMWrapper::getInstance()->markExtentsInvalid(lbids, colDataTypes); @@ -3726,13 +3746,13 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, { if (newExtent) { - rc = writeColumnRec(txnid, colStructList, colOldValueList, + rc = writeColumnRec(txnid, cscColTypeList, colStructList, colOldValueList, rowIdArray, newColStructList, colNewValueList, tableOid, false); // @bug 5572 HDFS tmp file } else { - rc = writeColumnRec(txnid, colStructList, colValueList, + rc = writeColumnRec(txnid, cscColTypeList, colStructList, colValueList, rowIdArray, newColStructList, colNewValueList, tableOid, true); // @bug 5572 HDFS tmp file } @@ -3741,18 +3761,12 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, #ifdef PROFILE timer.stop("writeColumnRec"); #endif -// for (ColTupleList::size_type i = 0; i < totalRow; i++) -// ridList.push_back((RID) rowIdArray[i]); - -// if (rc == NO_ERROR) -// rc = flushDataFiles(NO_ERROR); //-------------------------------------------------------------------------- // Update BRM //-------------------------------------------------------------------------- if ( !newExtent ) { - //flushVMCache(); bool succFlag = false; unsigned colWidth = 0; int extState; @@ -3762,7 +3776,6 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, for (unsigned i = 0; i < totalColumns; i++) { - //colOp = m_colOp[op(colStructList[i].fCompressionType)]; //Set all columns hwm together BulkSetHWMArg aHwmEntry; RETURN_ON_ERROR(BRMWrapper::getInstance()->getLastHWM_DBroot( @@ -3813,7 +3826,6 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, RETURN_ON_ERROR(BRMWrapper::getInstance()->bulkSetHWMAndCP( hwmVecOldext, mergeCPDataArgs)); - //flushVMCache(); #ifdef PROFILE timer.stop("flushVMCache"); #endif @@ -3822,33 +3834,6 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, #ifdef PROFILE timer.finish(); #endif - //flush PrimProc FD cache moved to we_dmlcommandproc.cpp - /* ColsExtsInfoMap colsExtsInfoMap = aTableMetaData->getColsExtsInfoMap(); - ColsExtsInfoMap::iterator it = colsExtsInfoMap.begin(); - ColExtsInfo::iterator aIt; - std::vector files; - BRM::FileInfo aFile; - while (it != colsExtsInfoMap.end()) - { - aIt = (it->second).begin(); - aFile.oid = it->first; - //cout << "OID:" << aArg.oid; - while (aIt != (it->second).end()) - { - aFile.partitionNum = aIt->partNum; - aFile.dbRoot =aIt->dbRoot; - aFile.segmentNum = aIt->segNum; - aFile.compType = aIt->compType; - files.push_back(aFile); - //cout <<"Added to files oid:dbroot:part:seg:compType = " << aFile.oid<<":"< 0)) - cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); - TableMetaData::removeTableMetaData(tableOid); */ return rc; } @@ -3909,11 +3894,13 @@ void WriteEngineWrapper::printInputValue(const ColStructList& colStructList, curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); else if (curTuple.data.type() == typeid(long long)) curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); + else if (curTuple.data.type() == typeid(int128_t)) + { + datatypes::TSInt128 val(boost::any_cast(curTuple.data)); + curStr = val.toString(); + } else if (curTuple.data.type() == typeid(double)) curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); -// else -// if (curTuple.data.type() == typeid(bool)) -// curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); else if (curTuple.data.type() == typeid(short)) curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); else if (curTuple.data.type() == typeid(char)) @@ -4177,6 +4164,7 @@ void WriteEngineWrapper::writeVBEnd(const TxnID& txnid, std::vector& colExtentsColType, vector& colExtentsStruct, ColValueList& colValueList, vector& colOldValueList, @@ -4186,40 +4174,20 @@ int WriteEngineWrapper::updateColumnRec(const TxnID& txnid, const int32_t tableOid) { int rc = 0; - //RID* rowIdArray = NULL; - //RIDList::size_type i; unsigned numExtents = colExtentsStruct.size(); - // ColValueList tmpColValueList; - RIDList::const_iterator ridsIter; ColStructList colStructList; DctnryStructList dctnryStructList; + WriteEngine::CSCTypesList cscColTypeList; ColumnOp* colOp = NULL; for (unsigned extent = 0; extent < numExtents; extent++) { - ridsIter = ridLists[extent].begin(); - - //rowIdArray = (RID*)calloc(sizeof(RID), ridLists[extent].size()); - colStructList = colExtentsStruct[extent]; dctnryStructList = dctnryExtentsStruct[extent]; + cscColTypeList = colExtentsColType[extent]; if (m_opType != DELETE) { - - /* ColTuple colTuple; - ColTupleList colTupleList; - for (i=0; i < colValueList.size(); i++) - { - colTupleList = colValueList[i]; - colTuple = colTupleList[0]; - for (unsigned i = 1; i < ridLists[extent].size(); i++) - { - colTupleList.push_back(colTuple); - } - tmpColValueList.push_back(colTupleList); - } - */ //Tokenize data if needed vector tokenList; @@ -4232,7 +4200,6 @@ int WriteEngineWrapper::updateColumnRec(const TxnID& txnid, { // only need to tokenize once dctCol_iter = dctnryValueList[i].begin(); - //col_iter = colValueList[i].begin(); Token token; if (!dctCol_iter->isNull) @@ -4245,14 +4212,7 @@ int WriteEngineWrapper::updateColumnRec(const TxnID& txnid, //timer.stop("tokenize"); #endif } - else - { - //if (dctnryStructList[i].dctnryOid == 2001) - // std::cout << " got null token for string " << dctCol_iter->sigValue <sigValue << " op:fbo = " << token.op <<":"< lbids; vector colDataTypes; bool successFlag = true; unsigned width = 0; - int curFbo = 0, curBio, lastFbo = -1; + int curFbo = 0, curBio, lastFbo = -1; rid_iter = ridLists[extent].begin(); RID aRid = *rid_iter; @@ -4319,24 +4270,18 @@ int WriteEngineWrapper::updateColumnRec(const TxnID& txnid, } } - //cout << "lbids size = " << lbids.size()<< endl; //#ifdef PROFILE //timer.start("markExtentsInvalid"); //#endif if (lbids.size() > 0) rc = BRMWrapper::getInstance()->markExtentsInvalid(lbids, colDataTypes); - //} - - if ( m_opType != DELETE) + if (m_opType != DELETE) m_opType = UPDATE; - rc = writeColumnRec(txnid, colStructList, colValueList, colOldValueList, + rc = writeColumnRec(txnid, cscColTypeList, colStructList, colValueList, colOldValueList, ridLists[extent], tableOid, true, ridLists[extent].size()); -// if (rowIdArray) -// free(rowIdArray); - m_opType = NOOP; if (rc != NO_ERROR) @@ -4347,21 +4292,18 @@ int WriteEngineWrapper::updateColumnRec(const TxnID& txnid, } int WriteEngineWrapper::updateColumnRecs(const TxnID& txnid, + const CSCTypesList& cscColTypeList, vector& colExtentsStruct, ColValueList& colValueList, const RIDList& ridLists, const int32_t tableOid) { //Mark extents invalid - //int rc = 0; - //if (colExtentsStruct[0].dataOid < 3000) - //{ vector lbids; vector colDataTypes; ColumnOp* colOp = NULL; bool successFlag = true; unsigned width = 0; - \ int curFbo = 0, curBio, lastFbo = -1; RID aRid = ridLists[0]; int rc = 0; @@ -4391,20 +4333,19 @@ int WriteEngineWrapper::updateColumnRecs(const TxnID& txnid, if (lbids.size() > 0) { -// cout << "BRMWrapper::getInstance()->markExtentsInvalid(lbids); " << lbids.size() << " lbids" << endl; rc = BRMWrapper::getInstance()->markExtentsInvalid(lbids, colDataTypes); } - //} - if ( m_opType != DELETE) + if (m_opType != DELETE) m_opType = UPDATE; - rc = writeColumnRecords (txnid, colExtentsStruct, colValueList, ridLists, tableOid); + rc = writeColumnRecords(txnid, cscColTypeList, colExtentsStruct, colValueList, ridLists, tableOid); m_opType = NOOP; return rc; } int WriteEngineWrapper::writeColumnRecords(const TxnID& txnid, + const CSCTypesList& cscColTypeList, vector& colStructList, ColValueList& colValueList, const RIDList& ridLists, const int32_t tableOid, bool versioning) @@ -4414,6 +4355,7 @@ int WriteEngineWrapper::writeColumnRecords(const TxnID& txnid, void* valArray = NULL; Column curCol; ColStruct curColStruct; + CalpontSystemCatalog::ColType curColType; ColTupleList curTupleList; ColStructList::size_type totalColumn; ColStructList::size_type i; @@ -4428,6 +4370,7 @@ int WriteEngineWrapper::writeColumnRecords(const TxnID& txnid, { valArray = NULL; curColStruct = colStructList[i]; + curColType = cscColTypeList[i]; curTupleList = colValueList[i]; ColumnOp* colOp = m_colOp[op(curColStruct.fCompressionType)]; @@ -4489,68 +4432,14 @@ int WriteEngineWrapper::writeColumnRecords(const TxnID& txnid, break; } - switch (curColStruct.colType) - { - case WriteEngine::WR_INT: - case WriteEngine::WR_MEDINT: - valArray = (int*) calloc(sizeof(int), totalRow); - break; - - case WriteEngine::WR_UINT: - case WriteEngine::WR_UMEDINT: - valArray = (uint32_t*) calloc(sizeof(uint32_t), totalRow); - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR: - case WriteEngine::WR_BLOB: - case WriteEngine::WR_TEXT: - valArray = (char*) calloc(sizeof(char), totalRow * MAX_COLUMN_BOUNDARY); - break; - - case WriteEngine::WR_FLOAT: - valArray = (float*) calloc(sizeof(float), totalRow); - break; - - case WriteEngine::WR_DOUBLE: - valArray = (double*) calloc(sizeof(double), totalRow); - break; - - case WriteEngine::WR_BYTE: - valArray = (char*) calloc(sizeof(char), totalRow); - break; - - case WriteEngine::WR_UBYTE: - valArray = (uint8_t*) calloc(sizeof(uint8_t), totalRow); - break; - - case WriteEngine::WR_SHORT: - valArray = (short*) calloc(sizeof(short), totalRow); - break; - - case WriteEngine::WR_USHORT: - valArray = (uint16_t*) calloc(sizeof(uint16_t), totalRow); - break; - - case WriteEngine::WR_LONGLONG: - valArray = (long long*) calloc(sizeof(long long), totalRow); - break; - - case WriteEngine::WR_ULONGLONG: - valArray = (uint64_t*) calloc(sizeof(uint64_t), totalRow); - break; - - case WriteEngine::WR_TOKEN: - valArray = (Token*) calloc(sizeof(Token), totalRow); - break; - } + allocateValArray(valArray, totalRow, curColStruct.colType, curColStruct.colWidth); // convert values to valArray bExcp = false; try { - convertValArray(totalRow, curColStruct.colType, curTupleList, valArray); + convertValArray(totalRow, cscColTypeList[i], curColStruct.colType, curTupleList, valArray); } catch (...) { @@ -4606,6 +4495,7 @@ int WriteEngineWrapper::writeColumnRecords(const TxnID& txnid, * Write values to a column * PARAMETERS: * tableOid - table object id + * cscColTypesList - CSC ColType list * colStructList - column struct list * colValueList - column value list * colNewStructList - the new extent struct list @@ -4617,6 +4507,7 @@ int WriteEngineWrapper::writeColumnRecords(const TxnID& txnid, * others if something wrong in inserting the value ***********************************************************/ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, + const CSCTypesList& cscColTypeList, const ColStructList& colStructList, ColValueList& colValueList, RID* rowIdArray, @@ -4643,16 +4534,11 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, StopWatch timer; #endif + totalRow1 = colValueList[0].size(); + totalRow2 = 0; + if (newColValueList.size() > 0) - { - totalRow1 = colValueList[0].size(); totalRow2 = newColValueList[0].size(); - } - else - { - totalRow1 = colValueList[0].size(); - totalRow2 = 0; - } TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); @@ -4725,73 +4611,23 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, } } - //totalRow1 -= totalRow2; + // WIP We can allocate based on column size and not colType // have to init the size here - // nullArray = (bool*) malloc(sizeof(bool) * totalRow); - switch (colStructList[i].colType) - { - case WriteEngine::WR_INT: - case WriteEngine::WR_MEDINT: - valArray = (int*) calloc(sizeof(int), totalRow1); - break; - - case WriteEngine::WR_UINT: - case WriteEngine::WR_UMEDINT: - valArray = (uint32_t*) calloc(sizeof(uint32_t), totalRow1); - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR: - case WriteEngine::WR_BLOB: - case WriteEngine::WR_TEXT: - valArray = (char*) calloc(sizeof(char), totalRow1 * MAX_COLUMN_BOUNDARY); - break; - - case WriteEngine::WR_FLOAT: - valArray = (float*) calloc(sizeof(float), totalRow1); - break; - - case WriteEngine::WR_DOUBLE: - valArray = (double*) calloc(sizeof(double), totalRow1); - break; - - case WriteEngine::WR_BYTE: - valArray = (char*) calloc(sizeof(char), totalRow1); - break; - - case WriteEngine::WR_UBYTE: - valArray = (uint8_t*) calloc(sizeof(uint8_t), totalRow1); - break; - - case WriteEngine::WR_SHORT: - valArray = (short*) calloc(sizeof(short), totalRow1); - break; - - case WriteEngine::WR_USHORT: - valArray = (uint16_t*) calloc(sizeof(uint16_t), totalRow1); - break; - - case WriteEngine::WR_LONGLONG: - valArray = (long long*) calloc(sizeof(long long), totalRow1); - break; - - case WriteEngine::WR_ULONGLONG: - valArray = (uint64_t*) calloc(sizeof(uint64_t), totalRow1); - break; - - case WriteEngine::WR_TOKEN: - valArray = (Token*) calloc(sizeof(Token), totalRow1); - break; - } + allocateValArray(valArray, totalRow1, colStructList[i].colType, colStructList[i].colWidth); // convert values to valArray + // WIP Is m_opType ever set to DELETE? if (m_opType != DELETE) { bExcp = false; try { - convertValArray(totalRow1, colStructList[i].colType, colValueList[i], valArray); + // WIP We convert values twice!? + // dmlcommandproc converts strings to boost::any and this converts + // into actual type value masked by *void + // It is not clear why we need to convert to boost::any b/c we can convert from the original string here + convertValArray(totalRow1, cscColTypeList[i], colStructList[i].colType, colValueList[i], valArray); } catch (...) { @@ -4900,64 +4736,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, } } - //totalRow1 -= totalRow2; - // have to init the size here -// nullArray = (bool*) malloc(sizeof(bool) * totalRow); - switch (newColStructList[i].colType) - { - case WriteEngine::WR_INT: - case WriteEngine::WR_MEDINT: - valArray = (int*) calloc(sizeof(int), totalRow2); - break; - - case WriteEngine::WR_UINT: - case WriteEngine::WR_UMEDINT: - valArray = (uint32_t*) calloc(sizeof(uint32_t), totalRow2); - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR: - case WriteEngine::WR_BLOB: - case WriteEngine::WR_TEXT: - valArray = (char*) calloc(sizeof(char), totalRow2 * MAX_COLUMN_BOUNDARY); - break; - - case WriteEngine::WR_FLOAT: - valArray = (float*) calloc(sizeof(float), totalRow2); - break; - - case WriteEngine::WR_DOUBLE: - valArray = (double*) calloc(sizeof(double), totalRow2); - break; - - case WriteEngine::WR_BYTE: - valArray = (char*) calloc(sizeof(char), totalRow2); - break; - - case WriteEngine::WR_UBYTE: - valArray = (uint8_t*) calloc(sizeof(uint8_t), totalRow2); - break; - - case WriteEngine::WR_SHORT: - valArray = (short*) calloc(sizeof(short), totalRow2); - break; - - case WriteEngine::WR_USHORT: - valArray = (uint16_t*) calloc(sizeof(uint16_t), totalRow2); - break; - - case WriteEngine::WR_LONGLONG: - valArray = (long long*) calloc(sizeof(long long), totalRow2); - break; - - case WriteEngine::WR_ULONGLONG: - valArray = (uint64_t*) calloc(sizeof(uint64_t), totalRow2); - break; - - case WriteEngine::WR_TOKEN: - valArray = (Token*) calloc(sizeof(Token), totalRow2); - break; - } + allocateValArray(valArray, totalRow2, newColStructList[i].colType, newColStructList[i].colWidth); // convert values to valArray if (m_opType != DELETE) @@ -4966,7 +4745,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, try { - convertValArray(totalRow2, newColStructList[i].colType, newColValueList[i], valArray); + convertValArray(totalRow2, cscColTypeList[i], newColStructList[i].colType, newColValueList[i], valArray); } catch (...) { @@ -5074,63 +4853,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, } } - // have to init the size here -// nullArray = (bool*) malloc(sizeof(bool) * totalRow); - switch (colStructList[i].colType) - { - case WriteEngine::WR_INT: - case WriteEngine::WR_MEDINT: - valArray = (int*) calloc(sizeof(int), totalRow1); - break; - - case WriteEngine::WR_UINT: - case WriteEngine::WR_UMEDINT: - valArray = (uint32_t*) calloc(sizeof(uint32_t), totalRow1); - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR: - case WriteEngine::WR_BLOB: - case WriteEngine::WR_TEXT: - valArray = (char*) calloc(sizeof(char), totalRow1 * MAX_COLUMN_BOUNDARY); - break; - - case WriteEngine::WR_FLOAT: - valArray = (float*) calloc(sizeof(float), totalRow1); - break; - - case WriteEngine::WR_DOUBLE: - valArray = (double*) calloc(sizeof(double), totalRow1); - break; - - case WriteEngine::WR_BYTE: - valArray = (char*) calloc(sizeof(char), totalRow1); - break; - - case WriteEngine::WR_UBYTE: - valArray = (uint8_t*) calloc(sizeof(uint8_t), totalRow1); - break; - - case WriteEngine::WR_SHORT: - valArray = (short*) calloc(sizeof(short), totalRow1); - break; - - case WriteEngine::WR_USHORT: - valArray = (uint16_t*) calloc(sizeof(uint16_t), totalRow1); - break; - - case WriteEngine::WR_LONGLONG: - valArray = (long long*) calloc(sizeof(long long), totalRow1); - break; - - case WriteEngine::WR_ULONGLONG: - valArray = (uint64_t*) calloc(sizeof(uint64_t), totalRow1); - break; - - case WriteEngine::WR_TOKEN: - valArray = (Token*) calloc(sizeof(Token), totalRow1); - break; - } + allocateValArray(valArray, totalRow1, colStructList[i].colType, colStructList[i].colWidth); // convert values to valArray if (m_opType != DELETE) @@ -5139,7 +4862,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, try { - convertValArray(totalRow1, colStructList[i].colType, colValueList[i], valArray); + convertValArray(totalRow1, cscColTypeList[i], colStructList[i].colType, colValueList[i], valArray); } catch (...) { @@ -5349,6 +5072,12 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, tmp16 = curValue; ((uint16_t*)valArray)[j] = tmp16; break; + + // WIP + case WriteEngine::WR_BINARY: + ((uint64_t*)valArray)[j] = curValue; //FIXME maybe + break; + } } @@ -5492,6 +5221,11 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, tmp16 = curValue; ((uint16_t*)valArray)[j] = tmp16; break; + + // WIP + case WriteEngine::WR_BINARY: + ((uint64_t*)valArray)[j] = curValue; // FIXME maybe + break; } } @@ -5525,8 +5259,8 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, return rc; } - int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, + const CSCTypesList& cscColTypeList, const ColStructList& colStructList, const ColValueList& colValueList, vector& colOldValueList, @@ -5649,11 +5383,6 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, } // handling versioning - //cout << " pass to processVersionBuffer rid " << rowIdArray[0] << endl; - //cout << "dataOid:fColPartition = " << curColStruct.dataOid << ":" << curColStruct.fColPartition << endl; -//timer.start("processVersionBuffers"); - //vector rangeList; - // rc = processVersionBuffers(curCol.dataFile.pFile, txnid, curColStruct, curColStruct.colWidth, totalRow, ridList, rangeList); std::vector curFreeList; uint32_t blockUsed = 0; @@ -5667,7 +5396,6 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, aRange.vbFBO = freeList[0].vbFBO + blocksProcessed; aRange.size = rangeLists[i].size(); curFreeList.push_back(aRange); - //cout << "range size = " << aRange.size <<" and blocksProcessed = " << blocksProcessed<< endl; } else { @@ -5690,14 +5418,10 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, rc = 1; break; } - - //cout << "curFreeList size = " << curFreeList.size() << endl; - } blocksProcessed += rangeLists[i].size(); - //timer.start("Delete:writeVB"); rc = BRMWrapper::getInstance()-> writeVB(curCol.dataFile.pFile, (BRM::VER_t)txnid, curColStruct.dataOid, fboLists[i], rangeLists[i], @@ -5705,9 +5429,6 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, } } - //timer.stop("Delete:writeVB"); -//timer.stop("processVersionBuffers"); - // cout << " rc for processVersionBuffer is " << rc << endl; if (rc != NO_ERROR) { if (curColStruct.fCompressionType == 0) @@ -5721,61 +5442,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, break; } - switch (curColStruct.colType) - { - case WriteEngine::WR_INT: - case WriteEngine::WR_MEDINT: - valArray = (int*) calloc(sizeof(int), 1); - break; - - case WriteEngine::WR_UINT: - case WriteEngine::WR_UMEDINT: - valArray = (uint32_t*) calloc(sizeof(uint32_t), 1); - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR: - case WriteEngine::WR_BLOB: - case WriteEngine::WR_TEXT: - valArray = (char*) calloc(sizeof(char), 1 * MAX_COLUMN_BOUNDARY); - break; - - case WriteEngine::WR_FLOAT: - valArray = (float*) calloc(sizeof(float), 1); - break; - - case WriteEngine::WR_DOUBLE: - valArray = (double*) calloc(sizeof(double), 1); - break; - - case WriteEngine::WR_BYTE: - valArray = (char*) calloc(sizeof(char), 1); - break; - - case WriteEngine::WR_UBYTE: - valArray = (uint8_t*) calloc(sizeof(uint8_t), 1); - break; - - case WriteEngine::WR_SHORT: - valArray = (short*) calloc(sizeof(short), 1); - break; - - case WriteEngine::WR_USHORT: - valArray = (uint16_t*) calloc(sizeof(uint16_t), 1); - break; - - case WriteEngine::WR_LONGLONG: - valArray = (long long*) calloc(sizeof(long long), 1); - break; - - case WriteEngine::WR_ULONGLONG: - valArray = (uint64_t*) calloc(sizeof(uint64_t), 1); - break; - - case WriteEngine::WR_TOKEN: - valArray = (Token*) calloc(sizeof(Token), 1); - break; - } + allocateValArray(valArray, 1, curColStruct.colType, curColStruct.colWidth); // convert values to valArray if (m_opType != DELETE) @@ -5786,7 +5453,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, try { - convertValue(curColStruct.colType, valArray, curTuple.data); + convertValue(cscColTypeList[i], curColStruct.colType, valArray, curTuple.data); } catch (...) { @@ -5820,7 +5487,6 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, #endif } -// colOldValueList.push_back(oldValArray); //timer.start("Delete:closefile"); colOp->clearColumn(curCol); @@ -5838,8 +5504,6 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, if ((idbdatafile::IDBPolicy::useHdfs()) && (files.size() > 0)) cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); -//if (idbdatafile::IDBPolicy::useHdfs()) -// cacheutils::dropPrimProcFdCache(); //timer.stop("Delete:purgePrimProcFdCache"); if (rangeListTot.size() > 0) BRMWrapper::getInstance()->writeVBEnd(txnid, rangeListTot); @@ -6304,17 +5968,20 @@ int WriteEngineWrapper::updateNextValue(const TxnID txnId, const OID& columnoid, ColValueList colValueList; WriteEngine::ColTupleList colTuples; ColStructList colStructList; + WriteEngine::CSCTypesList cscColTypeList; WriteEngine::ColStruct colStruct; - colStruct.dataOid = OID_SYSCOLUMN_NEXTVALUE; - colStruct.colWidth = 8; + CalpontSystemCatalog::ColType colType; + colType.columnOID = colStruct.dataOid = OID_SYSCOLUMN_NEXTVALUE; + colType.colWidth = colStruct.colWidth = 8; colStruct.tokenFlag = false; - colStruct.colDataType = CalpontSystemCatalog::UBIGINT; + colType.colDataType = colStruct.colDataType = CalpontSystemCatalog::UBIGINT; colStruct.fColDbRoot = dbRoot; if (idbdatafile::IDBPolicy::useHdfs()) colStruct.fCompressionType = 2; colStructList.push_back(colStruct); + cscColTypeList.push_back(colType); ColTuple colTuple; systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); systemCatalogPtr->identity(CalpontSystemCatalog::EC); @@ -6336,8 +6003,7 @@ int WriteEngineWrapper::updateNextValue(const TxnID txnId, const OID& columnoid, colTuple.data = nextVal; colTuples.push_back(colTuple); colValueList.push_back(colTuples); - //TxnID txnid; - rc = writeColumnRecords(txnId, colStructList, colValueList, ridList, SYSCOLUMN_BASE, false); + rc = writeColumnRecords(txnId, cscColTypeList, colStructList, colValueList, ridList, SYSCOLUMN_BASE, false); if (rc != NO_ERROR) return rc; diff --git a/writeengine/wrapper/writeengine.h b/writeengine/wrapper/writeengine.h index 864d064b1..95de97e8d 100644 --- a/writeengine/wrapper/writeengine.h +++ b/writeengine/wrapper/writeengine.h @@ -157,7 +157,9 @@ public: /** * @brief Convert interface value list to internal value array */ - EXPORT void convertValArray(size_t totalRow, const ColType colType, + EXPORT void convertValArray(const size_t totalRow, + const execplan::CalpontSystemCatalog::ColType& cscColType, + const ColType colType, ColTupleList& curTupleList, void* valArray, bool bFromList = true) ; @@ -187,8 +189,8 @@ public: * @param refColDataType Data-type of the referecne column * @param refColWidth Width of the reference column */ - EXPORT int fillColumn(const TxnID& txnid, const OID& dataOid, execplan::CalpontSystemCatalog::ColDataType dataType, - int dataWidth, ColTuple defaultVal, + EXPORT int fillColumn(const TxnID& txnid, const OID& dataOid, const execplan::CalpontSystemCatalog::ColType& colType, + ColTuple defaultVal, const OID& refColOID, execplan::CalpontSystemCatalog::ColDataType refColDataType, int refColWidth, int refCompressionType, bool isNULL, int compressionType, const std::string& defaultValStr, const OID& dictOid = 0, bool autoincrement = false); @@ -222,7 +224,7 @@ public: * @param colOldValueList column old values list (return value) * @param rowIdList row id list */ - EXPORT int deleteRow(const TxnID& txnid, std::vector& colExtentsStruct, + EXPORT int deleteRow(const TxnID& txnid, const std::vector& colExtentsColType, std::vector& colExtentsStruct, std::vector& colOldValueList, std::vector& ridLists, const int32_t tableOid); /** @@ -318,6 +320,7 @@ public: * @param isFirstBatchPm to track if this batch is first batch for this PM. */ EXPORT int insertColumnRecs(const TxnID& txnid, + const CSCTypesList& cscColTypeList, ColStructList& colStructList, ColValueList& colValueList, DctnryStructList& dctnryStructList, @@ -351,6 +354,7 @@ public: * @param dicStringListt dictionary values list */ EXPORT int insertColumnRec_SYS(const TxnID& txnid, + const CSCTypesList& cscColTypeList, ColStructList& colStructList, ColValueList& colValueList, DctnryStructList& dctnryStructList, @@ -364,6 +368,7 @@ public: * @param dicStringListt dictionary values list */ EXPORT int insertColumnRec_Single(const TxnID& txnid, + const CSCTypesList& cscColTypeList, ColStructList& colStructList, ColValueList& colValueList, DctnryStructList& dctnryStructList, @@ -541,6 +546,7 @@ public: * @param ridList row id list */ EXPORT int updateColumnRec(const TxnID& txnid, + const std::vector& colExtentsColType, std::vector& colExtentsStruct, ColValueList& colValueList, std::vector& colOldValueList, @@ -557,6 +563,7 @@ public: */ EXPORT int updateColumnRecs(const TxnID& txnid, + const CSCTypesList& cscColTypeList, std::vector& colStructList, ColValueList& colValueList, const RIDList& ridLists, @@ -621,7 +628,17 @@ public: */ // todo: add implementation when we work on version control // int endTran(const TransID transOid) { return NO_ERROR; } + // WIP + void setDebugLevel(const DebugLevel level) + { + WEObj::setDebugLevel(level); + for (int i = 0; i < TOTAL_COMPRESS_OP; i++) + { + m_colOp[i]->setDebugLevel(level); + m_dctnry[i]->setDebugLevel(level); + } + } // todo: cleanup /************************************************************************ * Internal use definitions @@ -638,11 +655,9 @@ private: void findSmallestColumn(uint32_t &colId, ColStructList colStructList); /** - * @brief Convert interface column type to a internal column type + * @brief Convert interface column type to an internal column type */ - // void convertColType(void* curStruct, const FuncType curType = FUNC_WRITE_ENGINE) const; - - void convertValue(const ColType colType, void* valArray, size_t pos, boost::any& data, bool fromList = true); + void convertValue(const execplan::CalpontSystemCatalog::ColType& cscColType, ColType colType, void* valArray, size_t pos, boost::any& data, bool fromList = true); /** * @brief Convert column value to its internal representation @@ -651,7 +666,7 @@ private: * @param value Memory pointer for storing output value. Should be pre-allocated * @param data Column data */ - void convertValue(const ColType colType, void* value, boost::any& data); + void convertValue(const execplan::CalpontSystemCatalog::ColType& cscColType, const ColType colType, void* value, boost::any& data); /** * @brief Print input value from DDL/DML processors @@ -676,21 +691,13 @@ private: std::vector& freeList, std::vector >& fboLists, std::vector >& rangeLists, std::vector& rangeListTot); - void setDebugLevel(const DebugLevel level) - { - WEObj::setDebugLevel(level); - - for (int i = 0; i < TOTAL_COMPRESS_OP; i++) - { - m_colOp[i]->setDebugLevel(level); - m_dctnry[i]->setDebugLevel(level); - } - } // todo: cleanup - + /** * @brief Common methods to write values to a column */ - int writeColumnRec(const TxnID& txnid, const ColStructList& colStructList, + int writeColumnRec(const TxnID& txnid, + const CSCTypesList& cscColTypes, + const ColStructList& colStructList, ColValueList& colValueList, RID* rowIdArray, const ColStructList& newColStructList, ColValueList& newColValueList, const int32_t tableOid, @@ -703,15 +710,17 @@ private: const int32_t tableOid, bool useTmpSuffix, bool versioning = true); - //@Bug 1886,2870 pass the address of ridList vector - int writeColumnRec(const TxnID& txnid, const ColStructList& colStructList, + int writeColumnRec(const TxnID& txnid, + const CSCTypesList& cscColTypeList, + const ColStructList& colStructList, const ColValueList& colValueList, std::vector& colOldValueList, const RIDList& ridList, const int32_t tableOid, bool convertStructFlag = true, ColTupleList::size_type nRows = 0); //For update column from column to use - int writeColumnRecords(const TxnID& txnid, std::vector& colStructList, + int writeColumnRecords(const TxnID& txnid, const CSCTypesList& cscColTypeList, + std::vector& colStructList, ColValueList& colValueList, const RIDList& ridLists, const int32_t tableOid, bool versioning = true); diff --git a/writeengine/xml/we_xmljob.cpp b/writeengine/xml/we_xmljob.cpp index 863e5cbef..f93859b61 100644 --- a/writeengine/xml/we_xmljob.cpp +++ b/writeengine/xml/we_xmljob.cpp @@ -48,6 +48,7 @@ using namespace execplan; namespace WriteEngine { // Maximum saturation value for DECIMAL types based on precision +// TODO MCOL-641 add support here. see dataconvert.cpp const long long columnstore_precision[19] = { 0,