From 32f616706728be195d3a8bebd6829dc11253ff74 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Thu, 24 Oct 2019 14:01:47 -0400 Subject: [PATCH 01/78] MCOL-641 Work of Ivan Zuniga on basic read and write support for Binary16 --- dbcon/ddlpackage/ddl.l | 1 + dbcon/ddlpackage/ddl.y | 12 +- dbcon/ddlpackage/ddlpkg.h | 4 +- dbcon/ddlpackageproc/altertableprocessor.cpp | 5 + dbcon/ddlpackageproc/ddlpackageprocessor.cpp | 4 + dbcon/execplan/calpontsystemcatalog.cpp | 4 + dbcon/execplan/calpontsystemcatalog.h | 10 +- dbcon/execplan/predicateoperator.cpp | 1 + dbcon/execplan/simplecolumn.cpp | 7 + dbcon/execplan/simplecolumn_decimal.h | 8 +- dbcon/execplan/simplecolumn_int.h | 5 + dbcon/execplan/simplecolumn_uint.h | 8 +- dbcon/execplan/simplefilter.cpp | 2 + dbcon/execplan/treenode.h | 5 + dbcon/execplan/windowfunctioncolumn.cpp | 3 +- dbcon/joblist/batchprimitiveprocessor-jl.cpp | 3 +- dbcon/joblist/columncommand-jl.cpp | 3 +- dbcon/joblist/jlf_common.cpp | 2 + dbcon/joblist/joblisttypes.h | 3 + dbcon/joblist/lbidlist.cpp | 4 + dbcon/joblist/passthrucommand-jl.cpp | 7 +- dbcon/joblist/pcolscan.cpp | 2 +- dbcon/joblist/pcolstep.cpp | 2 +- dbcon/mysql/ha_mcs_ddl.cpp | 4 + dbcon/mysql/ha_mcs_impl.cpp | 9 + primitives/linux-port/column.cpp | 340 +++++++++++++++++- primitives/linux-port/tdriver.cpp | 181 +++++++++- primitives/primproc/columncommand.cpp | 50 ++- primitives/primproc/passthrucommand.cpp | 18 + primitives/primproc/pseudocc.cpp | 18 +- utils/common/nullvaluemanip.cpp | 8 +- utils/dataconvert/dataconvert.cpp | 10 + utils/funcexp/func_hex.cpp | 9 + utils/rowgroup/rowgroup.cpp | 27 +- utils/rowgroup/rowgroup.h | 45 ++- .../thrift/protocol/TCompactProtocol.tcc | 2 + utils/windowfunction/wf_count.cpp | 4 +- utils/windowfunction/wf_udaf.cpp | 10 +- utils/windowfunction/windowfunctiontype.cpp | 4 +- writeengine/server/we_ddlcommandproc.cpp | 10 + writeengine/server/we_ddlcommon.h | 4 + .../shared/shared_components_tests.cpp | 176 ++++++++- writeengine/shared/we_blockop.cpp | 10 +- writeengine/shared/we_convertor.cpp | 16 +- writeengine/shared/we_type.h | 5 +- writeengine/wrapper/we_colop.cpp | 79 ++-- writeengine/wrapper/we_colop.h | 2 +- writeengine/wrapper/writeengine.cpp | 43 ++- 48 files changed, 1114 insertions(+), 75 deletions(-) diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index b4034c732..a6ab67d59 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -188,6 +188,7 @@ LONGTEXT {return LONGTEXT;} BOOL {return BOOL;} BOOLEAN {return BOOLEAN;} MEDIUMINT {return MEDIUMINT;} +BINARY {return BINARY;} \n { lineno++;} diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index f95f3e3fb..510c7a3b0 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -112,7 +112,7 @@ MIN_ROWS MODIFY NO NOT NULL_TOK NUMBER NUMERIC ON PARTIAL PRECISION PRIMARY REFERENCES RENAME RESTRICT SET SMALLINT TABLE TEXT TINYBLOB TINYTEXT TINYINT TO UNIQUE UNSIGNED UPDATE USER SESSION_USER SYSTEM_USER VARCHAR VARBINARY VARYING WITH ZONE DOUBLE IDB_FLOAT REAL CHARSET COLLATE IDB_IF EXISTS CHANGE TRUNCATE -BOOL BOOLEAN MEDIUMINT TIMESTAMP +BOOL BOOLEAN MEDIUMINT TIMESTAMP BINARY %token DQ_IDENT IDENT FCONST SCONST CP_SEARCH_CONDITION_TEXT ICONST DATE TIME @@ -131,6 +131,7 @@ BOOL BOOLEAN MEDIUMINT TIMESTAMP %type ata_rename_table %type character_string_type %type binary_string_type +%type fixed_binary_string_type %type blob_type %type text_type %type check_constraint_def @@ -747,6 +748,7 @@ opt_column_collate: data_type: character_string_type opt_column_charset opt_column_collate | binary_string_type + | fixed_binary_string_type | numeric_type | datetime_type | blob_type @@ -918,6 +920,14 @@ binary_string_type: } ; +fixed_binary_string_type: + BINARY '(' ICONST ')' + { + $$ = new ColumnType(DDL_BINARY); + $$->fLength = atoi($3); + } + ; + blob_type: BLOB '(' ICONST ')' { diff --git a/dbcon/ddlpackage/ddlpkg.h b/dbcon/ddlpackage/ddlpkg.h index 0c7edd389..821efa440 100644 --- a/dbcon/ddlpackage/ddlpkg.h +++ b/dbcon/ddlpackage/ddlpkg.h @@ -238,6 +238,7 @@ enum DDL_DATATYPES DDL_TEXT, DDL_TIME, DDL_TIMESTAMP, + DDL_BINARY, DDL_INVALID_DATATYPE }; @@ -276,7 +277,8 @@ const std::string DDLDatatypeString[] = "unsigned-numeric", "text", "time", - "timestamp" + "timestamp", + "binary", "" }; diff --git a/dbcon/ddlpackageproc/altertableprocessor.cpp b/dbcon/ddlpackageproc/altertableprocessor.cpp index 743779c68..1e0fa76e2 100644 --- a/dbcon/ddlpackageproc/altertableprocessor.cpp +++ b/dbcon/ddlpackageproc/altertableprocessor.cpp @@ -247,6 +247,11 @@ bool typesAreSame(const CalpontSystemCatalog::ColType& colType, const ColumnType break; + case (CalpontSystemCatalog::BINARY): + if (newType.fType == DDL_BINARY && colType.colWidth == newType.fLength) return true; + + break; + default: break; } diff --git a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp index 0419fc70e..b26392b0c 100644 --- a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp +++ b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp @@ -251,6 +251,10 @@ execplan::CalpontSystemCatalog::ColDataType DDLPackageProcessor::convertDataType case ddlpackage::DDL_TEXT: colDataType = CalpontSystemCatalog::TEXT; break; + + case ddlpackage::DDL_BINARY: + colDataType = CalpontSystemCatalog::BINARY; + break; default: throw runtime_error("Unsupported datatype!"); diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index 042abcc04..02c72c573 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -226,6 +226,10 @@ const string colDataTypeToString(CalpontSystemCatalog::ColDataType cdt) return "udouble"; break; + case CalpontSystemCatalog::BINARY: + return "binary"; + break; + default: break; } diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index e59663375..b89861b07 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -170,7 +170,8 @@ public: NUM_OF_COL_DATA_TYPE, /* NEW TYPES ABOVE HERE */ LONGDOUBLE, /* @bug3241, dev and variance calculation only */ STRINT, /* @bug3532, string as int for fast comparison */ - UNDEFINED /*!< Undefined - used in UDAF API */ + UNDEFINED, /*!< Undefined - used in UDAF API */ + BINARY, /*!< BINARY type */ }; /** the set of column constraint types @@ -1212,6 +1213,13 @@ inline bool isNull(int64_t val, const execplan::CalpontSystemCatalog::ColType& c break; } + case execplan::CalpontSystemCatalog::BINARY: + { + ret = false; + + break; + } + default: break; } diff --git a/dbcon/execplan/predicateoperator.cpp b/dbcon/execplan/predicateoperator.cpp index 3f4aee1f2..02dc3a710 100644 --- a/dbcon/execplan/predicateoperator.cpp +++ b/dbcon/execplan/predicateoperator.cpp @@ -126,6 +126,7 @@ bool PredicateOperator::operator!=(const TreeNode* t) const } //FIXME: VARBINARY??? +//FIXME: BINARY??? void PredicateOperator::setOpType(Type& l, Type& r) { fOperationType = l; // Default to left side. Modify as needed. diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index 8110c5296..a58ad66b7 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -690,6 +690,13 @@ void SimpleColumn::evaluate(Row& row, bool& isNull) break; } + case CalpontSystemCatalog::BINARY: + + { + fResult.strVal = row.getBinaryField(fInputIndex); + break; + } + default: // treat as int64 { fResult.intVal = row.getUintField<8>(fInputIndex); diff --git a/dbcon/execplan/simplecolumn_decimal.h b/dbcon/execplan/simplecolumn_decimal.h index c49b597c5..7a5acbb56 100644 --- a/dbcon/execplan/simplecolumn_decimal.h +++ b/dbcon/execplan/simplecolumn_decimal.h @@ -25,6 +25,7 @@ #ifndef SIMPLECOLUMNDECIMAL_H #define SIMPLECOLUMNDECIMAL_H +#include #include #include @@ -141,7 +142,8 @@ void SimpleColumn_Decimal::setNullVal() case 1: fNullVal = joblist::TINYINTNULL; break; - + case 16: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; default: fNullVal = joblist::BIGINTNULL; } @@ -223,6 +225,8 @@ void SimpleColumn_Decimal::serialize(messageqcpp::ByteStream& b) const case 8: b << (ObjectReader::id_t) ObjectReader::SIMPLECOLUMN_DECIMAL8; break; + case 16: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; } SimpleColumn::serialize(b); @@ -248,6 +252,8 @@ void SimpleColumn_Decimal::unserialize(messageqcpp::ByteStream& b) case 8: ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_DECIMAL8); break; + case 16: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix 16 Bytes ?" << std::endl; } SimpleColumn::unserialize(b); diff --git a/dbcon/execplan/simplecolumn_int.h b/dbcon/execplan/simplecolumn_int.h index 9454bba86..4a770ec89 100644 --- a/dbcon/execplan/simplecolumn_int.h +++ b/dbcon/execplan/simplecolumn_int.h @@ -25,6 +25,7 @@ #ifndef SIMPLECOLUMNINT_H #define SIMPLECOLUMNINT_H +#include #include #include "simplecolumn.h" @@ -241,6 +242,8 @@ void SimpleColumn_INT::serialize(messageqcpp::ByteStream& b) const case 8: b << (ObjectReader::id_t) ObjectReader::SIMPLECOLUMN_INT8; break; + case 16: + std::cout << __FILE__<< ":" << __LINE__ << " Fix for 16 Bytes ?" << std::endl; } SimpleColumn::serialize(b); @@ -266,6 +269,8 @@ void SimpleColumn_INT::unserialize(messageqcpp::ByteStream& b) case 8: ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_INT8); break; + case 16: + std::cout << __FILE__<< ":" << __LINE__ << " Fix for 16 Bytes ?" << std::endl; } SimpleColumn::unserialize(b); diff --git a/dbcon/execplan/simplecolumn_uint.h b/dbcon/execplan/simplecolumn_uint.h index 731242809..289a62463 100644 --- a/dbcon/execplan/simplecolumn_uint.h +++ b/dbcon/execplan/simplecolumn_uint.h @@ -25,6 +25,7 @@ #ifndef SIMPLECOLUMNUINT_H #define SIMPLECOLUMNUINT_H +#include #include #include "simplecolumn.h" @@ -140,7 +141,8 @@ void SimpleColumn_UINT::setNullVal() case 1: fNullVal = joblist::UTINYINTNULL; break; - + case 16: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix 16 Bytes ?" << std::endl; default: fNullVal = joblist::UBIGINTNULL; } @@ -241,6 +243,8 @@ void SimpleColumn_UINT::serialize(messageqcpp::ByteStream& b) const case 8: b << (ObjectReader::id_t) ObjectReader::SIMPLECOLUMN_UINT8; break; + case 16: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix 16 Bytes ?" << std::endl; } SimpleColumn::serialize(b); @@ -266,6 +270,8 @@ void SimpleColumn_UINT::unserialize(messageqcpp::ByteStream& b) case 8: ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_UINT8); break; + case 16: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix 16 Bytes ?" << std::endl; } SimpleColumn::unserialize(b); diff --git a/dbcon/execplan/simplefilter.cpp b/dbcon/execplan/simplefilter.cpp index 3fcc8fcd6..f394f5e51 100644 --- a/dbcon/execplan/simplefilter.cpp +++ b/dbcon/execplan/simplefilter.cpp @@ -219,6 +219,7 @@ const string SimpleFilter::data() const fRhs->resultType().colDataType == CalpontSystemCatalog::DATE || fRhs->resultType().colDataType == CalpontSystemCatalog::DATETIME || fRhs->resultType().colDataType == CalpontSystemCatalog::TIMESTAMP || + fRhs->resultType().colDataType == CalpontSystemCatalog::BINARY || fRhs->resultType().colDataType == CalpontSystemCatalog::TIME)) rhs = "'" + SimpleFilter::escapeString(fRhs->data()) + "'"; else @@ -233,6 +234,7 @@ const string SimpleFilter::data() const fLhs->resultType().colDataType == CalpontSystemCatalog::DATE || fLhs->resultType().colDataType == CalpontSystemCatalog::TIME || fLhs->resultType().colDataType == CalpontSystemCatalog::TIMESTAMP || + fLhs->resultType().colDataType == CalpontSystemCatalog::BINARY || fLhs->resultType().colDataType == CalpontSystemCatalog::DATETIME)) lhs = "'" + SimpleFilter::escapeString(fLhs->data()) + "'"; else diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 5fab31a02..e50c3f30f 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -741,6 +741,10 @@ inline const std::string& TreeNode::getStrVal(const std::string& timeZone) break; } + case CalpontSystemCatalog::BINARY: + { + break; + } default: throw logging::InvalidConversionExcept("TreeNode::getStrVal: Invalid conversion."); } @@ -1078,6 +1082,7 @@ inline IDB_Decimal TreeNode::getDecimalVal() case CalpontSystemCatalog::VARBINARY: case CalpontSystemCatalog::BLOB: + case CalpontSystemCatalog::BINARY: throw logging::InvalidConversionExcept("TreeNode::getDecimalVal: non-support conversion from binary string"); case CalpontSystemCatalog::BIGINT: diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index b4ebd8b42..f14d9656f 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -478,7 +478,8 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull) fResult.origIntVal = row.getUintField<8>(fInputIndex); break; - + case 16: + cout << __FILE__<< ":" <<__LINE__ << " Fix 16 Bytes ?" << endl; default: if (row.equals(CPNULLSTRMARK, fInputIndex)) isNull = true; diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.cpp b/dbcon/joblist/batchprimitiveprocessor-jl.cpp index 0f04c8a8e..523e9a44a 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.cpp +++ b/dbcon/joblist/batchprimitiveprocessor-jl.cpp @@ -667,7 +667,8 @@ void BatchPrimitiveProcessorJL::getTuples(messageqcpp::ByteStream& in, columnData[j]++; pos++; break; - + case 16: + cout << __FILE__<< ":" <<__LINE__ << " Fix 16 Bytes ?" << endl; default: cout << "BPP::getTuples(): bad column width of " << colWidths[j] << endl; diff --git a/dbcon/joblist/columncommand-jl.cpp b/dbcon/joblist/columncommand-jl.cpp index 1ea451b19..b8ef18f5d 100644 --- a/dbcon/joblist/columncommand-jl.cpp +++ b/dbcon/joblist/columncommand-jl.cpp @@ -269,7 +269,8 @@ uint8_t ColumnCommandJL::getTableColumnType() case 1: return TableColumn::UINT8; - + case 16: + cout << __FILE__<< ":" <<__LINE__ << " Fix 16 Bytes ?" << endl; default: throw logic_error("ColumnCommandJL: bad column width"); } diff --git a/dbcon/joblist/jlf_common.cpp b/dbcon/joblist/jlf_common.cpp index c40425b16..d6bd3e5a6 100644 --- a/dbcon/joblist/jlf_common.cpp +++ b/dbcon/joblist/jlf_common.cpp @@ -332,6 +332,8 @@ string extractTableAlias(const SSC& sc) //------------------------------------------------------------------------------ CalpontSystemCatalog::OID isDictCol(const CalpontSystemCatalog::ColType& colType) { + if (colType.colDataType == CalpontSystemCatalog::BINARY) return 0; + if (colType.colWidth > 8) return colType.ddn.dictOID; if (colType.colDataType == CalpontSystemCatalog::VARCHAR && diff --git a/dbcon/joblist/joblisttypes.h b/dbcon/joblist/joblisttypes.h index 6d5664c57..9ad0e071c 100644 --- a/dbcon/joblist/joblisttypes.h +++ b/dbcon/joblist/joblisttypes.h @@ -83,6 +83,9 @@ const uint16_t NULL_UINT16 = USMALLINTNULL; const uint32_t NULL_UINT32 = UINTNULL; const uint64_t NULL_UINT64 = UBIGINTNULL; +const uint64_t BINARYEMPTYROW = 0; +const uint64_t BINARYNULL = 0; + const std::string CPNULLSTRMARK("_CpNuLl_"); const std::string CPSTRNOTFOUND("_CpNoTf_"); diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index 631e9a597..b6cc575ae 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -699,6 +699,8 @@ bool LBIDList::CasualPartitionPredicate(const int64_t Min, uint64_t val = *(int64_t*)MsgDataPtr; value = static_cast(val); } + case 16: + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; } } else @@ -731,6 +733,8 @@ bool LBIDList::CasualPartitionPredicate(const int64_t Min, int64_t val = *(int64_t*)MsgDataPtr; value = val; } + case 16: + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; } } diff --git a/dbcon/joblist/passthrucommand-jl.cpp b/dbcon/joblist/passthrucommand-jl.cpp index a11571c87..222f9e5bd 100644 --- a/dbcon/joblist/passthrucommand-jl.cpp +++ b/dbcon/joblist/passthrucommand-jl.cpp @@ -73,7 +73,12 @@ PassThruCommandJL::PassThruCommandJL(const PassThruStep& p) case 8: tableColumnType = TableColumn::UINT64; break; - + + case 16: + case 32: + tableColumnType = TableColumn::STRING; + break; + default: throw logic_error("PassThruCommandJL(): bad column width?"); } diff --git a/dbcon/joblist/pcolscan.cpp b/dbcon/joblist/pcolscan.cpp index da35ad8ce..3a9384353 100644 --- a/dbcon/joblist/pcolscan.cpp +++ b/dbcon/joblist/pcolscan.cpp @@ -178,7 +178,7 @@ pColScanStep::pColScanStep( fColType.colWidth = 8; fIsDict = true; } - else if (fColType.colWidth > 8 ) + else if (fColType.colWidth > 8 && fColType.colDataType != CalpontSystemCatalog::BINARY) { fColType.colWidth = 8; fIsDict = true; diff --git a/dbcon/joblist/pcolstep.cpp b/dbcon/joblist/pcolstep.cpp index b9809e8e0..5b07d9374 100644 --- a/dbcon/joblist/pcolstep.cpp +++ b/dbcon/joblist/pcolstep.cpp @@ -177,7 +177,7 @@ pColStep::pColStep( fColType.colWidth = 8; fIsDict = true; } - else if (fColType.colWidth > 8 ) + else if (fColType.colWidth > 8 && fColType.colDataType != CalpontSystemCatalog::BINARY ) { fColType.colWidth = 8; fIsDict = true; diff --git a/dbcon/mysql/ha_mcs_ddl.cpp b/dbcon/mysql/ha_mcs_ddl.cpp index 82ea00a17..08cfe199d 100644 --- a/dbcon/mysql/ha_mcs_ddl.cpp +++ b/dbcon/mysql/ha_mcs_ddl.cpp @@ -251,6 +251,10 @@ uint32_t convertDataType(int dataType) case ddlpackage::DDL_UNSIGNED_DOUBLE: calpontDataType = CalpontSystemCatalog::UDOUBLE; break; + + case ddlpackage::DDL_BINARY: + calpontDataType = CalpontSystemCatalog::BINARY; + break; default: throw runtime_error("Unsupported datatype!"); diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 2f8cbe220..b3018be57 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -817,7 +817,16 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h break; } + case CalpontSystemCatalog::BINARY: + { + Field_varstring* f2 = (Field_varstring*)*f; + f2->store(row.getBinaryField(s).c_str(), 16, f2->charset()); + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + break; + } default: // treat as int64 { intColVal = row.getUintField<8>(s); diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index a8b3dc838..243858157 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -273,6 +273,21 @@ inline bool colStrCompare_(uint64_t val1, uint64_t val2, uint8_t COP, const idb_ template inline bool isEmptyVal(uint8_t type, const uint8_t* val8); +template<> +inline bool isEmptyVal<32>(uint8_t type, const uint8_t* ival) // For BINARY +{ + const uint64_t* val = reinterpret_cast(ival); + return ((val[0] == joblist::BINARYEMPTYROW) && (val[1] == joblist::BINARYEMPTYROW) + && (val[2] == joblist::BINARYEMPTYROW) && (val[3] == joblist::BINARYEMPTYROW)); +} + +template<> +inline bool isEmptyVal<16>(uint8_t type, const uint8_t* ival) // For BINARY +{ + const uint64_t* val = reinterpret_cast(ival); + return ((val[0] == joblist::BINARYEMPTYROW) && (val[1] == joblist::BINARYEMPTYROW)); +} + template<> inline bool isEmptyVal<8>(uint8_t type, const uint8_t* ival) { @@ -394,6 +409,21 @@ inline bool isEmptyVal<1>(uint8_t type, const uint8_t* ival) template inline bool isNullVal(uint8_t type, const uint8_t* val8); +template<> +inline bool isNullVal<16>(uint8_t type, const uint8_t* ival) // For BINARY +{ + const uint64_t* val = reinterpret_cast(ival); + return ((val[0] == joblist::BINARYNULL) && (val[1] == joblist::BINARYNULL)); +} + +template<> +inline bool isNullVal<32>(uint8_t type, const uint8_t* ival) // For BINARY +{ + const uint64_t* val = reinterpret_cast(ival); + return ((val[0] == joblist::BINARYNULL) && (val[1] == joblist::BINARYNULL) + && (val[2] == joblist::BINARYNULL) && (val[3] == joblist::BINARYNULL)); +} + template<> inline bool isNullVal<8>(uint8_t type, const uint8_t* ival) { @@ -521,6 +551,12 @@ inline bool isNullVal(uint32_t length, uint8_t type, const uint8_t* val8) { switch (length) { + case 32: + return isNullVal<32>(type, val8); + + case 16: + return isNullVal<16>(type, val8); + case 8: return isNullVal<8>(type, val8); @@ -703,6 +739,16 @@ inline void store(const NewColRequestHeader* in, switch (in->DataSize) { + case 32: + ptr2 += (rid << 5); + memcpy(ptr1, ptr2, 32); + break; + + case 16: + ptr2 += (rid << 4); + memcpy(ptr1, ptr2, 16); + break; + default: case 8: ptr2 += (rid << 3); @@ -724,7 +770,6 @@ inline void store(const NewColRequestHeader* in, memcpy(ptr1, ptr2, 1); break; } - *written += in->DataSize; } @@ -811,6 +856,66 @@ inline uint64_t nextUnsignedColValue(int type, return -1; } } +template +inline uint8_t* nextBinColValue(int type, + const uint16_t* ridArray, + int NVALS, + int* index, + bool* done, + bool* isNull, + bool* isEmpty, + uint16_t* rid, + uint8_t OutputType, uint8_t* val8, unsigned itemsPerBlk) +{ + if (ridArray == NULL) + { + while (static_cast(*index) < itemsPerBlk && + isEmptyVal(type, &val8[*index * W]) && + (OutputType & OT_RID)) + { + (*index)++; + } + + + if (static_cast(*index) >= itemsPerBlk) + { + *done = true; + return NULL; + } + *rid = (*index)++; + } + else + { + //FIXME: not complete nor tested . How make execution flow pass here + // whe is ridArray not NULL ? fidn by id? how? + while (*index < NVALS && + isEmptyVal(type, &val8[ridArray[*index] * W])) + { + (*index)++; + } + + if (*index >= NVALS) + { + *done = true; + return NULL; + } + *rid = ridArray[(*index)++]; + } + + *isNull = isNullVal(type, val8); + *isEmpty = isEmptyVal(type, val8); + //cout << "nextUnsignedColValue index " << *index << " rowid " << *rid << endl; + // at this point, nextRid is the index to return, and index is... + // if RIDs are not specified, nextRid + 1, + // if RIDs are specified, it's the next index in the rid array. + return &val8[*rid * W]; + +#ifdef PRIM_DEBUG + throw logic_error("PrimitiveProcessor::nextColBinValue() bad width"); +#endif + return NULL; +} +} template inline int64_t nextColValue(int type, @@ -1426,6 +1531,225 @@ inline void p_Col_ridArray(NewColRequestHeader* in, #endif } +// for BINARY +template +inline void p_Col_bin_ridArray(NewColRequestHeader* in, + NewColResultHeader* out, + unsigned outSize, + unsigned* written, int* block, Stats* fStatsPtr, unsigned itemsPerBlk, + boost::shared_ptr parsedColumnFilter) +{ + uint16_t* ridArray = 0; + uint8_t* in8 = reinterpret_cast(in); + const uint8_t filterSize = sizeof(uint8_t) + sizeof(uint8_t) + W; + idb_regex_t placeholderRegex; + placeholderRegex.used = false; + + //FIXME: pCol is setting it to 8192 cause logicalBlockMode is true + if(itemsPerBlk == BLOCK_SIZE){ + itemsPerBlk = BLOCK_SIZE/W; + } + + if (in->NVALS > 0) + ridArray = reinterpret_cast(&in8[sizeof(NewColRequestHeader) + + (in->NOPS * filterSize)]); + + if (ridArray && 1 == in->sort ) + { + qsort(ridArray, in->NVALS, sizeof(uint16_t), compareBlock); + + if (fStatsPtr) +#ifdef _MSC_VER + fStatsPtr->markEvent(in->LBID, GetCurrentThreadId(), in->hdr.SessionID, 'O'); + +#else + fStatsPtr->markEvent(in->LBID, pthread_self(), in->hdr.SessionID, 'O'); +#endif + } + + // Set boolean indicating whether to capture the min and max values. + out->ValidMinMax = isMinMaxValid(in); + + if (out->ValidMinMax) + { + if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType)) + { + out->Min = static_cast(numeric_limits::max()); + out->Max = 0; + } + else + { + out->Min = numeric_limits::max(); + out->Max = numeric_limits::min(); + } + } + else + { + out->Min = 0; + out->Max = 0; + } + + typedef char binWtype [W]; + + const ColArgs* args = NULL; + int64_t val = 0; + binWtype* bval; + int nextRidIndex = 0, argIndex = 0; + bool done = false, cmp = false, isNull = false, isEmpty = false; + uint16_t rid = 0; + prestored_set_t::const_iterator it; + + binWtype* argVals = (binWtype*)alloca(in->NOPS * W); + uint8_t* std_cops = (uint8_t*)alloca(in->NOPS * sizeof(uint8_t)); + uint8_t* std_rfs = (uint8_t*)alloca(in->NOPS * sizeof(uint8_t)); + uint8_t* cops = NULL; + uint8_t* rfs = NULL; + + scoped_array std_regex; + idb_regex_t* regex = NULL; + uint8_t likeOps = 0; + +// no pre-parsed column filter is set, parse the filter in the message + if (parsedColumnFilter.get() == NULL) { + std_regex.reset(new idb_regex_t[in->NOPS]); + regex = &(std_regex[0]); + + cops = std_cops; + rfs = std_rfs; + + for (argIndex = 0; argIndex < in->NOPS; argIndex++) { + args = reinterpret_cast (&in8[sizeof (NewColRequestHeader) + + (argIndex * filterSize)]); + cops[argIndex] = args->COP; + rfs[argIndex] = args->rf; + + memcpy(argVals[argIndex],args->val, W); + } + + regex[argIndex].used = false; + } + + + // else we have a pre-parsed filter, and it's an unordered set for quick == comparisons + bval = (binWtype*)nextBinColValue(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull, + &isEmpty, &rid, in->OutputType, reinterpret_cast(block), itemsPerBlk); + + while (!done) + { + +// if((*((uint64_t *) (bval))) != 0) +// { +// cout << "rid "<< rid << " value "; +// if(W > 16) printf("%016X%016X ",( *(((uint64_t *) (bval)) +3)),(*(((uint64_t *) (bval)) +2))); +// printf("%016X%016X ",( *(((uint64_t *) (bval)) +1)),(*((uint64_t *) (bval))) ); +// +// cout << endl; +// } + + if (cops == NULL) // implies parsedColumnFilter && columnFilterMode == SET + { + /* bug 1920: ignore NULLs in the set and in the column data */ + if (!(isNull && in->BOP == BOP_AND)) + { + + it = parsedColumnFilter->prestored_set->find(val); + + + if (in->BOP == BOP_OR) + { + // assume COP == COMPARE_EQ + if (it != parsedColumnFilter->prestored_set->end()) + { + store(in, out, outSize, written, rid, reinterpret_cast(block)); + } + } + else if (in->BOP == BOP_AND) + { + // assume COP == COMPARE_NE + if (it == parsedColumnFilter->prestored_set->end()) + { + store(in, out, outSize, written, rid, reinterpret_cast(block)); + } + } + } + } + else + { + for (argIndex = 0; argIndex < in->NOPS; argIndex++) + { + +// if((*((uint64_t *) (uval))) != 0) cout << "comparing " << dec << (*((uint64_t *) (uval))) << " to " << (*((uint64_t *) (argVals[argIndex]))) << endl; + + int val1 = memcmp(*bval, &argVals[argIndex], W); + + switch (cops[argIndex]) { + case COMPARE_NIL: + cmp = false; + break; + case COMPARE_LT: + cmp = val1 < 0; + break; + case COMPARE_EQ: + cmp = val1 == 0; + break; + case COMPARE_LE: + cmp = val1 <= 0; + break; + case COMPARE_GT: + cmp = val1 > 0; + break; + case COMPARE_NE: + cmp = val1 != 0; + break; + case COMPARE_GE: + cmp = val1 >= 0; + break; + default: + logIt(34, cops[argIndex], "colCompare"); + cmp = false; // throw an exception here? + } + +// cout << cmp << endl; + + if (in->NOPS == 1) + { + if (cmp == true) + { + store(in, out, outSize, written, rid, reinterpret_cast(block)); + } + + break; + } + else if (in->BOP == BOP_AND && cmp == false) + { + break; + } + else if (in->BOP == BOP_OR && cmp == true) + { + store(in, out, outSize, written, rid, reinterpret_cast(block)); + break; + } + } + + if ((argIndex == in->NOPS && in->BOP == BOP_AND) || in->NOPS == 0) + { + store(in, out, outSize, written, rid, reinterpret_cast(block)); + } + } + + bval = (binWtype*)nextBinColValue(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull, + &isEmpty, &rid, in->OutputType, reinterpret_cast(block), itemsPerBlk); + + } + + if (fStatsPtr) +#ifdef _MSC_VER + fStatsPtr->markEvent(in->LBID, GetCurrentThreadId(), in->hdr.SessionID, 'K'); + +#else + fStatsPtr->markEvent(in->LBID, pthread_self(), in->hdr.SessionID, 'K'); +#endif + } //namespace anon namespace primitives @@ -1476,6 +1800,14 @@ void PrimitiveProcessor::p_Col(NewColRequestHeader* in, NewColResultHeader* out, switch (in->DataSize) { + case 32: + p_Col_bin_ridArray<32>(in, out, outSize, written, block, fStatsPtr, itemsPerBlk, parsedColumnFilter); + break; + + case 16: + p_Col_bin_ridArray<16>(in, out, outSize, written, block, fStatsPtr, itemsPerBlk, parsedColumnFilter); + break; + case 8: p_Col_ridArray<8>(in, out, outSize, written, block, fStatsPtr, itemsPerBlk, parsedColumnFilter); break; @@ -1578,7 +1910,9 @@ boost::shared_ptr parseColumnFilter case 8: ret->prestored_argVals[argIndex] = *reinterpret_cast(args->val); - break; + break; + case 16: + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; } } else @@ -1614,6 +1948,8 @@ boost::shared_ptr parseColumnFilter case 8: ret->prestored_argVals[argIndex] = *reinterpret_cast(args->val); break; + case 16: + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; } } diff --git a/primitives/linux-port/tdriver.cpp b/primitives/linux-port/tdriver.cpp index 0d7a00c9b..0d80141a5 100644 --- a/primitives/linux-port/tdriver.cpp +++ b/primitives/linux-port/tdriver.cpp @@ -39,9 +39,9 @@ #include #include "primitiveprocessor.h" +using namespace primitives; using namespace std; - int done; void alarm_handler(int sig) @@ -87,7 +87,6 @@ class PrimTest : public CppUnit::TestFixture CPPUNIT_TEST(p_IdxList_1); CPPUNIT_TEST(p_IdxList_2); - // whole block tests CPPUNIT_TEST(p_Col_1); CPPUNIT_TEST(p_Col_2); @@ -162,7 +161,11 @@ class PrimTest : public CppUnit::TestFixture // CPPUNIT_TEST(p_Dictionary_like_prefixbench_1); // CPPUNIT_TEST(p_Dictionary_like_substrbench_1); - + +// binary data type + CPPUNIT_TEST(p_Col_bin_16); + CPPUNIT_TEST(p_Col_bin_32); + CPPUNIT_TEST_SUITE_END(); private: @@ -3744,6 +3747,178 @@ public: close(fd); } + + template struct binary; + typedef binary<16> binary16; + typedef binary<32> binary32; + template + struct binary { + unsigned char data[W]; // May be ok for empty value ? + void operator=(uint64_t v) {*((uint64_t *) data) = v; memset(data + 8, 0, W - 8);} + inline uint8_t& operator[](const int index) {return *((uint8_t*) (data + index));} + inline uint64_t& uint64(const int index) {return *((uint64_t*) (data + (index << 3)));} + }; + + void p_Col_bin_16() + { + PrimitiveProcessor pp; + uint8_t input[BLOCK_SIZE], output[4 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + NewColResultHeader* out; + ColArgs* args; + binary16* results; + uint32_t written, i; + int fd; + binary16 tmp; + binary16* bin16 = (binary16*) block; + + for(int i = 0; i < BLOCK_SIZE/16; i++) + { + bin16[i] = 0; + } + + bin16[0].uint64(0) = 10UL; + + bin16[1].uint64(0) = 1000UL; + + bin16[3].uint64(0) = 1000UL; + bin16[3].uint64(1) = 1; + + bin16[4].uint64(0) = 256; + bin16[4].uint64(1) = 1; + + typedef char bin16_t[16]; + + *(uint64_t*)(((bin16_t*)block) + 5) = 500; + + *(uint64_t*)&((bin16_t*)block)[6] = 501; + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); + + in->DataSize = sizeof(binary16); + in->DataType = execplan::CalpontSystemCatalog::BINARY; + in->OutputType = OT_DATAVALUE; + in->NOPS = 3; + in->BOP = BOP_OR; + in->NVALS = 0; + + tmp = 10; + args->COP = COMPARE_EQ; + memcpy(args->val, &tmp, in->DataSize); + args = reinterpret_cast (args->val + in->DataSize); + + args->COP = COMPARE_EQ; + tmp = 1000; + memcpy(args->val, &tmp, in->DataSize); + + args = reinterpret_cast (args->val + in->DataSize); + tmp.uint64(0) = 256; + tmp.uint64(1) = 1; + args->COP = COMPARE_EQ; + memcpy(args->val, &tmp, in->DataSize); + + pp.setBlockPtr((int*) block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + results = reinterpret_cast(&output[sizeof(NewColResultHeader)]); +// cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT_EQUAL((uint16_t)3, out->NVALS); + CPPUNIT_ASSERT_EQUAL((u_int64_t)10, results[0].uint64(0)); + CPPUNIT_ASSERT_EQUAL((u_int64_t)1000, results[1].uint64(0)); + for (i = 0; i < out->NVALS; i++) { + printf("Result %d Value %016X%016X\n",i ,results[i].uint64(1),results[i].uint64(0) ); +// CPPUNIT_ASSERT(results[i] == (uint32_t) (i < 10 ? i : i - 10 + 1001)); + } + } + + void p_Col_bin_32() + { + PrimitiveProcessor pp; + uint8_t input[2 * BLOCK_SIZE], output[8 * BLOCK_SIZE], block[BLOCK_SIZE]; + NewColRequestHeader* in; + NewColResultHeader* out; + ColArgs* args; + binary32* results; + uint32_t written, i; + int fd; + binary32 tmp; + binary32* bin32 = (binary32*) block; + + for(int i = 0; i < BLOCK_SIZE/32; i++) + { + bin32[i].uint64(0) = 0; + } + + bin32[0].uint64(0) = 10UL; + + bin32[1].uint64(0) = 1000UL; + + bin32[3].uint64(0) = 1000UL; + bin32[3].uint64(1) = 1; + + bin32[4].uint64(0) = 256; + bin32[4].uint64(1) = 254; + bin32[4].uint64(2) = 253; + bin32[4].uint64(3) = 252; + + typedef char bin32_t[32]; + + *(uint64_t*)(((bin32_t*)block) + 5) = 500; + + *(uint64_t*)&((bin32_t*)block)[6] = 501; + + memset(input, 0, BLOCK_SIZE); + memset(output, 0, 4 * BLOCK_SIZE); + + in = reinterpret_cast(input); + out = reinterpret_cast(output); + args = reinterpret_cast(&input[sizeof(NewColRequestHeader)]); + + in->DataSize = sizeof(binary32); + in->DataType = execplan::CalpontSystemCatalog::BINARY; + in->OutputType = OT_DATAVALUE; + in->NOPS = 3; + in->BOP = BOP_OR; + in->NVALS = 0; + + tmp = 10; + args->COP = COMPARE_EQ; + memcpy(args->val, &tmp, in->DataSize); + args = reinterpret_cast (args->val + in->DataSize); + + args->COP = COMPARE_EQ; + tmp = 1000; + memcpy(args->val, &tmp, in->DataSize); + + args = reinterpret_cast (args->val + in->DataSize); + tmp.uint64(0) = 256; + tmp.uint64(1) = 254; + tmp.uint64(2) = 253; + tmp.uint64(3) = 252; + + args->COP = COMPARE_EQ; + memcpy(args->val, &tmp, in->DataSize); + + pp.setBlockPtr((int*) block); + pp.p_Col(in, out, 4 * BLOCK_SIZE, &written); + + results = reinterpret_cast(&output[sizeof(NewColResultHeader)]); +// cout << "NVALS = " << out->NVALS << endl; + CPPUNIT_ASSERT_EQUAL((uint16_t)3, out->NVALS); +// CPPUNIT_ASSERT_EQUAL((u_int64_t)10, results[0].uint64(0)); +// CPPUNIT_ASSERT_EQUAL((u_int64_t)1000, results[1].uint64(0)); + for (i = 0; i < out->NVALS; i++) { + printf("Result %d Value %016X%016X%016X%016X\n",i ,results[i].uint64(3),results[i].uint64(2),results[i].uint64(1),results[i].uint64(0) ); +// CPPUNIT_ASSERT(results[i] == (uint32_t) (i < 10 ? i : i - 10 + 1001)); + } + } + + void p_Dictionary_1() { diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index db90b3577..7f24f6931 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -245,7 +245,7 @@ void ColumnCommand::issuePrimitive() bpp->pp.setParsedColumnFilter(parsedColumnFilter); else bpp->pp.setParsedColumnFilter(emptyFilter); - + bpp->pp.p_Col(primMsg, outMsg, bpp->outMsgSize, (unsigned int*)&resultSize); /* Update CP data, the PseudoColumn code should always be !_isScan. Should be safe @@ -273,6 +273,31 @@ void ColumnCommand::process_OT_BOTH() /* this is verbose and repetative to minimize the work per row */ switch (colType.colWidth) { + case 16: + for (i = 0, pos = sizeof(NewColResultHeader); i < outMsg->NVALS; ++i) + { + if (makeAbsRids) + bpp->absRids[i] = *((uint16_t*) &bpp->outputMsg[pos]) + bpp->baseRid; + + bpp->relRids[i] = *((uint16_t*) &bpp->outputMsg[pos]); + pos += 2; + // values[i] is 8 Bytes wide so coping the pointer to bpp->outputMsg[pos] and crossing fingers + // I dont know the liveness of bpp->outputMsg but also I dont know if there is other memory area I can use + values[i] = (int64_t) &bpp->outputMsg[pos]; + +// cout<< "CC: BIN16 " << i << " " +// << hex +// << *((int64_t*)values[i]) +// << " " +// << *(((int64_t*)values[i]) +1) +// << endl; + pos += 16; + } + + break; + + + case 8: for (i = 0, pos = sizeof(NewColResultHeader); i < outMsg->NVALS; ++i) { @@ -346,6 +371,14 @@ void ColumnCommand::process_OT_DATAVALUE() // cout << "rid Count is " << bpp->ridCount << endl; switch (colType.colWidth) { + case 16: + { + memcpy(values, outMsg + 1, outMsg->NVALS << 3); + cout << " CC: first value is " << values[0] << endl; + break; + } + + case 8: { memcpy(values, outMsg + 1, outMsg->NVALS << 3); @@ -459,6 +492,9 @@ void ColumnCommand::createCommand(ByteStream& bs) bs >> BOP; bs >> filterCount; deserializeInlineVector(bs, lastLbid); + +// cout << __func__ << " colType.colWidth " << colType.colWidth << endl; + // cout << "lastLbid count=" << lastLbid.size() << endl; // for (uint32_t i = 0; i < lastLbid.size(); i++) // cout << " " << lastLbid[i]; @@ -488,7 +524,7 @@ void ColumnCommand::resetCommand(ByteStream& bs) void ColumnCommand::prep(int8_t outputType, bool absRids) { /* make the template NewColRequestHeader */ - + baseMsgLength = sizeof(NewColRequestHeader) + (suppressFilter ? 0 : filterString.length()); @@ -554,7 +590,12 @@ void ColumnCommand::prep(int8_t outputType, bool absRids) shift = 1; mask = 0x01; break; - + case 16: + cout << __FILE__<< ":" <<__LINE__ << " Fix shift and mask for 16 Bytes ?"<< endl; + shift = 1; + mask = 0x01; + break; + default: cout << "CC: colWidth is " << colType.colWidth << endl; throw logic_error("ColumnCommand: bad column width?"); @@ -751,6 +792,9 @@ void ColumnCommand::projectResultRG(RowGroup& rg, uint32_t pos) break; } + + case 16: + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; } } diff --git a/primitives/primproc/passthrucommand.cpp b/primitives/primproc/passthrucommand.cpp index 407c257dc..d33d7a6f0 100644 --- a/primitives/primproc/passthrucommand.cpp +++ b/primitives/primproc/passthrucommand.cpp @@ -77,6 +77,9 @@ void PassThruCommand::project() switch (colWidth) { + case 16: + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + case 8: bpp->serialized->append((uint8_t*) bpp->values, bpp->ridCount << 3); break; @@ -153,6 +156,21 @@ void PassThruCommand::projectIntoRowGroup(RowGroup& rg, uint32_t col) } break; + case 16: + cout << __FILE__ << ":" << __LINE__ << " PassThruCommand::projectIntoRowGroup" << " Addition for 16 Bytes" << endl; + for (i = 0; i < bpp->ridCount; i++) + { + cout << "PTC: " << "BIN16 " << i << " " + << hex + << *((int64_t*) bpp->values[i]) + << " " + << *(((int64_t*) bpp->values[i]) +1) + << endl; + // values[i] is 8 bytes so it contains the pointer to bpp->outputMsg set by ColumnCommand::process_OT_BOTH() + r.setBinaryField((uint8_t*)bpp->values[i], 16, offset); + + r.nextRow(rowSize); + } } } diff --git a/primitives/primproc/pseudocc.cpp b/primitives/primproc/pseudocc.cpp index 1ee33a04b..7550c1b47 100644 --- a/primitives/primproc/pseudocc.cpp +++ b/primitives/primproc/pseudocc.cpp @@ -91,7 +91,8 @@ void PseudoCC::loadData() case 8: loadPMNumber(); break; - + case 16: + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; default: cout << "PC::loadData(): bad column width" << endl; break; @@ -143,7 +144,8 @@ void PseudoCC::loadData() case 8: loadSegmentNum(); break; - + case 16: + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; default: cout << "PC::loadData(): bad column width" << endl; break; @@ -170,6 +172,8 @@ void PseudoCC::loadData() loadPartitionNum(); break; + case 16: + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; default: cout << "PC::loadData(): bad column width" << endl; break; @@ -196,6 +200,8 @@ void PseudoCC::loadData() loadLBID(); break; + case 16: + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; default: cout << "PC::loadData(): bad column width" << endl; break; @@ -221,7 +227,9 @@ void PseudoCC::loadData() case 8: loadDBRootNum(); break; - + case 16: + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + default: cout << "PC::loadData(): bad column width" << endl; break; @@ -250,7 +258,9 @@ void PseudoCC::loadData() case 8: loadSingleValue(valueFromUM); break; - + case 16: + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + default: cout << "PC::loadData(): bad column width" << endl; break; diff --git a/utils/common/nullvaluemanip.cpp b/utils/common/nullvaluemanip.cpp index 3b4255dea..63b0c8693 100644 --- a/utils/common/nullvaluemanip.cpp +++ b/utils/common/nullvaluemanip.cpp @@ -125,7 +125,10 @@ uint64_t getNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidth) case CalpontSystemCatalog::UBIGINT: return joblist::UBIGINTNULL; - + + case CalpontSystemCatalog::BINARY: + return joblist::BINARYNULL; + case CalpontSystemCatalog::VARBINARY: default: ostringstream os; @@ -239,6 +242,9 @@ int64_t getSignedNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidt case CalpontSystemCatalog::LONGDOUBLE: return (int64_t)joblist::LONGDOUBLENULL; + case CalpontSystemCatalog::BINARY: + return (int64_t)joblist::BINARYNULL; + case CalpontSystemCatalog::VARBINARY: default: ostringstream os; diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index d920f5ab3..f89c73d16 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1531,6 +1531,10 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, value = data; break; + case CalpontSystemCatalog::BINARY: + value = data; + break; + default: throw QueryDataExcept("convertColumnData: unknown column data type.", dataTypeErr); break; @@ -1727,6 +1731,12 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } break; + case CalpontSystemCatalog::BINARY: + { + value = data; + } + break; + case CalpontSystemCatalog::UTINYINT: { uint8_t utinyintvalue = joblist::UTINYINTNULL; diff --git a/utils/funcexp/func_hex.cpp b/utils/funcexp/func_hex.cpp index 42e6b04d2..97e608981 100644 --- a/utils/funcexp/func_hex.cpp +++ b/utils/funcexp/func_hex.cpp @@ -130,6 +130,15 @@ string Func_hex::getStrVal(rowgroup::Row& row, return string(hexPtr.get(), hexLen); } + case CalpontSystemCatalog::BINARY: + { + const string& arg = parm[0]->data()->getStrVal(row, isNull); + uint64_t hexLen = arg.size() * 2; + scoped_array hexPtr(new char[hexLen + 1]); // "+ 1" for the last \0 + octet2hex(hexPtr.get(), arg.data(), arg.size()); + return string(hexPtr.get(), hexLen); + } + default: { dec = (uint64_t)parm[0]->data()->getIntVal(row, isNull); diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 9f16123cf..df10ed108 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -628,7 +628,8 @@ string Row::toString() const os << " " << dec; break; } - + case CalpontSystemCatalog::BINARY: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; default: os << getIntField(i) << " "; break; @@ -690,7 +691,8 @@ string Row::toCSV() const os << dec; break; } - + case CalpontSystemCatalog::BINARY: + std::cout << __FILE__<< __LINE__ << ":" << "toCSV"<< std::endl; default: os << getIntField(i); break; @@ -852,7 +854,8 @@ void Row::initToNull() case CalpontSystemCatalog::UBIGINT: *((uint64_t*) &data[offsets[i]]) = joblist::UBIGINTNULL; break; - + case CalpontSystemCatalog::BINARY: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; default: ostringstream os; os << "Row::initToNull(): got bad column type (" << types[i] << @@ -934,7 +937,8 @@ bool Row::isNullValue(uint32_t colIndex) const case 8: return (*((uint64_t*) &data[offsets[colIndex]]) == joblist::CHAR8NULL); - + case 16: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; default: return (*((uint64_t*) &data[offsets[colIndex]]) == *((uint64_t*) joblist::CPNULLSTRMARK.c_str())); } @@ -1004,6 +1008,16 @@ bool Row::isNullValue(uint32_t colIndex) const return (*((long double*) &data[offsets[colIndex]]) == joblist::LONGDOUBLENULL); break; + case CalpontSystemCatalog::BINARY: + { + // When is null? I dont know. Wait for bitmap null empty implemtenttion ? + // Also still pendig rework discussed use pointers for empty null values + + std::cout << __FILE__<< ":" << __LINE__ << " isNullValue value " << (*((uint64_t*) &data[offsets[colIndex]])) << std::endl; + //return false; + return (*((uint64_t*) &data[offsets[colIndex]]) == joblist::BINARYEMPTYROW); + } + default: { ostringstream os; @@ -1624,7 +1638,8 @@ void RowGroup::addToSysDataList(execplan::CalpontSystemCatalog::NJLSysDataList& case 8: cr->PutData(row.getUintField<8>(j)); break; - + case 16: + default: { string s = row.getStringField(j); @@ -1645,6 +1660,8 @@ void RowGroup::addToSysDataList(execplan::CalpontSystemCatalog::NJLSysDataList& cr->PutData(row.getUintField<4>(j)); break; + case CalpontSystemCatalog::BINARY: + std::cout << __FILE__<< __LINE__ << __func__<< std::endl; default: cr->PutData(row.getIntField<8>(j)); } diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index e3b3652e9..c1789dc18 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -28,6 +28,7 @@ #ifndef ROWGROUP_H_ #define ROWGROUP_H_ +#include #include #include #include @@ -421,7 +422,7 @@ public: inline uint32_t getStringLength(uint32_t colIndex) const; void setStringField(const std::string& val, uint32_t colIndex); inline void setStringField(const uint8_t*, uint32_t len, uint32_t colIndex); - + inline void setBinaryField(const uint8_t* strdata, uint32_t length, uint32_t offset); // support VARBINARY // Add 2-byte length at the CHARSET_INFO*beginning of the field. NULL and zero length field are // treated the same, could use one of the length bit to distinguish these two cases. @@ -433,6 +434,8 @@ public: inline const uint8_t* getVarBinaryField(uint32_t& len, uint32_t colIndex) const; inline void setVarBinaryField(const uint8_t* val, uint32_t len, uint32_t colIndex); + inline std::string getBinaryField(uint32_t colIndex) const; + inline boost::shared_ptr getUserData(uint32_t colIndex) const; inline void setUserData(mcsv1sdk::mcsv1Context& context, boost::shared_ptr userData, @@ -664,7 +667,8 @@ inline bool Row::equals(uint64_t val, uint32_t colIndex) const case 8: return *((uint64_t*) &data[offsets[colIndex]]) == val; - + case 16: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; default: idbassert(0); throw std::logic_error("Row::equals(): bad length."); @@ -692,7 +696,8 @@ inline uint64_t Row::getUintField(uint32_t colIndex) const case 8: return *((uint64_t*) &data[offsets[colIndex]]); - + case 16: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; default: idbassert(0); throw std::logic_error("Row::getUintField(): bad length."); @@ -711,7 +716,8 @@ inline uint64_t Row::getUintField(uint32_t colIndex) const case 4: return *((uint32_t*) &data[offsets[colIndex]]); - + case 16: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; case 8: return *((uint64_t*) &data[offsets[colIndex]]); @@ -784,6 +790,12 @@ inline uint32_t Row::getStringLength(uint32_t colIndex) const return strnlen((char*) &data[offsets[colIndex]], getColumnWidth(colIndex)); } + +inline void Row::setBinaryField(const uint8_t* strdata, uint32_t length, uint32_t offset) +{ + memcpy(&data[offset], strdata, length); +} + inline void Row::setStringField(const uint8_t* strdata, uint32_t length, uint32_t colIndex) { uint64_t offset; @@ -817,6 +829,11 @@ inline std::string Row::getStringField(uint32_t colIndex) const strnlen((char*) &data[offsets[colIndex]], getColumnWidth(colIndex))); } +inline std::string Row::getBinaryField(uint32_t colIndex) const +{ + return std::string((char*) &data[offsets[colIndex]], getColumnWidth(colIndex)); +} + inline std::string Row::getVarBinaryStringField(uint32_t colIndex) const { if (inStringTable(colIndex)) @@ -936,7 +953,10 @@ inline void Row::setUintField_offset(uint64_t val, uint32_t offset) case 8: *((uint64_t*) &data[offset]) = val; break; - + case 16: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; + *((uint64_t*) &data[offset]) = val; + break; default: idbassert(0); throw std::logic_error("Row::setUintField called on a non-uint32_t field"); @@ -974,7 +994,10 @@ inline void Row::setUintField(uint64_t val, uint32_t colIndex) case 8: *((uint64_t*) &data[offsets[colIndex]]) = val; break; - + case 16: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; + *((uint64_t*) &data[offsets[colIndex]]) = val; + break; default: idbassert(0); throw std::logic_error("Row::setUintField called on a non-uint32_t field"); @@ -1000,7 +1023,9 @@ inline void Row::setUintField(uint64_t val, uint32_t colIndex) case 8: *((uint64_t*) &data[offsets[colIndex]]) = val; break; - + case 16: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; + *((uint64_t*) &data[offsets[colIndex]]) = val; default: idbassert(0); throw std::logic_error("Row::setUintField: bad length"); @@ -1027,7 +1052,8 @@ inline void Row::setIntField(int64_t val, uint32_t colIndex) case 8: *((int64_t*) &data[offsets[colIndex]]) = val; break; - + case 16: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; default: idbassert(0); throw std::logic_error("Row::setIntField: bad length"); @@ -1053,7 +1079,8 @@ inline void Row::setIntField(int64_t val, uint32_t colIndex) case 8: *((int64_t*) &data[offsets[colIndex]]) = val; break; - + case 16: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; default: idbassert(0); throw std::logic_error("Row::setIntField: bad length"); diff --git a/utils/thrift/thrift/protocol/TCompactProtocol.tcc b/utils/thrift/thrift/protocol/TCompactProtocol.tcc index 62d6485d7..89c943a8c 100644 --- a/utils/thrift/thrift/protocol/TCompactProtocol.tcc +++ b/utils/thrift/thrift/protocol/TCompactProtocol.tcc @@ -808,6 +808,8 @@ TType TCompactProtocolT::getTType(int8_t type) { case detail::compact::CT_STRUCT: return T_STRUCT; default: + + cout << __FILE__<< __LINE__ << __func__<< endl; throw TException(std::string("don't know what type: ") + (char)type); } return T_STOP; diff --git a/utils/windowfunction/wf_count.cpp b/utils/windowfunction/wf_count.cpp index 1e18c6a74..7af4acb0a 100644 --- a/utils/windowfunction/wf_count.cpp +++ b/utils/windowfunction/wf_count.cpp @@ -19,6 +19,7 @@ //#define NDEBUG +#include #include #include #include @@ -68,7 +69,8 @@ boost::shared_ptr WF_count::makeFunction(int id, const st break; } - default: + case CalpontSystemCatalog::BINARY: + std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; { func.reset(new WF_count(id, name)); break; diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index 2b8d7a4a9..c1dc5a908 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -19,6 +19,7 @@ //#define NDEBUG +#include #include #include #include @@ -490,7 +491,8 @@ bool WF_udaf::dropValues(int64_t b, int64_t e) datum.columnData = valIn; break; } - + case CalpontSystemCatalog::BINARY: + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; default: { string errStr = "(" + colType2String[(int)datum.dataType] + ")"; @@ -754,7 +756,8 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, setValue(colDataType, b, e, c, &strOut); } break; - + case CalpontSystemCatalog::BINARY: + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; default: { std::ostringstream errmsg; @@ -1101,7 +1104,8 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) datum.columnData = valIn; break; } - + case CalpontSystemCatalog::BINARY: + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; default: { string errStr = "(" + colType2String[(int)datum.dataType] + ")"; diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index cea636402..600f5784f 100644 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -19,6 +19,7 @@ */ //#define NDEBUG +#include #include #include #include @@ -741,7 +742,8 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos) case CalpontSystemCatalog::LONGDOUBLE: v = &longDoubleNull; break; - + case CalpontSystemCatalog::BINARY: + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; case CalpontSystemCatalog::VARBINARY: default: std::ostringstream oss; diff --git a/writeengine/server/we_ddlcommandproc.cpp b/writeengine/server/we_ddlcommandproc.cpp index 0b557a2c4..fd5fe2a5d 100644 --- a/writeengine/server/we_ddlcommandproc.cpp +++ b/writeengine/server/we_ddlcommandproc.cpp @@ -506,6 +506,16 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err throw std::runtime_error(os.str()); } + else if (dataType == CalpontSystemCatalog::BINARY + && ! (colDefPtr->fType->fLength == 16 + || colDefPtr->fType->fLength == 32)) + { + ostringstream os; + os << "binary length may not be other than 16 or 32"; + throw std::runtime_error(os.str()); + } + + unsigned int i = 0; column_iterator = columns.begin(); diff --git a/writeengine/server/we_ddlcommon.h b/writeengine/server/we_ddlcommon.h index 9fcc06f7b..d477880f1 100644 --- a/writeengine/server/we_ddlcommon.h +++ b/writeengine/server/we_ddlcommon.h @@ -495,6 +495,10 @@ inline int convertDataType(int dataType) case ddlpackage::DDL_UNSIGNED_DOUBLE: calpontDataType = execplan::CalpontSystemCatalog::UDOUBLE; break; + + case ddlpackage::DDL_BINARY: + calpontDataType = execplan::CalpontSystemCatalog::BINARY; + break; default: throw runtime_error("Unsupported datatype!"); diff --git a/writeengine/shared/shared_components_tests.cpp b/writeengine/shared/shared_components_tests.cpp index 682bf1251..db2f9d48c 100644 --- a/writeengine/shared/shared_components_tests.cpp +++ b/writeengine/shared/shared_components_tests.cpp @@ -91,7 +91,8 @@ CPPUNIT_TEST(setUp); // Extent & dict related testing CPPUNIT_TEST( testExtensionWOPrealloc ); CPPUNIT_TEST( testDictExtensionWOPrealloc ); -// Semaphore related testing + CPPUNIT_TEST( testExtentCrWOPreallocBin ); + // Semaphore related testing // CPPUNIT_TEST( testSem ); // Log related testing @@ -1542,7 +1543,180 @@ public: } */ + template struct binary; + typedef binary<16> binary16; + typedef binary<32> binary32; + template + struct binary { + unsigned char data[W]; // May be ok for empty value ? + void operator=(uint64_t v) {*((uint64_t *) data) = v; memset(data + 8, 0, W - 8);} + inline uint8_t& operator[](const int index) {return *((uint8_t*) (data + index));} + inline uint64_t& uint64(const int index) {return *((uint64_t*) (data + (index << 3)));} + }; + + void testExtentCrWOPreallocBin() { + IDBDataFile* pFile = NULL; + ColumnOpCompress1 fileOp; + BlockOp blockOp; + char fileName[20]; + int rc; + char hdrs[ IDBCompressInterface::HDR_BUF_LEN * 2 ]; + int dbRoot = 1; + idbdatafile::IDBPolicy::init(true, false, "", 0); + // Set to versionbuffer to satisfy IDBPolicy::getType + strcpy(fileName, "versionbuffer"); + fileOp.compressionType(1); + + fileOp.deleteFile(fileName); + CPPUNIT_ASSERT(fileOp.exists(fileName) == false); + + //binary16 emptyVal = blockOp.getEmptyBinRowValue( execplan::CalpontSystemCatalog::BINARY, 16 ); + uint64_t emptyVal = blockOp.getEmptyRowValue(execplan::CalpontSystemCatalog::BIGINT, 8); + int width = blockOp.getCorrectRowWidth(execplan::CalpontSystemCatalog::BINARY, sizeof (binary16)); + int nBlocks = INITIAL_EXTENT_ROWS_TO_DISK / BYTE_PER_BLOCK * width; + + // createFile runs IDBDataFile::open + initAbrevCompColumnExtent + // under the hood + // bigint column file + rc = fileOp.createFile(fileName, + nBlocks, // number of blocks + emptyVal, // NULL value + width, // width + dbRoot); // dbroot + CPPUNIT_ASSERT(rc == NO_ERROR); + + fileOp.closeFile(pFile); + + // open created compressed file and check its header + pFile = IDBDataFile::open(IDBPolicy::getType(fileName, + IDBPolicy::WRITEENG), fileName, "rb", dbRoot); + + rc = pFile->seek(0, 0); + CPPUNIT_ASSERT(rc == NO_ERROR); + rc = fileOp.readHeaders(pFile, hdrs); + CPPUNIT_ASSERT(rc == NO_ERROR); + // Couldn't use IDBDataFile->close() here w/o excplicit cast + fileOp.closeFile(pFile); + + // Extend the extent up to 64MB + pFile = IDBDataFile::open(IDBPolicy::getType(fileName, + IDBPolicy::WRITEENG), fileName, "rb", dbRoot); + + // disable disk space preallocation + idbdatafile::IDBPolicy::setPreallocSpace(dbRoot); + rc = fileOp.initColumnExtent(pFile, + dbRoot, + BYTE_PER_BLOCK - nBlocks, // number of blocks + emptyVal, + width, + false, // use existing file + false, // don't expand; new extent + false, // add full (not abbreviated) extent + true); // optimize extention + + CPPUNIT_ASSERT(rc == NO_ERROR); + fileOp.closeFile(pFile); + // file has been extended + cout << endl << "file has been extended"; + + // write up to INITIAL_EXTENT_ROWS_TO_DISK + 1 rows into the file + + Column curCol; + binary16 valArray[INITIAL_EXTENT_ROWS_TO_DISK + 1]; + RID rowIdArray[INITIAL_EXTENT_ROWS_TO_DISK + 1]; + // This is the magic for the stub in FileOp::oid2FileName + int fid = 42; + + for (uint64_t it = 0; it <= INITIAL_EXTENT_ROWS_TO_DISK; it++) { + rowIdArray[it] = it; + valArray[it].uint64(0) = it + 3; + valArray[it].uint64(1) = it + 5; + } + + fileOp.initColumn(curCol); + fileOp.setColParam(curCol, + 1, // column number + width, + execplan::CalpontSystemCatalog::BINARY, + WriteEngine::WR_BINARY, + fid, + 1); //compression type + + string segFile; + // openColumnFile uses DBRM's oid server but we + // have to get the chunks' pointers from the header. + curCol.dataFile.pFile = fileOp.openFile( + curCol, + dbRoot, + 0, + 0, + segFile, + false, + "r+b", + BYTE_PER_BLOCK * BYTE_PER_BLOCK); // buffer size is 64MB + + CPPUNIT_ASSERT(rc == NO_ERROR); + + rc = fileOp.writeRow(curCol, INITIAL_EXTENT_ROWS_TO_DISK + 1, + (RID*) rowIdArray, valArray); + CPPUNIT_ASSERT_EQUAL(NO_ERROR, rc); // I prefer this way as it prints values + + // flush and close the file used for reading + fileOp.clearColumn(curCol); + + std::map oids; + oids[fid] = fid; + + // flush changed chunks from the Manager + int rtn1 = fileOp.chunkManager()->flushChunks(rc, oids); + + // read back the file + cout << endl << "Read file "; + DataBlock block; + binary16* bin16 = (binary16*) block.data; + + fileOp.initColumn(curCol); + fileOp.setColParam(curCol, + 1, // column number + width, + execplan::CalpontSystemCatalog::BINARY, + WriteEngine::WR_BINARY, + fid, + 1); //compression type + + curCol.dataFile.pFile = fileOp.openFile( + curCol, + dbRoot, + 0, + 0, + segFile, + false, + "r+b", + BYTE_PER_BLOCK * BYTE_PER_BLOCK); // buffer size is 64MB + + CPPUNIT_ASSERT_EQUAL(NO_ERROR, rc); + + int blocks = fileOp.blocksInFile(curCol.dataFile.pFile); + + for (int b = 0; b < blocks; b++) { + rc = fileOp.chunkManager()->readBlock(curCol.dataFile.pFile, block.data, b); // ColumnOpCompress1.readBlock() is protected so ... + CPPUNIT_ASSERT_EQUAL(NO_ERROR, rc); + //cout << endl << bin16[0].uint64(0); + CPPUNIT_ASSERT_EQUAL(b * 512UL + 3, bin16[0].uint64(0)); // Checking just first value of each block as it was written before + CPPUNIT_ASSERT_EQUAL(b * 512UL + 5, bin16[0].uint64(1)); + } + + fileOp.clearColumn(curCol); + fileOp.closeFile(curCol.dataFile.pFile); // Seems done by clearColumn, but anyways... + + cout << endl << "Delete file "; + + fileOp.deleteFile(fileName); + CPPUNIT_ASSERT(fileOp.exists(fileName) == false); + cout << endl << "End of test"; + } + void testCleanup() { // shutdown diff --git a/writeengine/shared/we_blockop.cpp b/writeengine/shared/we_blockop.cpp index fb8dfe89b..da104ad9b 100644 --- a/writeengine/shared/we_blockop.cpp +++ b/writeengine/shared/we_blockop.cpp @@ -159,6 +159,10 @@ uint64_t BlockOp::getEmptyRowValue( case CalpontSystemCatalog::UBIGINT : emptyVal = joblist::UBIGINTEMPTYROW; break; + + case CalpontSystemCatalog::BINARY : + emptyVal = joblist::BINARYEMPTYROW; + break; case CalpontSystemCatalog::CHAR : case CalpontSystemCatalog::VARCHAR : @@ -267,9 +271,11 @@ void BlockOp::setEmptyBuf( // Optimize buffer initialization by constructing and copying in an array // instead of individual values. This reduces the number of calls to // memcpy(). - for (int j = 0; j < ARRAY_COUNT; j++) + + int w = width > 8 ? 8: width; + for(uint8_t* pos = emptyValArray, * end = pos + NBYTES_IN_ARRAY; pos < end; pos += w) //FIXME for no loop { - memcpy(emptyValArray + (j * width), &emptyVal, width); + memcpy(pos, &emptyVal, w); } int countFull128 = (bufSize / width) / ARRAY_COUNT; diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index c550c1050..2af0df827 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -434,6 +434,11 @@ void Convertor::convertColType(CalpontSystemCatalog::ColDataType dataType, case CalpontSystemCatalog::UBIGINT: internalType = WriteEngine::WR_ULONGLONG; break; + + // Map BINARY to WR_BINARY + case CalpontSystemCatalog::BINARY: + internalType = WriteEngine::WR_BINARY; + break; default: internalType = WriteEngine::WR_CHAR; @@ -682,6 +687,11 @@ void Convertor::convertColType(ColStruct* curStruct) case CalpontSystemCatalog::UBIGINT: *internalType = WriteEngine::WR_ULONGLONG; break; + + // Map BINARY to WR_BINARY + case CalpontSystemCatalog::BINARY: + *internalType = WriteEngine::WR_BINARY; + break; default: *internalType = WriteEngine::WR_CHAR; @@ -772,7 +782,11 @@ int Convertor::getCorrectRowWidth(CalpontSystemCatalog::ColDataType dataType, in case CalpontSystemCatalog::TIMESTAMP: newWidth = 8; break; - + + case CalpontSystemCatalog::BINARY: + newWidth = width; + break; + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: case CalpontSystemCatalog::VARBINARY: // treat same as varchar for now diff --git a/writeengine/shared/we_type.h b/writeengine/shared/we_type.h index 71581b44b..6732b48b1 100644 --- a/writeengine/shared/we_type.h +++ b/writeengine/shared/we_type.h @@ -108,9 +108,10 @@ enum ColType /** @brief Column type enumeration*/ WR_USHORT = 14, /** @brief Unsigned Short */ WR_UINT = 15, /** @brief Unsigned Int */ WR_ULONGLONG = 16, /** @brief Unsigned Long long*/ - WR_TEXT = 17, /** @brief TEXT */ + WR_TEXT = 17, /** @brief TEXT */ WR_MEDINT = 18, /** @brief Medium Int */ - WR_UMEDINT = 19 /** @brief Unsigned Medium Int */ + WR_UMEDINT = 19, /** @brief Unsigned Medium Int */ + WR_BINARY = 20 /** @brief BINARY */ }; // Describes relation of field to column for a bulk load diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index 69e2d6f58..6e4e8c91f 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -123,7 +123,9 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, newFile = false; Column newCol; unsigned char buf[BYTE_PER_BLOCK]; - + unsigned char* curVal; + int64_t emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); // Seems is ok have it here and just once + if (useStartingExtent) { // ZZ. For insert select, skip the hwm block and start inserting from the next block @@ -137,10 +139,10 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, if ( rc != NO_ERROR) return rc; - - for (j = 0; j < totalRowPerBlock; j++) + + for (j = 0, curVal = buf; j < totalRowPerBlock; j++, curVal += column.colWidth) { - if (isEmptyRow(buf, j, column)) + if (isEmptyRow((uint64_t*)curVal, emptyVal, column.colWidth)) { rowIdArray[counter] = getRowId(hwm, column.colWidth, j); rowsallocated++; @@ -192,9 +194,9 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, } } - for (j = 0; j < totalRowPerBlock; j++) + for (j = 0, curVal = buf; j < totalRowPerBlock; j++, curVal += column.colWidth) { - if (isEmptyRow(buf, j, column)) + if (isEmptyRow((uint64_t*)curVal, emptyVal, column.colWidth)) { rowIdArray[counter] = getRowId(hwm, column.colWidth, j); rowsallocated++; @@ -492,9 +494,9 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, } } - for (j = 0; j < totalRowPerBlock; j++) + for (j = 0, curVal = buf; j < totalRowPerBlock; j++, curVal += column.colWidth) { - if (isEmptyRow(buf, j, column)) + if (isEmptyRow((uint64_t*)curVal, emptyVal, column.colWidth)) // Why to check it if beacause line 483 is always true ? { rowIdArray[counter] = getRowId(newHwm, column.colWidth, j); rowsallocated++; @@ -533,9 +535,9 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, } } - for (j = 0; j < totalRowPerBlock; j++) + for (j = 0, curVal = buf; j < totalRowPerBlock; j++, curVal += column.colWidth) { - if (isEmptyRow(buf, j, column)) + if (isEmptyRow((uint64_t*)curVal, emptyVal, column.colWidth)) { rowIdArray[counter] = getRowId(newHwm, newCol.colWidth, j); rowsallocated++; @@ -1064,7 +1066,7 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi startColFbo++; colBufOffset = 0; } - + while (((refBufOffset + refCol.colWidth) <= BYTE_PER_BLOCK) && ((colBufOffset + column.colWidth) <= BYTE_PER_BLOCK)) { @@ -1080,7 +1082,8 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi } else if (column.compressionType != 0) //@Bug 3866, fill the empty row value for compressed chunk { - memcpy(colBuf + colBufOffset, &emptyVal, column.colWidth); + for(int b = 0, w = column.colWidth; b < column.colWidth; b += 8, w = 8) //FIXME for no loop! + memcpy(colBuf + colBufOffset + b, &emptyVal, w); dirty = true; } @@ -1405,18 +1408,39 @@ void ColumnOp::initColumn(Column& column) const * RETURN: * true if success, false otherwise ***********************************************************/ -bool ColumnOp::isEmptyRow(unsigned char* buf, int offset, const Column& column) + +// It is called at just 4 places on allocRowId() but all the time inside extend scanning loops +inline bool ColumnOp::isEmptyRow(uint64_t* curVal, uint64_t emptyVal, const int colWidth) { - bool emptyFlag = true; - uint64_t curVal, emptyVal; + //Calling it here makes calling it "i" times from the calling loop at allocRowId() + //uint64_t emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); + + // No need for it if change param type.. just been lazy to add extra castings + //uint64_t &emptyVal = column.emptyVal; + + //no need to multiply over and over if just increment the pointer on the caller + //uint64_t *curVal = (uint64_t*)(buf + offset * column.colWidth); - memcpy(&curVal, buf + offset * column.colWidth, column.colWidth); - emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); + switch(colWidth){ + case 1: + return *(uint8_t*)curVal == emptyVal; - if (/*curVal != emptyVal*/memcmp(&curVal, &emptyVal, column.colWidth)) - emptyFlag = false; - - return emptyFlag; + case 2: + return *(uint16_t*)curVal == emptyVal; + + case 4: + return *(uint32_t*)curVal == emptyVal; + + case 8: + return *curVal == emptyVal; + + case 16: + return ((curVal[0] == emptyVal) && (curVal[1] == emptyVal)); + + case 32: + return ((curVal[0] == emptyVal) && (curVal[1] == emptyVal) + && (curVal[2] == emptyVal) && (curVal[3] == emptyVal)); + } } /*********************************************************** @@ -1534,7 +1558,7 @@ void ColumnOp::setColParam(Column& column, column.colWidth = colWidth; column.colType = colType; column.colDataType = colDataType; - + column.dataFile.fid = dataFid; column.dataFile.fDbRoot = dbRoot; column.dataFile.fPartition = partition; @@ -1662,6 +1686,12 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, if (!bDelete) pVal = &((uint64_t*) valArray)[i]; break; + case WriteEngine::WR_BINARY: + if (!bDelete) pVal = (uint8_t*) valArray + i * curCol.colWidth; + + //pOldVal = (uint8_t*) oldValArray + i * curCol.colWidth; + break; + default : if (!bDelete) pVal = &((int*) valArray)[i]; break; @@ -1669,7 +1699,7 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, if (bDelete) { - emptyVal = getEmptyRowValue(curCol.colDataType, curCol.colWidth); + emptyVal = getEmptyRowValue(curCol.colDataType, curCol.colWidth); pVal = &emptyVal; } @@ -1852,7 +1882,8 @@ int ColumnOp::writeRows(Column& curCol, uint64_t totalRow, const RIDList& ridLis } // This is the write stuff - writeBufValue(dataBuf + dataBio, pVal, curCol.colWidth); + for(int b = 0, w = curCol.colWidth > 8 ? 8 : curCol.colWidth; b < curCol.colWidth; b += 8) //FIXME for no loop + writeBufValue(dataBuf + dataBio + b, pVal, w); i++; diff --git a/writeengine/wrapper/we_colop.h b/writeengine/wrapper/we_colop.h index d1947d752..ad485a8e7 100644 --- a/writeengine/wrapper/we_colop.h +++ b/writeengine/wrapper/we_colop.h @@ -220,7 +220,7 @@ public: /** * @brief Check whether it is an empty row */ - EXPORT virtual bool isEmptyRow(unsigned char* buf, int offset, const Column& column); + EXPORT virtual bool isEmptyRow(uint64_t* curVal, uint64_t emptyVal, const int colWidth); /** * @brief Check whether it is a valid column diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 62842da0d..93a5c1200 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -388,6 +388,15 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* value, boost: memcpy(value, &val, size); } break; + + case WriteEngine::WR_BINARY: + { + char val = boost::any_cast(data); + //TODO:FIXME how to determine size ? 16, 32,48 ? + size = 16; + memcpy(value, &val, size); + } + break; } // end of switch (colType) } /*@convertValue - The base for converting values */ @@ -492,6 +501,12 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, con case WriteEngine::WR_TOKEN: ((Token*)valArray)[pos] = boost::any_cast(data); break; + + case WriteEngine::WR_BINARY: + curStr = boost::any_cast(data); + memcpy((char*)valArray + pos * curStr.length(), curStr.c_str(), curStr.length()); + break; + } // end of switch (colType) } else @@ -557,6 +572,16 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, con case WriteEngine::WR_TOKEN: data = ((Token*)valArray)[pos]; break; + + case WriteEngine::WR_BINARY : + { + char tmp[16]; + //TODO:FIXME how to determine size ? 16, 32,48 ? + memcpy(tmp, (char*)valArray + pos * 16, 16); + curStr = tmp; + data = curStr; + } + break; } // end of switch (colType) } // end of if } @@ -3234,7 +3259,7 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, } bool newFile; - + cout << "Datafile " << curCol.dataFile.fSegFileName << endl; #ifdef PROFILE timer.start("allocRowId"); #endif @@ -5130,6 +5155,10 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, case WriteEngine::WR_TOKEN: valArray = (Token*) calloc(sizeof(Token), totalRow1); break; + + case WriteEngine::WR_BINARY: + valArray = calloc(colStructList[i].colWidth, totalRow1); + break; } // convert values to valArray @@ -5349,6 +5378,11 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, tmp16 = curValue; ((uint16_t*)valArray)[j] = tmp16; break; + + case WriteEngine::WR_BINARY: + ((uint64_t*)valArray)[j] = curValue; //FIXME maybe + break; + } } @@ -5492,6 +5526,10 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, tmp16 = curValue; ((uint16_t*)valArray)[j] = tmp16; break; + + case WriteEngine::WR_BINARY: + ((uint64_t*)valArray)[j] = curValue; // FIXME maybe + break; } } @@ -5775,6 +5813,9 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, case WriteEngine::WR_TOKEN: valArray = (Token*) calloc(sizeof(Token), 1); break; + case WriteEngine::WR_BINARY: + valArray = (char*) calloc(sizeof(char), curColStruct.colWidth); //FIXME maybe + break; } // convert values to valArray From c9f42fb5cc204b3d1a496d0149d580847c2edf21 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 29 Oct 2019 01:21:17 -0500 Subject: [PATCH 02/78] MCOL-641 PoC version for DECIMAL(38) using BINARY as a basis. --- dbcon/ddlpackage/ddlpkg.cpp | 12 +++++++----- dbcon/joblist/passthrucommand-jl.cpp | 2 +- dbcon/joblist/pcolscan.cpp | 10 ++++------ dbcon/joblist/pcolstep.cpp | 5 ++++- dbcon/mysql/ha_mcs_impl.cpp | 9 ++++++++- primitives/linux-port/column.cpp | 6 +++++- utils/dataconvert/dataconvert.cpp | 10 ++++++++-- utils/rowgroup/rowgroup.h | 1 + writeengine/server/we_ddlcommandproc.cpp | 10 ++++++---- writeengine/server/we_dmlcommandproc.cpp | 4 ++-- writeengine/server/we_dmlcommandproc.h | 6 +++--- writeengine/shared/we_convertor.cpp | 6 +++++- writeengine/shared/we_type.h | 3 ++- writeengine/wrapper/we_colop.cpp | 2 +- writeengine/wrapper/writeengine.cpp | 2 ++ writeengine/wrapper/writeengine.h | 22 +++++++++++----------- 16 files changed, 70 insertions(+), 40 deletions(-) diff --git a/dbcon/ddlpackage/ddlpkg.cpp b/dbcon/ddlpackage/ddlpkg.cpp index 24404c52b..d048fff89 100644 --- a/dbcon/ddlpackage/ddlpkg.cpp +++ b/dbcon/ddlpackage/ddlpkg.cpp @@ -199,14 +199,12 @@ void ColumnDef::convertDecimal() } else if ((fType->fPrecision > 0) && (fType->fPrecision < 3)) { - //dataType = CalpontSystemCatalog::TINYINT; fType->fType = DDL_TINYINT; fType->fLength = 1; } else if (fType->fPrecision < 5 && (fType->fPrecision > 2)) { - //dataType = CalpontSystemCatalog::SMALLINT; fType->fType = DDL_SMALLINT; fType->fLength = 2; } @@ -217,15 +215,19 @@ void ColumnDef::convertDecimal() } else if (fType->fPrecision > 6 && fType->fPrecision < 10) { - //dataType = CalpontSystemCatalog::INT; fType->fType = DDL_INT; fType->fLength = 4; } else if (fType->fPrecision > 9 && fType->fPrecision < 19) { - //dataType = CalpontSystemCatalog::BIGINT; fType->fType = DDL_BIGINT; fType->fLength = 8; } + else if (fType->fPrecision > 19 && fType->fPrecision <39) + { + fType->fType = DDL_BINARY; + fType->fLength = 16; + + } } -} +} // end of namespace diff --git a/dbcon/joblist/passthrucommand-jl.cpp b/dbcon/joblist/passthrucommand-jl.cpp index 222f9e5bd..9fcf79131 100644 --- a/dbcon/joblist/passthrucommand-jl.cpp +++ b/dbcon/joblist/passthrucommand-jl.cpp @@ -76,7 +76,7 @@ PassThruCommandJL::PassThruCommandJL(const PassThruStep& p) case 16: case 32: - tableColumnType = TableColumn::STRING; + tableColumnType = TableColumn::STRING; break; default: diff --git a/dbcon/joblist/pcolscan.cpp b/dbcon/joblist/pcolscan.cpp index 3a9384353..d22fbf8e8 100644 --- a/dbcon/joblist/pcolscan.cpp +++ b/dbcon/joblist/pcolscan.cpp @@ -148,11 +148,6 @@ pColScanStep::pColScanStep( int err, i, mask; BRM::LBIDRange_v::iterator it; - //pthread_mutex_init(&mutex, NULL); - //pthread_mutex_init(&dlMutex, NULL); - //pthread_mutex_init(&cpMutex, NULL); - //pthread_cond_init(&condvar, NULL); - //pthread_cond_init(&condvarWakeupProducer, NULL); finishedSending = false; recvWaiting = 0; recvExited = 0; @@ -178,7 +173,10 @@ pColScanStep::pColScanStep( fColType.colWidth = 8; fIsDict = true; } - else if (fColType.colWidth > 8 && fColType.colDataType != CalpontSystemCatalog::BINARY) + // MCOL-641 WIP + else if (fColType.colWidth > 8 + && fColType.colDataType != CalpontSystemCatalog::BINARY + && fColType.colDataType != CalpontSystemCatalog::DECIMAL) { fColType.colWidth = 8; fIsDict = true; diff --git a/dbcon/joblist/pcolstep.cpp b/dbcon/joblist/pcolstep.cpp index 5b07d9374..c4f09ce4b 100644 --- a/dbcon/joblist/pcolstep.cpp +++ b/dbcon/joblist/pcolstep.cpp @@ -177,7 +177,10 @@ pColStep::pColStep( fColType.colWidth = 8; fIsDict = true; } - else if (fColType.colWidth > 8 && fColType.colDataType != CalpontSystemCatalog::BINARY ) + // WIP MCOL-641 + else if (fColType.colWidth > 8 + && fColType.colDataType != CalpontSystemCatalog::BINARY + && fColType.colDataType != CalpontSystemCatalog::DECIMAL) { fColType.colWidth = 8; fIsDict = true; diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index b3018be57..81df0c31e 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -801,7 +801,14 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - intColVal = row.getIntField(s); + if (row.getPrecision(s) > 18) + { + sscanf(row.getBinaryField(s).c_str(), "%ld",&intColVal); + } + else + { + intColVal = row.getIntField(s); + } storeNumericField(f, intColVal, colType); break; } diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 243858157..130707e09 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -285,7 +285,11 @@ template<> inline bool isEmptyVal<16>(uint8_t type, const uint8_t* ival) // For BINARY { const uint64_t* val = reinterpret_cast(ival); - return ((val[0] == joblist::BINARYEMPTYROW) && (val[1] == joblist::BINARYEMPTYROW)); + // WIP ugly speed hack + return ((val[0] == joblist::BINARYEMPTYROW) && (val[1] == joblist::BINARYEMPTYROW) + || (val[0] == joblist::BIGINTEMPTYROW) && (val[1] == joblist::BIGINTEMPTYROW)) +; + } template<> diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index f89c73d16..af0446616 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1224,8 +1224,12 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, value = (long long) number_int_value(data, colType, pushWarning, noRoundup); break; + // MCOL-641 WIP + // use string2BCD conversion here + set sign case CalpontSystemCatalog::DECIMAL: - if (colType.colWidth == 1) + if (colType.colWidth == 16) + value = data; + else if (colType.colWidth == 1) value = (char) number_int_value(data, colType, pushWarning, noRoundup); else if (colType.colWidth == 2) value = (short) number_int_value(data, colType, pushWarning, noRoundup); @@ -1233,9 +1237,11 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, value = (int) number_int_value(data, colType, pushWarning, noRoundup); else if (colType.colWidth == 8) value = (long long) number_int_value(data, colType, pushWarning, noRoundup); + else if (colType.colWidth == 32) + value = data; break; - + // MCOL-641 Implement UDECIMAL case CalpontSystemCatalog::UDECIMAL: // UDECIMAL numbers may not be negative diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index c1789dc18..8d89521bd 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -746,6 +746,7 @@ inline int64_t Row::getIntField(uint32_t colIndex) const return *((int64_t*) &data[offsets[colIndex]]); default: + std::cout << "Row::getIntField getColumnWidth(colIndex) " << getColumnWidth(colIndex) << std::endl; idbassert(0); throw std::logic_error("Row::getIntField(): bad length."); } diff --git a/writeengine/server/we_ddlcommandproc.cpp b/writeengine/server/we_ddlcommandproc.cpp index fd5fe2a5d..30b50ff4c 100644 --- a/writeengine/server/we_ddlcommandproc.cpp +++ b/writeengine/server/we_ddlcommandproc.cpp @@ -458,9 +458,11 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err { if (colDefPtr->fType->fPrecision > 18) //@Bug 5717 precision cannot be over 18. { - ostringstream os; - os << "Syntax error: The maximum precision (total number of digits) that can be specified is 18"; - throw std::runtime_error(os.str()); + // WIP MCOL-641 + //ostringstream os; + //os << "Syntax error: The maximum precision (total number of digits) that can be specified is 18"; + //throw std::runtime_error(os.str()); + colDefPtr->convertDecimal(); } else if (colDefPtr->fType->fPrecision < colDefPtr->fType->fScale) { @@ -507,7 +509,7 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err } else if (dataType == CalpontSystemCatalog::BINARY - && ! (colDefPtr->fType->fLength == 16 + && ! (colDefPtr->fType->fLength == 16 || colDefPtr->fType->fLength == 32)) { ostringstream os; diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 2592e58c9..15c898bc3 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -412,8 +412,8 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: // call the write engine to write the rows int error = NO_ERROR; - //fWriteEngine.setDebugLevel(WriteEngine::DEBUG_3); - //cout << "inserting a row with transaction id " << txnid.id << endl; + fWEWrapper.setDebugLevel(WriteEngine::DEBUG_3); + cout << "inserting a row with transaction id " << txnid.id << endl; fWEWrapper.setIsInsert(true); fWEWrapper.setBulkFlag(true); fWEWrapper.setTransId(txnid.id); diff --git a/writeengine/server/we_dmlcommandproc.h b/writeengine/server/we_dmlcommandproc.h index 8eb7d2fe2..3786052b1 100644 --- a/writeengine/server/we_dmlcommandproc.h +++ b/writeengine/server/we_dmlcommandproc.h @@ -106,12 +106,12 @@ private: WriteEngineWrapper fWEWrapper; boost::scoped_ptr fRBMetaWriter; std::vector > dbRootExtTrackerVec; - inline bool isDictCol ( execplan::CalpontSystemCatalog::ColType colType ) + inline bool isDictCol ( execplan::CalpontSystemCatalog::ColType &colType ) { if (((colType.colDataType == execplan::CalpontSystemCatalog::CHAR) && (colType.colWidth > 8)) || ((colType.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7)) - || ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 18)) - || ((colType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && (colType.precision > 18)) + || ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 65)) + || ((colType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && (colType.precision > 65)) || (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY) || (colType.colDataType == execplan::CalpontSystemCatalog::BLOB) || (colType.colDataType == execplan::CalpontSystemCatalog::TEXT)) diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index 2af0df827..dbe51b1d7 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -633,9 +633,13 @@ void Convertor::convertColType(ColStruct* curStruct) *internalType = WriteEngine::WR_INT; break; - default: + case 8: *internalType = WriteEngine::WR_LONGLONG; break; + + default: + *internalType = WriteEngine::WR_BCDECIMAL; + break; } break; diff --git a/writeengine/shared/we_type.h b/writeengine/shared/we_type.h index 6732b48b1..64cb1d6c3 100644 --- a/writeengine/shared/we_type.h +++ b/writeengine/shared/we_type.h @@ -111,7 +111,8 @@ enum ColType /** @brief Column type enumeration*/ WR_TEXT = 17, /** @brief TEXT */ WR_MEDINT = 18, /** @brief Medium Int */ WR_UMEDINT = 19, /** @brief Unsigned Medium Int */ - WR_BINARY = 20 /** @brief BINARY */ + WR_BINARY = 20, /** @brief BINARY */ + WR_BCDECIMAL = 21 /** @brief BINARY CODED DECIMAL */ }; // Describes relation of field to column for a bulk load diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index 6e4e8c91f..8d7bc3d4d 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -1687,9 +1687,9 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, break; case WriteEngine::WR_BINARY: + case WriteEngine::WR_BCDECIMAL: if (!bDelete) pVal = (uint8_t*) valArray + i * curCol.colWidth; - //pOldVal = (uint8_t*) oldValArray + i * curCol.colWidth; break; default : diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 93a5c1200..620428303 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -503,6 +503,7 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, con break; case WriteEngine::WR_BINARY: + case WriteEngine::WR_BCDECIMAL: curStr = boost::any_cast(data); memcpy((char*)valArray + pos * curStr.length(), curStr.c_str(), curStr.length()); break; @@ -5157,6 +5158,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, break; case WriteEngine::WR_BINARY: + case WriteEngine::WR_BCDECIMAL: valArray = calloc(colStructList[i].colWidth, totalRow1); break; } diff --git a/writeengine/wrapper/writeengine.h b/writeengine/wrapper/writeengine.h index 864d064b1..229d8c76e 100644 --- a/writeengine/wrapper/writeengine.h +++ b/writeengine/wrapper/writeengine.h @@ -621,7 +621,17 @@ public: */ // todo: add implementation when we work on version control // int endTran(const TransID transOid) { return NO_ERROR; } + // WIP + void setDebugLevel(const DebugLevel level) + { + WEObj::setDebugLevel(level); + for (int i = 0; i < TOTAL_COMPRESS_OP; i++) + { + m_colOp[i]->setDebugLevel(level); + m_dctnry[i]->setDebugLevel(level); + } + } // todo: cleanup /************************************************************************ * Internal use definitions @@ -676,17 +686,7 @@ private: std::vector& freeList, std::vector >& fboLists, std::vector >& rangeLists, std::vector& rangeListTot); - void setDebugLevel(const DebugLevel level) - { - WEObj::setDebugLevel(level); - - for (int i = 0; i < TOTAL_COMPRESS_OP; i++) - { - m_colOp[i]->setDebugLevel(level); - m_dctnry[i]->setDebugLevel(level); - } - } // todo: cleanup - + /** * @brief Common methods to write values to a column */ From df65543dd449fdaa9ca11010250ffff8742ec51d Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Mon, 30 Dec 2019 06:09:53 -0600 Subject: [PATCH 03/78] MCOL-641 This commit contains fixes for the rebase that mostly adds WE_BINARY and WE_INT128 into switch-case blocks. --- primitives/linux-port/column.cpp | 5 ++-- writeengine/shared/we_convertor.cpp | 2 +- writeengine/shared/we_type.h | 5 ++-- writeengine/wrapper/we_colop.cpp | 16 ++++-------- writeengine/wrapper/writeengine.cpp | 40 +++++++++++++++++++++++++++-- 5 files changed, 49 insertions(+), 19 deletions(-) diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 130707e09..b76b79655 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -286,8 +286,8 @@ inline bool isEmptyVal<16>(uint8_t type, const uint8_t* ival) // For BINARY { const uint64_t* val = reinterpret_cast(ival); // WIP ugly speed hack - return ((val[0] == joblist::BINARYEMPTYROW) && (val[1] == joblist::BINARYEMPTYROW) - || (val[0] == joblist::BIGINTEMPTYROW) && (val[1] == joblist::BIGINTEMPTYROW)) + return (((val[0] == joblist::BINARYEMPTYROW) && (val[1] == joblist::BINARYEMPTYROW)) + || ((val[0] == joblist::BIGINTEMPTYROW) && (val[1] == joblist::BIGINTEMPTYROW))) ; } @@ -1611,7 +1611,6 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, scoped_array std_regex; idb_regex_t* regex = NULL; - uint8_t likeOps = 0; // no pre-parsed column filter is set, parse the filter in the message if (parsedColumnFilter.get() == NULL) { diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index dbe51b1d7..a98e4b275 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -638,7 +638,7 @@ void Convertor::convertColType(ColStruct* curStruct) break; default: - *internalType = WriteEngine::WR_BCDECIMAL; + *internalType = WriteEngine::WR_INT128; break; } diff --git a/writeengine/shared/we_type.h b/writeengine/shared/we_type.h index 64cb1d6c3..fcbed9161 100644 --- a/writeengine/shared/we_type.h +++ b/writeengine/shared/we_type.h @@ -111,8 +111,9 @@ enum ColType /** @brief Column type enumeration*/ WR_TEXT = 17, /** @brief TEXT */ WR_MEDINT = 18, /** @brief Medium Int */ WR_UMEDINT = 19, /** @brief Unsigned Medium Int */ - WR_BINARY = 20, /** @brief BINARY */ - WR_BCDECIMAL = 21 /** @brief BINARY CODED DECIMAL */ + WR_BINARY = 20, /** @brief BINARY */ +// WIP We might be good using WR_BINARY + WR_INT128 = 21 /** @brief __int128 */ }; // Describes relation of field to column for a bulk load diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index 8d7bc3d4d..b5d5eb06f 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -1410,17 +1410,9 @@ void ColumnOp::initColumn(Column& column) const ***********************************************************/ // It is called at just 4 places on allocRowId() but all the time inside extend scanning loops +// WIP Template this method inline bool ColumnOp::isEmptyRow(uint64_t* curVal, uint64_t emptyVal, const int colWidth) { - //Calling it here makes calling it "i" times from the calling loop at allocRowId() - //uint64_t emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); - - // No need for it if change param type.. just been lazy to add extra castings - //uint64_t &emptyVal = column.emptyVal; - - //no need to multiply over and over if just increment the pointer on the caller - //uint64_t *curVal = (uint64_t*)(buf + offset * column.colWidth); - switch(colWidth){ case 1: return *(uint8_t*)curVal == emptyVal; @@ -1439,8 +1431,10 @@ inline bool ColumnOp::isEmptyRow(uint64_t* curVal, uint64_t emptyVal, const int case 32: return ((curVal[0] == emptyVal) && (curVal[1] == emptyVal) - && (curVal[2] == emptyVal) && (curVal[3] == emptyVal)); + && (curVal[2] == emptyVal) && (curVal[3] == emptyVal)); } + // WIP + return false; } /*********************************************************** @@ -1687,7 +1681,7 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, break; case WriteEngine::WR_BINARY: - case WriteEngine::WR_BCDECIMAL: + case WriteEngine::WR_INT128: if (!bDelete) pVal = (uint8_t*) valArray + i * curCol.colWidth; break; diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 620428303..8455ce26f 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -390,9 +390,11 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* value, boost: break; case WriteEngine::WR_BINARY: + case WriteEngine::WR_INT128: { char val = boost::any_cast(data); //TODO:FIXME how to determine size ? 16, 32,48 ? + // WIP size = 16; memcpy(value, &val, size); } @@ -503,7 +505,7 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, con break; case WriteEngine::WR_BINARY: - case WriteEngine::WR_BCDECIMAL: + case WriteEngine::WR_INT128: curStr = boost::any_cast(data); memcpy((char*)valArray + pos * curStr.length(), curStr.c_str(), curStr.length()); break; @@ -575,9 +577,11 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, con break; case WriteEngine::WR_BINARY : + case WriteEngine::WR_INT128: { char tmp[16]; //TODO:FIXME how to determine size ? 16, 32,48 ? + // WIP memcpy(tmp, (char*)valArray + pos * 16, 16); curStr = tmp; data = curStr; @@ -897,6 +901,14 @@ int WriteEngineWrapper::deleteBadRows(const TxnID& txnid, ColStructList& colStru case WriteEngine::WR_TOKEN: valArray = (Token*) calloc(sizeof(Token), 1); break; + + case WriteEngine::WR_BINARY: + case WriteEngine::WR_INT128: + // WIP use column width here + // remove all C-casts from above + valArray = calloc(1, 16); + break; + } rc = colOp->writeRows(curCol, ridList.size(), ridList, valArray, 0, true); @@ -4569,6 +4581,14 @@ int WriteEngineWrapper::writeColumnRecords(const TxnID& txnid, case WriteEngine::WR_TOKEN: valArray = (Token*) calloc(sizeof(Token), totalRow); break; + + // WIP + case WriteEngine::WR_BINARY: + case WriteEngine::WR_INT128: + // Use column width and remove all C-casts from above + valArray = calloc(totalRow, 16); + break; + } // convert values to valArray @@ -4808,6 +4828,12 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, case WriteEngine::WR_TOKEN: valArray = (Token*) calloc(sizeof(Token), totalRow1); break; + + // WIP + case WriteEngine::WR_BINARY: + case WriteEngine::WR_INT128: + valArray = calloc(totalRow1, 16); + break; } // convert values to valArray @@ -4983,6 +5009,13 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, case WriteEngine::WR_TOKEN: valArray = (Token*) calloc(sizeof(Token), totalRow2); break; + + case WriteEngine::WR_BINARY: + case WriteEngine::WR_INT128: + // WIP + valArray = calloc(totalRow2, 16); + break; + } // convert values to valArray @@ -5158,7 +5191,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, break; case WriteEngine::WR_BINARY: - case WriteEngine::WR_BCDECIMAL: + case WriteEngine::WR_INT128: valArray = calloc(colStructList[i].colWidth, totalRow1); break; } @@ -5382,6 +5415,7 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, break; case WriteEngine::WR_BINARY: + case WriteEngine::WR_INT128: ((uint64_t*)valArray)[j] = curValue; //FIXME maybe break; @@ -5530,6 +5564,7 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, break; case WriteEngine::WR_BINARY: + case WriteEngine::WR_INT128: ((uint64_t*)valArray)[j] = curValue; // FIXME maybe break; } @@ -5816,6 +5851,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, valArray = (Token*) calloc(sizeof(Token), 1); break; case WriteEngine::WR_BINARY: + case WriteEngine::WR_INT128: valArray = (char*) calloc(sizeof(char), curColStruct.colWidth); //FIXME maybe break; } From 63dcaa387f3c5e9748414179393b5bdc533dc8e5 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 9 Jan 2020 13:22:56 +0000 Subject: [PATCH 04/78] MCOL-641 Simple INSERT with one record works with this commit. --- primitives/linux-port/column.cpp | 2 +- utils/dataconvert/dataconvert.cpp | 71 ++++++++++++++++++++++++++++- writeengine/shared/we_convertor.cpp | 5 +- writeengine/wrapper/we_colop.cpp | 10 +++- writeengine/wrapper/writeengine.cpp | 67 +++++++++++++++++++++++++-- 5 files changed, 144 insertions(+), 11 deletions(-) diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index b76b79655..f6d4e09d9 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -1627,9 +1627,9 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, rfs[argIndex] = args->rf; memcpy(argVals[argIndex],args->val, W); + regex[argIndex].used = false; } - regex[argIndex].used = false; } diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index af0446616..df29d1d36 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1161,11 +1161,73 @@ bool stringToTimestampStruct(const string& data, TimeStamp& timeStamp, const str } +// WIP +#include +using int128_t = __int128; +using uint128_t = unsigned __int128; + +struct uint128_pod +{ + uint64_t lo; + uint64_t hi; +}; + +inline void toString(uint128_t i, char *p) +{ + uint64_t div = 10000000000000000000ULL; + size_t div_log = 19; + uint128_t high = i; + uint128_t low; + low = high % div; + high /= div; + uint128_t mid; + mid = high % div; + high /= div; + + uint128_pod *high_pod = reinterpret_cast(&high); + uint128_pod *mid_pod = reinterpret_cast(&mid); + uint128_pod *low_pod = reinterpret_cast(&low); + int printed_chars = 0; + + // WIP replace snprintf with streams + if (high_pod->lo != 0) { + printed_chars = snprintf(p, div_log+1, "%ld", high_pod->lo); + p += printed_chars; + printed_chars = snprintf(p, div_log+1, "%019lu", mid_pod->lo); + p += printed_chars; + } else if (mid_pod->lo != 0) { + printed_chars = snprintf(p, div_log+1, "%lu", mid_pod->lo); + p += printed_chars; + } + snprintf(p, div_log+1, "%019ld", low_pod->lo); +} + + +// Template this +// result must be calloc-ed +void atoi_(const string &arg, int128_t &res, size_t &size) +{ + // WIP + //char buf[40]; + //int128_t *res_ptr = reinterpret_cast(result); + res = 0; + for (int j = 0; j < arg.size(); j++) + { + // WIP + res = res*10 + arg[j] - '0'; + } + //toString(res, buf); + //std::cerr << "atoi_ " << buf <(&high); + uint128_pod *mid_pod = reinterpret_cast(&mid); + uint128_pod *low_pod = reinterpret_cast(&low); + int printed_chars = 0; + + // WIP replace snprintf with streams + if (high_pod->lo != 0) { + printed_chars = snprintf(p, div_log+1, "%ld", high_pod->lo); + p += printed_chars; + printed_chars = snprintf(p, div_log+1, "%019lu", mid_pod->lo); + p += printed_chars; + } else if (mid_pod->lo != 0) { + printed_chars = snprintf(p, div_log+1, "%lu", mid_pod->lo); + p += printed_chars; + } + snprintf(p, div_log+1, "%019ld", low_pod->lo); +} + + + /*@convertValArray - Convert interface values to internal values */ /*********************************************************** @@ -389,8 +433,14 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* value, boost: } break; - case WriteEngine::WR_BINARY: case WriteEngine::WR_INT128: + { + int128_t val = boost::any_cast(data); + size = 16; + // WIP Why do we use memcpy here? + memcpy(value, &val, size); + } + case WriteEngine::WR_BINARY: { char val = boost::any_cast(data); //TODO:FIXME how to determine size ? 16, 32,48 ? @@ -416,7 +466,6 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* value, boost: void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, const size_t pos, boost::any& data, bool fromList) { string curStr; -// ColTuple curTuple; if (fromList) { @@ -505,11 +554,15 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, con break; case WriteEngine::WR_BINARY: - case WriteEngine::WR_INT128: curStr = boost::any_cast(data); memcpy((char*)valArray + pos * curStr.length(), curStr.c_str(), curStr.length()); break; - + case WriteEngine::WR_INT128: + int128_t val = boost::any_cast(data); + size_t size = 16; + // WIP Why do we use memcpy here? + memcpy((uint8_t*)valArray+pos*size, &val, size); + break; } // end of switch (colType) } else @@ -576,8 +629,12 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, con data = ((Token*)valArray)[pos]; break; - case WriteEngine::WR_BINARY : case WriteEngine::WR_INT128: + { + data = ((int128_t*)valArray)[pos]; + break; + } + case WriteEngine::WR_BINARY : { char tmp[16]; //TODO:FIXME how to determine size ? 16, 32,48 ? From 77e1d6abe36a23c7c86c3cd4a632c0ab35328c58 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Thu, 9 Jan 2020 12:37:21 -0500 Subject: [PATCH 05/78] Basic SELECT support for Decimal38 --- dbcon/ddlpackage/ddlpkg.cpp | 3 +- dbcon/mysql/ha_mcs_impl.cpp | 16 +++++- primitives/linux-port/column.cpp | 2 - utils/dataconvert/dataconvert.cpp | 73 ++++++++++++++++++++++-- utils/dataconvert/dataconvert.h | 2 + utils/rowgroup/rowgroup.h | 7 +++ writeengine/server/we_ddlcommandproc.cpp | 11 ++-- 7 files changed, 98 insertions(+), 16 deletions(-) diff --git a/dbcon/ddlpackage/ddlpkg.cpp b/dbcon/ddlpackage/ddlpkg.cpp index d048fff89..ecf6264dc 100644 --- a/dbcon/ddlpackage/ddlpkg.cpp +++ b/dbcon/ddlpackage/ddlpkg.cpp @@ -223,11 +223,10 @@ void ColumnDef::convertDecimal() fType->fType = DDL_BIGINT; fType->fLength = 8; } - else if (fType->fPrecision > 19 && fType->fPrecision <39) + else if (fType->fPrecision > 18 && fType->fPrecision < 39) { fType->fType = DDL_BINARY; fType->fLength = 16; - } } } // end of namespace diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 81df0c31e..4d26ab5a0 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -247,6 +247,9 @@ void force_close_fep_conn(THD *thd, cal_connection_info* ci, bool check_prev_rc ci->cal_conn_hndl = 0; } +// WIP MCOL-641 +using uint128_t = unsigned __int128; + void storeNumericField(Field** f, int64_t value, CalpontSystemCatalog::ColType& ct) { // unset null bit first @@ -801,15 +804,24 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { + // WIP MCOL-641 if (row.getPrecision(s) > 18) { - sscanf(row.getBinaryField(s).c_str(), "%ld",&intColVal); + // unset null bit first + if ((*f)->null_ptr) + *(*f)->null_ptr &= ~(*f)->null_bit; + + const uint128_t val = *reinterpret_cast(row.getBinaryField2(s)); + char buf[256]; + dataconvert::DataConvert::decimalToString(val, (unsigned)colType.scale, buf, 256, colType.colDataType); + Field_new_decimal* f2 = (Field_new_decimal*)*f; + f2->store(buf, strlen(buf), f2->charset()); } else { intColVal = row.getIntField(s); + storeNumericField(f, intColVal, colType); } - storeNumericField(f, intColVal, colType); break; } diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index f6d4e09d9..5d5885cf4 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -1629,7 +1629,6 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, memcpy(argVals[argIndex],args->val, W); regex[argIndex].used = false; } - } @@ -1639,7 +1638,6 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, while (!done) { - // if((*((uint64_t *) (bval))) != 0) // { // cout << "rid "<< rid << " value "; diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index df29d1d36..a28a5b394 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1172,7 +1172,8 @@ struct uint128_pod uint64_t hi; }; -inline void toString(uint128_t i, char *p) +// WIP MCOL-641 +void DataConvert::toString(unsigned __int128 i, char *p) { uint64_t div = 10000000000000000000ULL; size_t div_log = 19; @@ -1191,7 +1192,7 @@ inline void toString(uint128_t i, char *p) // WIP replace snprintf with streams if (high_pod->lo != 0) { - printed_chars = snprintf(p, div_log+1, "%ld", high_pod->lo); + printed_chars = snprintf(p, div_log+1, "%lu", high_pod->lo); p += printed_chars; printed_chars = snprintf(p, div_log+1, "%019lu", mid_pod->lo); p += printed_chars; @@ -1199,10 +1200,11 @@ inline void toString(uint128_t i, char *p) printed_chars = snprintf(p, div_log+1, "%lu", mid_pod->lo); p += printed_chars; } - snprintf(p, div_log+1, "%019ld", low_pod->lo); + snprintf(p, div_log+1, "%019lu", low_pod->lo); } +// WIP MCOL-641 // Template this // result must be calloc-ed void atoi_(const string &arg, int128_t &res, size_t &size) @@ -1211,7 +1213,7 @@ void atoi_(const string &arg, int128_t &res, size_t &size) //char buf[40]; //int128_t *res_ptr = reinterpret_cast(result); res = 0; - for (int j = 0; j < arg.size(); j++) + for (size_t j = 0; j < arg.size(); j++) { // WIP res = res*10 + arg[j] - '0'; @@ -1222,6 +1224,69 @@ void atoi_(const string &arg, int128_t &res, size_t &size) size = 16; } +// WIP MCOL-641 +void DataConvert::decimalToString(unsigned __int128 int_val, uint8_t scale, char* buf, unsigned int buflen, + execplan::CalpontSystemCatalog::ColDataType colDataType) +{ + toString(int_val, buf); + + // Biggest ColumnStore supports is DECIMAL(38,x), or 38 total digits+dp+sign for column + + if (scale == 0) + return; + + //we want to move the last scale chars right by one spot to insert the dp + //we want to move the trailing null as well, so it's really scale+1 chars + size_t l1 = strlen(buf); + char* ptr = &buf[0]; + + if (int_val < 0) + { + ptr++; + idbassert(l1 >= 2); + l1--; + } + + //need to make sure we have enough leading zeros for this to work... + //at this point scale is always > 0 + size_t l2 = 1; + + if ((unsigned)scale > l1) + { + const char* zeros = "00000000000000000000000000000000000000"; //38 0's + size_t diff = 0; + + if (int_val != 0) + diff = scale - l1; //this will always be > 0 + else + diff = scale; + + memmove((ptr + diff), ptr, l1 + 1); //also move null + memcpy(ptr, zeros, diff); + + if (int_val != 0) + l1 = 0; + else + l1 = 1; + } + else if ((unsigned)scale == l1) + { + l1 = 0; + l2 = 2; + } + else + { + l1 -= scale; + } + + memmove((ptr + l1 + l2), (ptr + l1), scale + 1); //also move null + + if (l2 == 2) + *(ptr + l1++) = '0'; + + *(ptr + l1) = '.'; +} + boost::any DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, const std::string& dataOrig, bool& pushWarning, const std::string& timeZone, bool nulFlag, bool noRoundup, bool isUpdate) diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index c2d27b1b8..18a079fbb 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -1011,6 +1011,8 @@ public: EXPORT static bool isNullData(execplan::ColumnResult* cr, int rownum, execplan::CalpontSystemCatalog::ColType colType); static inline std::string decimalToString(int64_t value, uint8_t scale, execplan::CalpontSystemCatalog::ColDataType colDataType); static inline void decimalToString(int64_t value, uint8_t scale, char* buf, unsigned int buflen, execplan::CalpontSystemCatalog::ColDataType colDataType); + EXPORT static void decimalToString(unsigned __int128 value, uint8_t scale, char* buf, unsigned int buflen, execplan::CalpontSystemCatalog::ColDataType colDataType); + EXPORT static void toString(unsigned __int128 i, char *p); static inline std::string constructRegexp(const std::string& str); static inline void trimWhitespace(int64_t& charData); static inline bool isEscapedChar(char c) diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 8d89521bd..03c087cb4 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -435,6 +435,7 @@ public: inline void setVarBinaryField(const uint8_t* val, uint32_t len, uint32_t colIndex); inline std::string getBinaryField(uint32_t colIndex) const; + inline const uint8_t* getBinaryField2(uint32_t colIndex) const; inline boost::shared_ptr getUserData(uint32_t colIndex) const; inline void setUserData(mcsv1sdk::mcsv1Context& context, @@ -835,6 +836,12 @@ inline std::string Row::getBinaryField(uint32_t colIndex) const return std::string((char*) &data[offsets[colIndex]], getColumnWidth(colIndex)); } +// WIP MCOL-641 +inline const uint8_t* Row::getBinaryField2(uint32_t colIndex) const +{ + return &data[offsets[colIndex]]; +} + inline std::string Row::getVarBinaryStringField(uint32_t colIndex) const { if (inStringTable(colIndex)) diff --git a/writeengine/server/we_ddlcommandproc.cpp b/writeengine/server/we_ddlcommandproc.cpp index 30b50ff4c..988bb340a 100644 --- a/writeengine/server/we_ddlcommandproc.cpp +++ b/writeengine/server/we_ddlcommandproc.cpp @@ -456,13 +456,12 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err if (dataType == CalpontSystemCatalog::DECIMAL || dataType == CalpontSystemCatalog::UDECIMAL) { - if (colDefPtr->fType->fPrecision > 18) //@Bug 5717 precision cannot be over 18. + // WIP MCOL-641 + if (colDefPtr->fType->fPrecision > 38) // precision cannot be over 38. { - // WIP MCOL-641 - //ostringstream os; - //os << "Syntax error: The maximum precision (total number of digits) that can be specified is 18"; - //throw std::runtime_error(os.str()); - colDefPtr->convertDecimal(); + ostringstream os; + os << "Syntax error: The maximum precision (total number of digits) that can be specified is 38"; + throw std::runtime_error(os.str()); } else if (colDefPtr->fType->fPrecision < colDefPtr->fType->fScale) { From 49a5573418f0bc18af124d2d7b1332d37e9dc216 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Fri, 10 Jan 2020 17:27:44 -0500 Subject: [PATCH 06/78] MCOL-641 Basic support for filtering operations for Decimal38. --- dbcon/joblist/jlf_execplantojoblist.cpp | 33 +++++++++++++-- dbcon/joblist/pcolstep.cpp | 12 ++++++ dbcon/joblist/primitivestep.h | 2 + primitives/linux-port/column.cpp | 55 ++++++++++++++++++++----- utils/messageqcpp/bytestream.cpp | 30 ++++++++++++++ utils/messageqcpp/bytestream.h | 15 +++++++ 6 files changed, 134 insertions(+), 13 deletions(-) diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 8c86908af..7027e3213 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -1602,6 +1602,16 @@ bool optimizeIdbPatitionSimpleFilter(SimpleFilter* sf, JobStepVector& jsv, JobIn } +// WIP MCOL-641 put this in dataconvert +void atoi_(const string &arg, unsigned __int128 &res) +{ + res = 0; + for (size_t j = 0; j < arg.size(); j++) + { + res = res*10 + arg[j] - '0'; + } +} + const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) { JobStepVector jsv; @@ -1842,6 +1852,7 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) // @bug 1151 string longer than colwidth of char/varchar. int64_t value = 0; uint8_t rf = 0; + unsigned __int128 val128 = 0; #ifdef FAILED_ATOI_IS_ZERO //if cvn throws (because there's non-digit data in the string, treat that as zero rather than @@ -1887,8 +1898,17 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) } #else - bool isNull = ConstantColumn::NULLDATA == cc->type(); - value = convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone); + // WIP MCOL-641 + if (ct.colDataType == CalpontSystemCatalog::DECIMAL && + ct.colWidth == 16) + { + atoi_(constval, val128); + } + else + { + bool isNull = ConstantColumn::NULLDATA == cc->type(); + value = convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone); + } if (ct.colDataType == CalpontSystemCatalog::FLOAT && !isNull) { @@ -1921,7 +1941,14 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) pcs = new PseudoColStep(sc->oid(), tbl_oid, pc->pseudoType(), ct, jobInfo); if (sc->isColumnStore()) - pcs->addFilter(cop, value, rf); + { + // WIP MCOL-641 + if (ct.colDataType == CalpontSystemCatalog::DECIMAL && + ct.colWidth == 16) + pcs->addFilter(cop, val128, rf); + else + pcs->addFilter(cop, value, rf); + } pcs->alias(alias); pcs->view(view); diff --git a/dbcon/joblist/pcolstep.cpp b/dbcon/joblist/pcolstep.cpp index c4f09ce4b..296452819 100644 --- a/dbcon/joblist/pcolstep.cpp +++ b/dbcon/joblist/pcolstep.cpp @@ -633,6 +633,18 @@ void pColStep::addFilter(int8_t COP, int64_t value, uint8_t roundFlag) fFilterCount++; } +// WIP MCOL-641 +void pColStep::addFilter(int8_t COP, unsigned __int128 value, uint8_t roundFlag) +{ + fFilterString << (uint8_t) COP; + fFilterString << roundFlag; + + // bitwise copies into the filter ByteStream + fFilterString << value; + + fFilterCount++; +} + void pColStep::setRidList(DataList* dl) { ridList = dl; diff --git a/dbcon/joblist/primitivestep.h b/dbcon/joblist/primitivestep.h index 35ecf86de..d22ac2821 100644 --- a/dbcon/joblist/primitivestep.h +++ b/dbcon/joblist/primitivestep.h @@ -195,6 +195,8 @@ public: */ void addFilter(int8_t COP, int64_t value, uint8_t roundFlag = 0); void addFilter(int8_t COP, float value); + // WIP MCOL-641 + void addFilter(int8_t COP, unsigned __int128 value, uint8_t roundFlag = 0); /** @brief Sets the DataList to get RID values from. * diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 5d5885cf4..4f3cea781 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -1535,6 +1535,16 @@ inline void p_Col_ridArray(NewColRequestHeader* in, #endif } + +// WIP MCOL-641 +using uint128_t = unsigned __int128; +using int128_t = __int128; + +struct uint128_pod { + uint64_t lo; + uint64_t hi; +}; + // for BINARY template inline void p_Col_bin_ridArray(NewColRequestHeader* in, @@ -1630,9 +1640,18 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, regex[argIndex].used = false; } } - + // WIP MCOL-641 + // we have a pre-parsed filter, and it's in the form of op and value arrays + else if (parsedColumnFilter->columnFilterMode == TWO_ARRAYS) + { + argVals = (binWtype*) parsedColumnFilter->prestored_argVals.get(); + cops = parsedColumnFilter->prestored_cops.get(); + rfs = parsedColumnFilter->prestored_rfs.get(); + regex = parsedColumnFilter->prestored_regex.get(); + } // else we have a pre-parsed filter, and it's an unordered set for quick == comparisons + bval = (binWtype*)nextBinColValue(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull, &isEmpty, &rid, in->OutputType, reinterpret_cast(block), itemsPerBlk); @@ -1681,29 +1700,39 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, // if((*((uint64_t *) (uval))) != 0) cout << "comparing " << dec << (*((uint64_t *) (uval))) << " to " << (*((uint64_t *) (argVals[argIndex]))) << endl; - int val1 = memcmp(*bval, &argVals[argIndex], W); + // WIP MCOL-641 + uint128_t val, filterVal; + uint128_pod *valPod, *filterValPod; + valPod = reinterpret_cast(&val); + filterValPod = reinterpret_cast(&filterVal); + + valPod->lo = *reinterpret_cast(*bval); + valPod->hi = *(reinterpret_cast(*bval) + 1); + + filterValPod->lo = *reinterpret_cast(argVals[argIndex]); + filterValPod->hi = *(reinterpret_cast(argVals[argIndex]) + 1); switch (cops[argIndex]) { case COMPARE_NIL: cmp = false; break; case COMPARE_LT: - cmp = val1 < 0; + cmp = val < filterVal; break; case COMPARE_EQ: - cmp = val1 == 0; + cmp = val == filterVal; break; case COMPARE_LE: - cmp = val1 <= 0; + cmp = val <= filterVal; break; case COMPARE_GT: - cmp = val1 > 0; + cmp = val > filterVal; break; case COMPARE_NE: - cmp = val1 != 0; + cmp = val != filterVal; break; case COMPARE_GE: - cmp = val1 >= 0; + cmp = val >= filterVal; break; default: logIt(34, cops[argIndex], "colCompare"); @@ -1854,7 +1883,11 @@ boost::shared_ptr parseColumnFilter ret.reset(new ParsedColumnFilter()); ret->columnFilterMode = TWO_ARRAYS; - ret->prestored_argVals.reset(new int64_t[filterCount]); + // WIP MCOL-641 + if (colWidth == 16) + ret->prestored_argVals.reset(new int64_t[filterCount * 2]); + else + ret->prestored_argVals.reset(new int64_t[filterCount]); ret->prestored_cops.reset(new uint8_t[filterCount]); ret->prestored_rfs.reset(new uint8_t[filterCount]); ret->prestored_regex.reset(new idb_regex_t[filterCount]); @@ -1949,8 +1982,10 @@ boost::shared_ptr parseColumnFilter case 8: ret->prestored_argVals[argIndex] = *reinterpret_cast(args->val); break; + + // WIP MCOL-641 case 16: - cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + memcpy(ret->prestored_argVals.get() + (argIndex * 2), args->val, 16); } } diff --git a/utils/messageqcpp/bytestream.cpp b/utils/messageqcpp/bytestream.cpp index 6ecbf625d..4cc4a103a 100644 --- a/utils/messageqcpp/bytestream.cpp +++ b/utils/messageqcpp/bytestream.cpp @@ -235,6 +235,18 @@ ByteStream& ByteStream::operator<<(const uint64_t o) return *this; } +// WIP MCOL-641 +ByteStream& ByteStream::operator<<(const unsigned __int128 o) +{ + if (fBuf == 0 || (fCurInPtr - fBuf + 16U > fMaxLen + ISSOverhead)) + growBuf(fMaxLen + BlockSize); + + *((unsigned __int128*) fCurInPtr) = o; + fCurInPtr += 16; + + return *this; +} + ByteStream& ByteStream::operator<<(const string& s) { int32_t len = s.size(); @@ -319,6 +331,14 @@ ByteStream& ByteStream::operator>>(uint64_t& o) return *this; } +// WIP MCOL-641 +ByteStream& ByteStream::operator>>(unsigned __int128& o) +{ + peek(o); + fCurOutPtr += 16; + return *this; +} + ByteStream& ByteStream::operator>>(string& s) { peek(s); @@ -399,6 +419,16 @@ void ByteStream::peek(uint64_t& o) const o = *((uint64_t*) fCurOutPtr); } +// WIP MCOL-641 +void ByteStream::peek(unsigned __int128& o) const +{ + + if (length() < 16) + throw underflow_error("ByteStream>unsigned __int128: not enough data in stream to fill datatype"); + + o = *((unsigned __int128*) fCurOutPtr); +} + void ByteStream::peek(string& s) const { int32_t len; diff --git a/utils/messageqcpp/bytestream.h b/utils/messageqcpp/bytestream.h index 316d9b0a1..16d547fba 100644 --- a/utils/messageqcpp/bytestream.h +++ b/utils/messageqcpp/bytestream.h @@ -143,6 +143,11 @@ public: * push an uint64_t onto the end of the stream. The byte order is whatever the native byte order is. */ EXPORT ByteStream& operator<<(const uint64_t o); + // WIP MCOL-641 + /** + * push an unsigned __int128 onto the end of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator<<(const unsigned __int128 o); /** * push a float onto the end of the stream. The byte order is * whatever the native byte order is. @@ -207,6 +212,11 @@ public: * extract an uint64_t from the front of the stream. The byte order is whatever the native byte order is. */ EXPORT ByteStream& operator>>(uint64_t& o); + // WIP MCOL-641 + /** + * extract an unsigned __int128 from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator>>(unsigned __int128& o); /** * extract a float from the front of the stream. The byte * order is whatever the native byte order is. @@ -277,6 +287,11 @@ public: * Peek at an uint64_t from the front of the stream. The byte order is whatever the native byte order is. */ EXPORT void peek(uint64_t& o) const; + // WIP MCOL-641 + /** + * Peek at an unsigned __int128 from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT void peek(unsigned __int128& o) const; /** * Peek at a float from the front of the stream. The byte order * is whatever the native byte order is. From 0c67b6ab501ad5c13a89a5dbcc6d8b7ad648598e Mon Sep 17 00:00:00 2001 From: drrtuy Date: Sun, 12 Jan 2020 15:11:31 +0300 Subject: [PATCH 07/78] MCOL-641 atoi128 now correctly processes decimal point and - signs. There are multiple overloaded version of the low level DML write methods to push down CSC column type. WE needs the type to convert values correctly. Replaced WE_INT128 with CSC data type that is more informative. Removed commented and obsolete code. Replaced switch-case blocks with oneliners. --- dbcon/joblist/jlf_execplantojoblist.cpp | 7 +- utils/dataconvert/dataconvert.cpp | 229 +--- writeengine/server/we_dmlcommandproc.cpp | 21 +- writeengine/shared/we_convertor.cpp | 3 +- writeengine/shared/we_type.h | 7 +- writeengine/wrapper/we_colop.cpp | 9 +- writeengine/wrapper/writeengine.cpp | 1522 ++++++++++++++++++++-- writeengine/wrapper/writeengine.h | 39 +- 8 files changed, 1475 insertions(+), 362 deletions(-) diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 7027e3213..0a226c38d 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -1603,12 +1603,13 @@ bool optimizeIdbPatitionSimpleFilter(SimpleFilter* sf, JobStepVector& jsv, JobIn // WIP MCOL-641 put this in dataconvert -void atoi_(const string &arg, unsigned __int128 &res) +void atoi128(const string& arg, unsigned __int128& res) { res = 0; for (size_t j = 0; j < arg.size(); j++) { - res = res*10 + arg[j] - '0'; + if (LIKELY(arg[j]-'0' >= 0)) + res = res*10 + arg[j] - '0'; } } @@ -1902,7 +1903,7 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) if (ct.colDataType == CalpontSystemCatalog::DECIMAL && ct.colWidth == 16) { - atoi_(constval, val128); + atoi128(constval, val128); } else { diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index a28a5b394..e90471b97 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -36,6 +36,7 @@ using namespace boost::algorithm; #include "calpontsystemcatalog.h" #include "calpontselectexecutionplan.h" #include "columnresult.h" +#include "common/branchpred.h" using namespace execplan; #include "joblisttypes.h" @@ -1207,7 +1208,9 @@ void DataConvert::toString(unsigned __int128 i, char *p) // WIP MCOL-641 // Template this // result must be calloc-ed -void atoi_(const string &arg, int128_t &res, size_t &size) +//template +//void atoi_(const string &arg, T &res) +void atoi128(const string& arg, int128_t& res) { // WIP //char buf[40]; @@ -1215,13 +1218,13 @@ void atoi_(const string &arg, int128_t &res, size_t &size) res = 0; for (size_t j = 0; j < arg.size(); j++) { - // WIP - res = res*10 + arg[j] - '0'; + // WIP Optimize this + if (LIKELY(arg[j]-'0' >= 0)) + res = res*10 + arg[j] - '0'; } //toString(res, buf); //std::cerr << "atoi_ " << buf <(data, bigint); + atoi128(data, bigint); value = bigint; } else if (colType.colWidth == 1) @@ -2651,216 +2656,6 @@ std::string DataConvert::timeToString1( long long datetimevalue ) return buf; } -#if 0 -bool DataConvert::isNullData(ColumnResult* cr, int rownum, CalpontSystemCatalog::ColType colType) -{ - switch (colType.colDataType) - { - case CalpontSystemCatalog::TINYINT: - if (cr->GetData(rownum) == joblist::TINYINTNULL) - return true; - - return false; - - case CalpontSystemCatalog::SMALLINT: - if (cr->GetData(rownum) == joblist::SMALLINTNULL) - return true; - - return false; - - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - if (cr->GetData(rownum) == joblist::INTNULL) - return true; - - return false; - - case CalpontSystemCatalog::BIGINT: - if (cr->GetData(rownum) == static_cast(joblist::BIGINTNULL)) - return true; - - return false; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - if (colType.colWidth <= CalpontSystemCatalog::FOUR_BYTE) - { - if (cr->GetData(rownum) == joblist::SMALLINTNULL) - return true; - - return false; - } - else if (colType.colWidth <= 9) - { - if (cr->GetData(rownum) == joblist::INTNULL) - return true; - else return false; - } - else if (colType.colWidth <= 18) - { - if (cr->GetData(rownum) == static_cast(joblist::BIGINTNULL)) - return true; - - return false; - } - else - { - if (cr->GetStringData(rownum) == "\376\377\377\377\377\377\377\377") - return true; - - return false; - } - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - - //if (cr->GetStringData(rownum) == joblist::FLOATNULL) - if (cr->GetStringData(rownum).compare("null") == 0 ) - return true; - - return false; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - - //if (cr->GetStringData(rownum) == joblist::DOUBLENULL) - if (cr->GetStringData(rownum).compare("null") == 0 ) - return true; - - return false; - - case CalpontSystemCatalog::DATE: - if (cr->GetData(rownum) == joblist::DATENULL) - return true; - - return false; - - case CalpontSystemCatalog::DATETIME: - if (cr->GetData(rownum) == static_cast(joblist::DATETIMENULL)) - return true; - - return false; - - case CalpontSystemCatalog::CHAR: - { - std::string charnull; - - if ( cr->GetStringData(rownum) == "") - { - return true; - } - - if (colType.colWidth == 1) - { - if (cr->GetStringData(rownum) == "\376") - return true; - - return false; - } - else if (colType.colWidth == 2) - { - if (cr->GetStringData(rownum) == "\377\376") - return true; - - return false; - } - else if (( colType.colWidth < 5 ) && ( colType.colWidth > 2 )) - { - if (cr->GetStringData(rownum) == "\377\377\377\376") - return true; - - return false; - } - else if (( colType.colWidth < 9 ) && ( colType.colWidth > 4 )) - { - if (cr->GetStringData(rownum) == "\377\377\377\377\377\377\377\376") - return true; - - return false; - } - else - { - if (cr->GetStringData(rownum) == "\376\377\377\377\377\377\377\377") - return true; - - return false; - } - } - - case CalpontSystemCatalog::VARCHAR: - { - std::string charnull; - - if ( cr->GetStringData(rownum) == "") - { - return true; - } - - if (colType.colWidth == 1) - { - if (cr->GetStringData(rownum) == "\377\376") - return true; - - return false; - } - else if ((colType.colWidth < 4) && (colType.colWidth > 1)) - { - if (cr->GetStringData(rownum) == "\377\377\377\376") - return true; - - return false; - } - else if ((colType.colWidth < 8) && (colType.colWidth > 3)) - { - if (cr->GetStringData(rownum) == "\377\377\377\377\377\377\377\376") - return true; - - return false; - } - else - { - WriteEngine::Token nullToken; - - // bytes reversed - if (cr->GetStringData(rownum) == "\376\377\377\377\377\377\377\377") - return true; - - return false; - } - } - - case CalpontSystemCatalog::UTINYINT: - if (cr->GetData(rownum) == joblist::UTINYINTNULL) - return true; - - return false; - - case CalpontSystemCatalog::USMALLINT: - if (cr->GetData(rownum) == joblist::USMALLINTNULL) - return true; - - return false; - - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - if (cr->GetData(rownum) == joblist::UINTNULL) - return true; - - return false; - - case CalpontSystemCatalog::UBIGINT: - if (cr->GetData(rownum) == joblist::UBIGINTNULL) - return true; - - return false; - - default: - throw QueryDataExcept("convertColumnData: unknown column data type.", dataTypeErr); - } -} -#endif int64_t DataConvert::dateToInt(const string& date) { return stringToDate(date); diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 15c898bc3..b5b00f81b 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -1,5 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2016 MariaDB Corporation + Copyright (C) 2016-2019 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -116,6 +116,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: RowList rows = tablePtr->get_RowList(); WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypes; WriteEngine::DctnryStructList dctnryStructList; WriteEngine::DctnryValueList dctnryValueList; WriteEngine::ColValueList colValuesList; @@ -139,12 +140,18 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: { Row* rowPtr = rows.at(0); ColumnList columns = rowPtr->get_ColumnList(); + unsigned int numcols = rowPtr->get_NumberOfColumns(); + cscColTypes.reserve(numcols); + // WIP + // We presume that DictCols number is low + colStructs.reserve(numcols); ColumnList::const_iterator column_iterator = columns.begin(); while (column_iterator != columns.end()) { DMLColumn* columnPtr = *column_iterator; tableColName.column = columnPtr->get_Name(); + // WIP MCOL-641 replace with getColRidsOidsTypes() CalpontSystemCatalog::ROPair roPair = systemCatalogPtr->columnRID(tableColName); CalpontSystemCatalog::OID oid = systemCatalogPtr->lookupOID(tableColName); @@ -152,6 +159,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: CalpontSystemCatalog::ColType colType; colType = systemCatalogPtr->colType(oid); + cscColTypes.push_back(colType); WriteEngine::ColStruct colStruct; colStruct.fColDbRoot = dbroot; WriteEngine::DctnryStruct dctnryStruct; @@ -163,6 +171,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: // Token if ( isDictCol(colType) ) { + // WIP Hardcoded value colStruct.colWidth = 8; colStruct.tokenFlag = true; } @@ -194,7 +203,6 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: ++column_iterator; } - unsigned int numcols = rowPtr->get_NumberOfColumns(); std::string tmpStr(""); for (unsigned int i = 0; i < numcols; i++) @@ -210,6 +218,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: const DMLColumn* columnPtr = rowPtr->get_ColumnAt(i); tableColName.column = columnPtr->get_Name(); + // WIP MCOL-641 remove these calls CalpontSystemCatalog::OID oid = systemCatalogPtr->lookupOID(tableColName); CalpontSystemCatalog::ColType colType; @@ -303,6 +312,8 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: { try { + // WIP What if we combine this and previous loop and fail + // after get nextAIValue ? nextVal = systemCatalogPtr->nextAutoIncrValue(tableName); fDbrm.startAISequence(oid, nextVal, colType.colWidth, colType.colDataType); } @@ -359,6 +370,8 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: try { + // WIP + // make convertColumnData a template datavalue = DataConvert::convertColumnData(colType, indata, pushWarning, insertPkg.get_TimeZone(), isNULL, false, false); } catch (exception&) @@ -412,6 +425,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: // call the write engine to write the rows int error = NO_ERROR; + // WIP fWEWrapper.setDebugLevel(WriteEngine::DEBUG_3); cout << "inserting a row with transaction id " << txnid.id << endl; fWEWrapper.setIsInsert(true); @@ -420,6 +434,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: //For hdfs use only uint32_t tblOid = tableRoPair.objnum; + // WIP are we saving HDFS? if (idbdatafile::IDBPolicy::useHdfs()) { @@ -523,7 +538,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: if (colValuesList[0].size() > 0) { if (NO_ERROR != - (error = fWEWrapper.insertColumnRec_Single(txnid.id, colStructs, colValuesList, dctnryStructList, dicStringList, tableRoPair.objnum))) + (error = fWEWrapper.insertColumnRec_Single(txnid.id, cscColTypes, colStructs, colValuesList, dctnryStructList, dicStringList, tableRoPair.objnum))) { if (error == ERR_BRM_DEAD_LOCK) { diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index 536972637..af6723f7d 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -639,7 +639,8 @@ void Convertor::convertColType(ColStruct* curStruct) default: // WIP replace with BINARY - *internalType = WriteEngine::WR_INT128; + //*internalType = WriteEngine::WR_INT128; + *internalType = WriteEngine::WR_BINARY; break; } diff --git a/writeengine/shared/we_type.h b/writeengine/shared/we_type.h index fcbed9161..c2b20a1af 100644 --- a/writeengine/shared/we_type.h +++ b/writeengine/shared/we_type.h @@ -111,9 +111,9 @@ enum ColType /** @brief Column type enumeration*/ WR_TEXT = 17, /** @brief TEXT */ WR_MEDINT = 18, /** @brief Medium Int */ WR_UMEDINT = 19, /** @brief Unsigned Medium Int */ - WR_BINARY = 20, /** @brief BINARY */ -// WIP We might be good using WR_BINARY - WR_INT128 = 21 /** @brief __int128 */ + WR_BINARY = 20 /** @brief BINARY */ + // WIP + //WR_INT128 }; // Describes relation of field to column for a bulk load @@ -302,6 +302,7 @@ struct ColStruct /** @brief Column Interface Struct*/ typedef std::vector ColStructList; /** @brief column struct list */ typedef std::vector ColValueList; /** @brief column value list */ typedef std::vector RIDList; /** @brief RID list */ +typedef std::vector CSCTypesList; /** @brief CSC column types list */ typedef std::vector dictStr; typedef std::vector DictStrList; diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index a2b15b511..8a92b9d61 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -1684,12 +1684,13 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, if (!bDelete) pVal = &((uint64_t*) valArray)[i]; break; - case WriteEngine::WR_INT128: - pVal = &((uint128_t*) valArray)[i]; - break; + // WIP + //case WriteEngine::WR_INT128: case WriteEngine::WR_BINARY: - if (!bDelete) pVal = (uint8_t*) valArray + i * curCol.colWidth; + // WIP CSCCol type + pVal = &((uint128_t*) valArray)[i]; + //pVal = (uint8_t*) valArray + i * curCol.colWidth; break; default : diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 871174d8b..7296654b6 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -1,5 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2016 MariaDB Corporation + Copyright (C) 2016-2019 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -281,19 +281,61 @@ void WriteEngineWrapper::convertValArray(const size_t totalRow, const ColType co ColTupleList::size_type i; if (bFromList) + { for (i = 0; i < curTupleList.size(); i++) { curTuple = curTupleList[i]; convertValue(colType, valArray, i, curTuple.data); - } // end of for (int i = 0 + } + } else + { for (i = 0; i < totalRow; i++) { convertValue(colType, valArray, i, curTuple.data, false); curTupleList.push_back(curTuple); } + } } +/*@convertValArray - Convert interface values to internal values + */ +/*********************************************************** + * DESCRIPTION: + * Convert interface values to internal values + * PARAMETERS: + * cscColType - CSC ColType struct list + * colStructList - column struct list + * colValueList - column value list + * RETURN: + * none + * valArray - output value array + * nullArray - output null flag array + ***********************************************************/ +void WriteEngineWrapper::convertValArray(const size_t totalRow, const CalpontSystemCatalog::ColType &cscColType, const ColType colType, ColTupleList& curTupleList, void* valArray, bool bFromList) +{ + ColTuple curTuple; + ColTupleList::size_type i; + + if (bFromList) + { + for (i = 0; i < curTupleList.size(); i++) + { + curTuple = curTupleList[i]; + convertValue(cscColType, colType, valArray, i, curTuple.data); + } + } + else + { + for (i = 0; i < totalRow; i++) + { + convertValue(cscColType, colType, valArray, i, curTuple.data, false); + curTupleList.push_back(curTuple); + } + } +} + + /* * @brief Convert column value to its internal representation */ @@ -433,13 +475,14 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* value, boost: } break; - case WriteEngine::WR_INT128: + // Replace INT128 with WR_BINARY using CSC::ColType data + /*case WriteEngine::WR_INT128: { int128_t val = boost::any_cast(data); size = 16; // WIP Why do we use memcpy here? memcpy(value, &val, size); - } + }*/ case WriteEngine::WR_BINARY: { char val = boost::any_cast(data); @@ -453,16 +496,166 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* value, boost: } // end of switch (colType) } /*@convertValue - The base for converting values */ -/*********************************************************** - * DESCRIPTION: - * The base for converting values - * PARAMETERS: - * colType - data type - * pos - array position - * data - value - * RETURN: - * none - ***********************************************************/ +// WIP this is ugly to have structs with the same name +void WriteEngineWrapper::convertValue(const execplan::CalpontSystemCatalog::ColType &fullColType, ColType colType, void* value, boost::any& data) +{ + string curStr; + int size; + + switch (colType) + { + case WriteEngine::WR_INT : + case WriteEngine::WR_MEDINT : + if (data.type() == typeid(int)) + { + int val = boost::any_cast(data); + size = sizeof(int); + memcpy(value, &val, size); + } + else + { + uint32_t val = boost::any_cast(data); + size = sizeof(uint32_t); + memcpy(value, &val, size); + } + + break; + + case WriteEngine::WR_UINT : + case WriteEngine::WR_UMEDINT : + { + uint32_t val = boost::any_cast(data); + size = sizeof(uint32_t); + memcpy(value, &val, size); + } + break; + + case WriteEngine::WR_VARBINARY : // treat same as char for now + case WriteEngine::WR_CHAR : + case WriteEngine::WR_BLOB : + case WriteEngine::WR_TEXT : + curStr = boost::any_cast(data); + + if ((int) curStr.length() > MAX_COLUMN_BOUNDARY) + curStr = curStr.substr(0, MAX_COLUMN_BOUNDARY); + + memcpy(value, curStr.c_str(), curStr.length()); + break; + + case WriteEngine::WR_FLOAT: + { + float val = boost::any_cast(data); + +//N.B.There is a bug in boost::any or in gcc where, if you store a nan, you will get back a nan, +// but not necessarily the same bits that you put in. This only seems to be for float (double seems +// to work). + if (isnan(val)) + { + uint32_t ti = joblist::FLOATNULL; + float* tfp = (float*)&ti; + val = *tfp; + } + + size = sizeof(float); + memcpy(value, &val, size); + } + break; + + case WriteEngine::WR_DOUBLE: + { + double val = boost::any_cast(data); + size = sizeof(double); + memcpy(value, &val, size); + } + break; + + case WriteEngine::WR_SHORT: + { + short val = boost::any_cast(data); + size = sizeof(short); + memcpy(value, &val, size); + } + break; + + case WriteEngine::WR_USHORT: + { + uint16_t val = boost::any_cast(data); + size = sizeof(uint16_t); + memcpy(value, &val, size); + } + break; + + case WriteEngine::WR_BYTE: + { + char val = boost::any_cast(data); + size = sizeof(char); + memcpy(value, &val, size); + } + break; + + case WriteEngine::WR_UBYTE: + { + uint8_t val = boost::any_cast(data); + size = sizeof(uint8_t); + memcpy(value, &val, size); + } + break; + + case WriteEngine::WR_LONGLONG: + if (data.type() == typeid(long long)) + { + long long val = boost::any_cast(data); + size = sizeof(long long); + memcpy(value, &val, size); + } + else + { + uint64_t val = boost::any_cast(data); + size = sizeof(uint64_t); + memcpy(value, &val, size); + } + + break; + + case WriteEngine::WR_ULONGLONG: + { + uint64_t val = boost::any_cast(data); + size = sizeof(uint64_t); + memcpy(value, &val, size); + } + break; + + case WriteEngine::WR_TOKEN: + { + Token val = boost::any_cast(data); + size = sizeof(Token); + memcpy(value, &val, size); + } + break; + + // WIP + case WriteEngine::WR_BINARY: + { + size = fullColType.colWidth; + if (fullColType.colDataType == CalpontSystemCatalog::DECIMAL) + { + int128_t val = boost::any_cast(data); + memcpy(value, &val, size); + } + else + { + char val = boost::any_cast(data); + memcpy(value, &val, size); + } + + } + break; + + } // end of switch (colType) +} /*@convertValue - The base for converting values */ + +// WIP +// Legacy version void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, const size_t pos, boost::any& data, bool fromList) { string curStr; @@ -554,14 +747,208 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, con break; case WriteEngine::WR_BINARY: + { + curStr = boost::any_cast(data); + // String length or column width? + memcpy((char*)valArray + pos * curStr.length(), + curStr.c_str(), curStr.length()); + break; + } + } // end of switch (colType) + } + else + { + switch (colType) + { + case WriteEngine::WR_INT : + case WriteEngine::WR_MEDINT : + data = ((int*)valArray)[pos]; + break; + + case WriteEngine::WR_UINT : + case WriteEngine::WR_UMEDINT : + data = ((uint64_t*)valArray)[pos]; + break; + + case WriteEngine::WR_VARBINARY : // treat same as char for now + case WriteEngine::WR_CHAR : + case WriteEngine::WR_BLOB : + case WriteEngine::WR_TEXT : + char tmp[10]; + memcpy(tmp, (char*)valArray + pos * 8, 8); + curStr = tmp; + data = curStr; + break; + +// case WriteEngine::WR_LONG : ((long*)valArray)[pos] = boost::any_cast(curTuple.data); +// break; + case WriteEngine::WR_FLOAT: + data = ((float*)valArray)[pos]; + break; + + case WriteEngine::WR_DOUBLE: + data = ((double*)valArray)[pos]; + break; + + case WriteEngine::WR_SHORT: + data = ((short*)valArray)[pos]; + break; + + case WriteEngine::WR_USHORT: + data = ((uint16_t*)valArray)[pos]; + break; + +// case WriteEngine::WR_BIT: data = ((bool*)valArray)[pos]; +// break; + case WriteEngine::WR_BYTE: + data = ((char*)valArray)[pos]; + break; + + case WriteEngine::WR_UBYTE: + data = ((uint8_t*)valArray)[pos]; + break; + + case WriteEngine::WR_LONGLONG: + data = ((long long*)valArray)[pos]; + break; + + case WriteEngine::WR_ULONGLONG: + data = ((uint64_t*)valArray)[pos]; + break; + + case WriteEngine::WR_TOKEN: + data = ((Token*)valArray)[pos]; + break; + // WIP + case WriteEngine::WR_BINARY : + { + // WIP do we need tmp here? + char *tmp = (char*)alloca (sizeof(char) * 16); + memcpy(tmp, (char*)valArray + pos * 16, 16); + curStr = tmp; + data = curStr; + } + break; + } // end of switch (colType) + } // end of if +} + + +/*********************************************************** + * DESCRIPTION: + * The base for converting values + * PARAMETERS: + * colType - data type + * pos - array position + * data - value + * RETURN: + * none + ***********************************************************/ +void WriteEngineWrapper::convertValue(const CalpontSystemCatalog::ColType &fullColType, const ColType colType, void* valArray, const size_t pos, boost::any& data, bool fromList) +{ + string curStr; + + if (fromList) + { + switch (colType) + { + case WriteEngine::WR_INT : + case WriteEngine::WR_MEDINT : + if (data.type() == typeid(long)) + ((int*)valArray)[pos] = static_cast(boost::any_cast(data)); + else if (data.type() == typeid(int)) + ((int*)valArray)[pos] = boost::any_cast(data); + else + ((int*)valArray)[pos] = boost::any_cast(data); + + break; + + case WriteEngine::WR_UINT : + case WriteEngine::WR_UMEDINT : + ((uint32_t*)valArray)[pos] = boost::any_cast(data); + break; + + case WriteEngine::WR_VARBINARY : // treat same as char for now + case WriteEngine::WR_CHAR : + case WriteEngine::WR_BLOB : + case WriteEngine::WR_TEXT : curStr = boost::any_cast(data); - memcpy((char*)valArray + pos * curStr.length(), curStr.c_str(), curStr.length()); - break; - case WriteEngine::WR_INT128: - int128_t val = boost::any_cast(data); - size_t size = 16; - // WIP Why do we use memcpy here? - memcpy((uint8_t*)valArray+pos*size, &val, size); + + if ((int) curStr.length() > MAX_COLUMN_BOUNDARY) + curStr = curStr.substr(0, MAX_COLUMN_BOUNDARY); + + memcpy((char*)valArray + pos * MAX_COLUMN_BOUNDARY, curStr.c_str(), curStr.length()); + break; + +// case WriteEngine::WR_LONG : ((long*)valArray)[pos] = boost::any_cast(curTuple.data); +// break; + case WriteEngine::WR_FLOAT: + ((float*)valArray)[pos] = boost::any_cast(data); + + if (isnan(((float*)valArray)[pos])) + { + uint32_t ti = joblist::FLOATNULL; + float* tfp = (float*)&ti; + ((float*)valArray)[pos] = *tfp; + } + + break; + + case WriteEngine::WR_DOUBLE: + ((double*)valArray)[pos] = boost::any_cast(data); + break; + + case WriteEngine::WR_SHORT: + ((short*)valArray)[pos] = boost::any_cast(data); + break; + + case WriteEngine::WR_USHORT: + ((uint16_t*)valArray)[pos] = boost::any_cast(data); + break; + +// case WriteEngine::WR_BIT: ((bool*)valArray)[pos] = boost::any_cast(data); +// break; + case WriteEngine::WR_BYTE: + ((char*)valArray)[pos] = boost::any_cast(data); + break; + + case WriteEngine::WR_UBYTE: + ((uint8_t*)valArray)[pos] = boost::any_cast(data); + break; + + case WriteEngine::WR_LONGLONG: + if (data.type() == typeid(long long)) + ((long long*)valArray)[pos] = boost::any_cast(data); + else if (data.type() == typeid(long)) + ((long long*)valArray)[pos] = (long long)boost::any_cast(data); + else + ((long long*)valArray)[pos] = boost::any_cast(data); + + break; + + case WriteEngine::WR_ULONGLONG: + ((uint64_t*)valArray)[pos] = boost::any_cast(data); + break; + + case WriteEngine::WR_TOKEN: + ((Token*)valArray)[pos] = boost::any_cast(data); + break; + + case WriteEngine::WR_BINARY: + if (fullColType.colDataType != CalpontSystemCatalog::DECIMAL) + { + curStr = boost::any_cast(data); + // String length or column width? + memcpy((char*)valArray + pos * curStr.length(), curStr.c_str(), curStr.length()); + } + else + { + int128_t val = boost::any_cast(data); + size_t size = fullColType.colWidth; + // WIP Why do we use memcpy here? + memcpy((uint8_t*)valArray+pos*size, &val, size); + } + break; } // end of switch (colType) } @@ -628,22 +1015,21 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, con case WriteEngine::WR_TOKEN: data = ((Token*)valArray)[pos]; break; - - case WriteEngine::WR_INT128: - { - data = ((int128_t*)valArray)[pos]; - break; - } + // WIP case WriteEngine::WR_BINARY : - { - char tmp[16]; - //TODO:FIXME how to determine size ? 16, 32,48 ? - // WIP - memcpy(tmp, (char*)valArray + pos * 16, 16); - curStr = tmp; - data = curStr; - } - break; + if (fullColType.colDataType == CalpontSystemCatalog::DECIMAL) + { + data = ((int128_t*)valArray)[pos]; + } + else + { + // WIP do we need tmp here? + char *tmp = (char*) alloca (sizeof(char) * fullColType.colWidth); + memcpy(tmp, (char*)valArray + pos * fullColType.colWidth, fullColType.colWidth); + curStr = tmp; + data = curStr; + } + break; } // end of switch (colType) } // end of if } @@ -725,10 +1111,6 @@ int WriteEngineWrapper::fillColumn(const TxnID& txnid, const OID& dataOid, Dctnry* dctnry = m_dctnry[op(compressionType)]; colOpNewCol->initColumn(newCol); refColOp->initColumn(refCol); - //boost::shared_ptr dctnry; - // boost::shared_ptr refColOp; - // refColOp.reset(colOpRefCol); - // dctnry.reset(dctOp); uint16_t dbRoot = 1; //not to be used int newDataWidth = dataWidth; //Convert HWM of the reference column for the new column @@ -746,6 +1128,8 @@ int WriteEngineWrapper::fillColumn(const TxnID& txnid, const OID& dataOid, Convertor::convertColType(dataType, newColType, isToken); + // WIP + // replace with isDictCol if (((refColDataType == CalpontSystemCatalog::VARCHAR) && (refColWidth > 7)) || ((refColDataType == CalpontSystemCatalog::CHAR) && (refColWidth > 8)) || (refColDataType == CalpontSystemCatalog::VARBINARY) || @@ -777,23 +1161,18 @@ int WriteEngineWrapper::fillColumn(const TxnID& txnid, const OID& dataOid, Token nullToken; memcpy(defVal.get(), &nullToken, size); } - //Tokenization is done when we create dictionary file } else + { + // WIP convertValue(newColType, defVal.get(), defaultVal.data); + } if (rc == NO_ERROR) rc = colOpNewCol->fillColumn(txnid, newCol, refCol, defVal.get(), dctnry, refColOp, dictOid, dataWidth, defaultValStr, autoincrement); -// colOpNewCol->clearColumn(newCol); -// colOpRefCol->clearColumn(refCol); - -// free(defVal); - // flushing files is in colOp->fillColumn() -// if (rc == NO_ERROR) -// rc = flushDataFiles(); return rc; } @@ -960,7 +1339,7 @@ int WriteEngineWrapper::deleteBadRows(const TxnID& txnid, ColStructList& colStru break; case WriteEngine::WR_BINARY: - case WriteEngine::WR_INT128: + //case WriteEngine::WR_INT128: // WIP use column width here // remove all C-casts from above valArray = calloc(1, 16); @@ -3209,6 +3588,7 @@ int WriteEngineWrapper::insertColumnRec_SYS(const TxnID& txnid, } int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, + CSCTypesList& cscColTypesList, ColStructList& colStructList, ColValueList& colValueList, DctnryStructList& dctnryStructList, @@ -3329,6 +3709,7 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, } bool newFile; + // WIP cout << "Datafile " << curCol.dataFile.fSegFileName << endl; #ifdef PROFILE timer.start("allocRowId"); @@ -3429,7 +3810,7 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, // Expand initial abbreviated extent if any RID in 1st extent is > 256K. // if totalRow == rowsLeft, then not adding rows to 1st extent, so skip it. //-------------------------------------------------------------------------- -// DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? + // DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? if ((colStructList[colId].fColPartition == 0) && (colStructList[colId].fColSegment == 0) && ((totalRow - rowsLeft) > 0) && @@ -3756,33 +4137,13 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, //-------------------------------------------------------------------------- //Mark extents invalid //-------------------------------------------------------------------------- + // WIP + // Set min/max in dmlprocprocessor if aplicable vector lbids; vector colDataTypes; bool successFlag = true; unsigned width = 0; - //BRM::LBID_t lbid; int curFbo = 0, curBio, lastFbo = -1; - /* if (totalRow-rowsLeft > 0) - { - for (unsigned i = 0; i < colStructList.size(); i++) - { - colOp = m_colOp[op(colStructList[i].fCompressionType)]; - width = colStructList[i].colWidth; - successFlag = colOp->calculateRowId(lastRid , - BYTE_PER_BLOCK/width, width, curFbo, curBio); - if (successFlag) { - if (curFbo != lastFbo) { - RETURN_ON_ERROR(BRMWrapper::getInstance()->getBrmInfo( - colStructList[i].dataOid, - colStructList[i].fColPartition, - colStructList[i].fColSegment, curFbo, lbid)); - lbids.push_back((BRM::LBID_t)lbid); - colDataTypes.push_back(colStructList[i].colDataType); - } - } - } - } - */ lastRid = rowIdArray[totalRow - 1]; for (unsigned i = 0; i < colStructList.size(); i++) @@ -3821,13 +4182,13 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, { if (newExtent) { - rc = writeColumnRec(txnid, colStructList, colOldValueList, + rc = writeColumnRec(txnid, cscColTypesList, colStructList, colOldValueList, rowIdArray, newColStructList, colNewValueList, tableOid, false); // @bug 5572 HDFS tmp file } else { - rc = writeColumnRec(txnid, colStructList, colValueList, + rc = writeColumnRec(txnid, cscColTypesList, colStructList, colValueList, rowIdArray, newColStructList, colNewValueList, tableOid, true); // @bug 5572 HDFS tmp file } @@ -3836,18 +4197,12 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, #ifdef PROFILE timer.stop("writeColumnRec"); #endif -// for (ColTupleList::size_type i = 0; i < totalRow; i++) -// ridList.push_back((RID) rowIdArray[i]); - -// if (rc == NO_ERROR) -// rc = flushDataFiles(NO_ERROR); //-------------------------------------------------------------------------- // Update BRM //-------------------------------------------------------------------------- if ( !newExtent ) { - //flushVMCache(); bool succFlag = false; unsigned colWidth = 0; int extState; @@ -3857,7 +4212,6 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, for (unsigned i = 0; i < totalColumns; i++) { - //colOp = m_colOp[op(colStructList[i].fCompressionType)]; //Set all columns hwm together BulkSetHWMArg aHwmEntry; RETURN_ON_ERROR(BRMWrapper::getInstance()->getLastHWM_DBroot( @@ -3908,7 +4262,6 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, RETURN_ON_ERROR(BRMWrapper::getInstance()->bulkSetHWMAndCP( hwmVecOldext, mergeCPDataArgs)); - //flushVMCache(); #ifdef PROFILE timer.stop("flushVMCache"); #endif @@ -3917,33 +4270,6 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, #ifdef PROFILE timer.finish(); #endif - //flush PrimProc FD cache moved to we_dmlcommandproc.cpp - /* ColsExtsInfoMap colsExtsInfoMap = aTableMetaData->getColsExtsInfoMap(); - ColsExtsInfoMap::iterator it = colsExtsInfoMap.begin(); - ColExtsInfo::iterator aIt; - std::vector files; - BRM::FileInfo aFile; - while (it != colsExtsInfoMap.end()) - { - aIt = (it->second).begin(); - aFile.oid = it->first; - //cout << "OID:" << aArg.oid; - while (aIt != (it->second).end()) - { - aFile.partitionNum = aIt->partNum; - aFile.dbRoot =aIt->dbRoot; - aFile.segmentNum = aIt->segNum; - aFile.compType = aIt->compType; - files.push_back(aFile); - //cout <<"Added to files oid:dbroot:part:seg:compType = " << aFile.oid<<":"< 0)) - cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); - TableMetaData::removeTableMetaData(tableOid); */ return rc; } @@ -4641,7 +4967,7 @@ int WriteEngineWrapper::writeColumnRecords(const TxnID& txnid, // WIP case WriteEngine::WR_BINARY: - case WriteEngine::WR_INT128: + //case WriteEngine::WR_INT128: // Use column width and remove all C-casts from above valArray = calloc(totalRow, 16); break; @@ -4709,6 +5035,7 @@ int WriteEngineWrapper::writeColumnRecords(const TxnID& txnid, * Write values to a column * PARAMETERS: * tableOid - table object id + * cscColTypesList - CSC ColType list * colStructList - column struct list * colValueList - column value list * colNewStructList - the new extent struct list @@ -4719,6 +5046,611 @@ int WriteEngineWrapper::writeColumnRecords(const TxnID& txnid, * NO_ERROR if success * others if something wrong in inserting the value ***********************************************************/ +int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, + const CSCTypesList& cscColTypeList, + const ColStructList& colStructList, + ColValueList& colValueList, + RID* rowIdArray, + const ColStructList& newColStructList, + ColValueList& newColValueList, + const int32_t tableOid, + bool useTmpSuffix, + bool versioning) +{ + bool bExcp; + int rc = 0; + void* valArray; + string segFile; + Column curCol; + ColTupleList oldTupleList; + ColStructList::size_type totalColumn; + ColStructList::size_type i; + ColTupleList::size_type totalRow1, totalRow2; + + setTransId(txnid); + + totalColumn = colStructList.size(); +#ifdef PROFILE + StopWatch timer; +#endif + + if (newColValueList.size() > 0) + { + totalRow1 = colValueList[0].size(); + totalRow2 = newColValueList[0].size(); + } + else + { + totalRow1 = colValueList[0].size(); + totalRow2 = 0; + } + + TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); + + for (i = 0; i < totalColumn; i++) + { + if (totalRow2 > 0) + { + RID* secondPart = rowIdArray + totalRow1; + + //@Bug 2205 Check if all rows go to the new extent + if (totalRow1 > 0) + { + //Write the first batch + valArray = NULL; + RID* firstPart = rowIdArray; + ColumnOp* colOp = m_colOp[op(colStructList[i].fCompressionType)]; + + // set params + colOp->initColumn(curCol); + // need to pass real dbRoot, partition, and segment to setColParam + colOp->setColParam(curCol, 0, colStructList[i].colWidth, + colStructList[i].colDataType, colStructList[i].colType, colStructList[i].dataOid, + colStructList[i].fCompressionType, colStructList[i].fColDbRoot, + colStructList[i].fColPartition, colStructList[i].fColSegment); + + ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(colStructList[i].dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == colStructList[i].fColDbRoot) && (it->partNum == colStructList[i].fColPartition) && (it->segNum == colStructList[i].fColSegment)) + break; + + it++; + } + + if (it == aColExtsInfo.end()) //add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = colStructList[i].fColDbRoot; + aExt.partNum = colStructList[i].fColPartition; + aExt.segNum = colStructList[i].fColSegment; + aExt.compType = colStructList[i].fCompressionType; + aColExtsInfo.push_back(aExt); + aTbaleMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); + } + + rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) + break; + + // handling versioning + vector rangeList; + + if (versioning) + { + rc = processVersionBuffer(curCol.dataFile.pFile, txnid, colStructList[i], + colStructList[i].colWidth, totalRow1, firstPart, rangeList); + + if (rc != NO_ERROR) + { + if (colStructList[i].fCompressionType == 0) + { + curCol.dataFile.pFile->flush(); + } + + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + break; + } + } + + // WIP We can allocate based on column size and not colType + // have to init the size here + valArray = calloc(totalRow1, colStructList[i].colWidth); +#if 0 + switch (colStructList[i].colType) + { + // WIP we don't need type cast here only size + case WriteEngine::WR_INT: + case WriteEngine::WR_MEDINT: + valArray = (int*) calloc(sizeof(int), totalRow1); + break; + + case WriteEngine::WR_UINT: + case WriteEngine::WR_UMEDINT: + valArray = (uint32_t*) calloc(sizeof(uint32_t), totalRow1); + break; + + case WriteEngine::WR_VARBINARY : // treat same as char for now + case WriteEngine::WR_CHAR: + case WriteEngine::WR_BLOB: + case WriteEngine::WR_TEXT: + valArray = (char*) calloc(sizeof(char), totalRow1 * MAX_COLUMN_BOUNDARY); + break; + + case WriteEngine::WR_FLOAT: + valArray = (float*) calloc(sizeof(float), totalRow1); + break; + + case WriteEngine::WR_DOUBLE: + valArray = (double*) calloc(sizeof(double), totalRow1); + break; + + case WriteEngine::WR_BYTE: + valArray = (char*) calloc(sizeof(char), totalRow1); + break; + + case WriteEngine::WR_UBYTE: + valArray = (uint8_t*) calloc(sizeof(uint8_t), totalRow1); + break; + + case WriteEngine::WR_SHORT: + valArray = (short*) calloc(sizeof(short), totalRow1); + break; + + case WriteEngine::WR_USHORT: + valArray = (uint16_t*) calloc(sizeof(uint16_t), totalRow1); + break; + + case WriteEngine::WR_LONGLONG: + valArray = (long long*) calloc(sizeof(long long), totalRow1); + break; + + case WriteEngine::WR_ULONGLONG: + valArray = (uint64_t*) calloc(sizeof(uint64_t), totalRow1); + break; + + case WriteEngine::WR_TOKEN: + valArray = (Token*) calloc(sizeof(Token), totalRow1); + break; + + // WIP + case WriteEngine::WR_BINARY: + valArray = calloc(totalRow1, colStructList[i].colWidth); + break; + } +#endif + + // convert values to valArray + // WIP + // Is this m_opType ever set to DELETE? + if (m_opType != DELETE) + { + bExcp = false; + + try + { + // WIP We convert values twice!? + // dmlcommandproc converts strings to boost::any and this converts + // into actual type value masked by *void + // It is not clear why we need to convert to boost::any b/c we can convert from the original string here + convertValArray(totalRow1, cscColTypeList[i], colStructList[i].colType, colValueList[i], valArray); + } + catch (...) + { + bExcp = true; + } + + if (bExcp) + { + if (versioning) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + + return ERR_PARSING; + } + +#ifdef PROFILE + iimer.start("writeRow "); +#endif + rc = colOp->writeRow(curCol, totalRow1, firstPart, valArray); +#ifdef PROFILE + timer.stop("writeRow "); +#endif + } + else + { +#ifdef PROFILE + timer.start("writeRow "); +#endif + rc = colOp->writeRow(curCol, totalRow1, rowIdArray, valArray, true); +#ifdef PROFILE + timer.stop("writeRow "); +#endif + } + + colOp->clearColumn(curCol); + + if (versioning) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + + if (valArray != NULL) + free(valArray); + + // check error + if (rc != NO_ERROR) + break; + } + + //Process the second batch + valArray = NULL; + + ColumnOp* colOp = m_colOp[op(newColStructList[i].fCompressionType)]; + + // set params + colOp->initColumn(curCol); + colOp->setColParam(curCol, 0, newColStructList[i].colWidth, + newColStructList[i].colDataType, newColStructList[i].colType, newColStructList[i].dataOid, + newColStructList[i].fCompressionType, newColStructList[i].fColDbRoot, + newColStructList[i].fColPartition, newColStructList[i].fColSegment); + + ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(newColStructList[i].dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == newColStructList[i].fColDbRoot) && (it->partNum == newColStructList[i].fColPartition) && (it->segNum == newColStructList[i].fColSegment)) + break; + + it++; + } + + if (it == aColExtsInfo.end()) //add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = newColStructList[i].fColDbRoot; + aExt.partNum = newColStructList[i].fColPartition; + aExt.segNum = newColStructList[i].fColSegment; + aExt.compType = newColStructList[i].fCompressionType; + aColExtsInfo.push_back(aExt); + aTbaleMetaData->setColExtsInfo(newColStructList[i].dataOid, aColExtsInfo); + } + + // Pass "false" for hdfs tmp file flag. Since we only allow 1 + // extent per segment file (with HDFS), we can assume a second + // extent is going to a new file (and won't need tmp file). + rc = colOp->openColumnFile(curCol, segFile, false, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) + break; + + // handling versioning + vector rangeList; + + if (versioning) + { + rc = processVersionBuffer(curCol.dataFile.pFile, txnid, newColStructList[i], + newColStructList[i].colWidth, totalRow2, secondPart, rangeList); + + if (rc != NO_ERROR) + { + if (newColStructList[i].fCompressionType == 0) + { + curCol.dataFile.pFile->flush(); + } + + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + break; + } + } + + // have to init the size here + valArray = calloc(totalRow2, newColStructList[i].colWidth); + /*switch (newColStructList[i].colType) + { + case WriteEngine::WR_INT: + case WriteEngine::WR_MEDINT: + valArray = (int*) calloc(sizeof(int), totalRow2); + break; + + case WriteEngine::WR_UINT: + case WriteEngine::WR_UMEDINT: + valArray = (uint32_t*) calloc(sizeof(uint32_t), totalRow2); + break; + + case WriteEngine::WR_VARBINARY : // treat same as char for now + case WriteEngine::WR_CHAR: + case WriteEngine::WR_BLOB: + case WriteEngine::WR_TEXT: + valArray = (char*) calloc(sizeof(char), totalRow2 * MAX_COLUMN_BOUNDARY); + break; + + case WriteEngine::WR_FLOAT: + valArray = (float*) calloc(sizeof(float), totalRow2); + break; + + case WriteEngine::WR_DOUBLE: + valArray = (double*) calloc(sizeof(double), totalRow2); + break; + + case WriteEngine::WR_BYTE: + valArray = (char*) calloc(sizeof(char), totalRow2); + break; + + case WriteEngine::WR_UBYTE: + valArray = (uint8_t*) calloc(sizeof(uint8_t), totalRow2); + break; + + case WriteEngine::WR_SHORT: + valArray = (short*) calloc(sizeof(short), totalRow2); + break; + + case WriteEngine::WR_USHORT: + valArray = (uint16_t*) calloc(sizeof(uint16_t), totalRow2); + break; + + case WriteEngine::WR_LONGLONG: + valArray = (long long*) calloc(sizeof(long long), totalRow2); + break; + + case WriteEngine::WR_ULONGLONG: + valArray = (uint64_t*) calloc(sizeof(uint64_t), totalRow2); + break; + + case WriteEngine::WR_TOKEN: + valArray = (Token*) calloc(sizeof(Token), totalRow2); + break; + + case WriteEngine::WR_BINARY: + //case WriteEngine::WR_INT128: + // WIP + valArray = calloc(totalRow2, 16); + break; + + }*/ + + // convert values to valArray + if (m_opType != DELETE) + { + bExcp = false; + + try + { + convertValArray(totalRow2, newColStructList[i].colType, newColValueList[i], valArray); + } + catch (...) + { + bExcp = true; + } + + if (bExcp) + { + if (versioning) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + + return ERR_PARSING; + } + +#ifdef PROFILE + timer.start("writeRow "); +#endif + rc = colOp->writeRow(curCol, totalRow2, secondPart, valArray); +#ifdef PROFILE + timer.stop("writeRow "); +#endif + } + else + { +#ifdef PROFILE + timer.start("writeRow "); +#endif + rc = colOp->writeRow(curCol, totalRow2, rowIdArray, valArray, true); +#ifdef PROFILE + timer.stop("writeRow "); +#endif + } + + + colOp->clearColumn(curCol); + + if (versioning) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + + if (valArray != NULL) + free(valArray); + + // check error + if (rc != NO_ERROR) + break; + } + else + { + valArray = NULL; + + ColumnOp* colOp = m_colOp[op(colStructList[i].fCompressionType)]; + + // set params + colOp->initColumn(curCol); + colOp->setColParam(curCol, 0, colStructList[i].colWidth, + colStructList[i].colDataType, colStructList[i].colType, colStructList[i].dataOid, + colStructList[i].fCompressionType, colStructList[i].fColDbRoot, + colStructList[i].fColPartition, colStructList[i].fColSegment); + + rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file + + //cout << " Opened file oid " << curCol.dataFile.pFile << endl; + if (rc != NO_ERROR) + break; + + ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(colStructList[i].dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == colStructList[i].fColDbRoot) && (it->partNum == colStructList[i].fColPartition) && (it->segNum == colStructList[i].fColSegment)) + break; + + it++; + } + + if (it == aColExtsInfo.end()) //add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = colStructList[i].fColDbRoot; + aExt.partNum = colStructList[i].fColPartition; + aExt.segNum = colStructList[i].fColSegment; + aExt.compType = colStructList[i].fCompressionType; + aColExtsInfo.push_back(aExt); + aTbaleMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); + } + + // handling versioning + vector rangeList; + + if (versioning) + { + rc = processVersionBuffer(curCol.dataFile.pFile, txnid, colStructList[i], + colStructList[i].colWidth, totalRow1, rowIdArray, rangeList); + + if (rc != NO_ERROR) + { + if (colStructList[i].fCompressionType == 0) + { + curCol.dataFile.pFile->flush(); + } + + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + break; + } + } + + // have to init the size here + // shared pointers or memory in a stack + valArray = calloc(totalRow1, colStructList[i].colWidth); + // WIP + /*switch (colStructList[i].colType) + { + case WriteEngine::WR_INT: + case WriteEngine::WR_MEDINT: + valArray = (int*) calloc(sizeof(int), totalRow1); + break; + + case WriteEngine::WR_UINT: + case WriteEngine::WR_UMEDINT: + valArray = (uint32_t*) calloc(sizeof(uint32_t), totalRow1); + break; + + case WriteEngine::WR_VARBINARY : // treat same as char for now + case WriteEngine::WR_CHAR: + case WriteEngine::WR_BLOB: + case WriteEngine::WR_TEXT: + valArray = (char*) calloc(sizeof(char), totalRow1 * MAX_COLUMN_BOUNDARY); + break; + + case WriteEngine::WR_FLOAT: + valArray = (float*) calloc(sizeof(float), totalRow1); + break; + + case WriteEngine::WR_DOUBLE: + valArray = (double*) calloc(sizeof(double), totalRow1); + break; + + case WriteEngine::WR_BYTE: + valArray = (char*) calloc(sizeof(char), totalRow1); + break; + + case WriteEngine::WR_UBYTE: + valArray = (uint8_t*) calloc(sizeof(uint8_t), totalRow1); + break; + + case WriteEngine::WR_SHORT: + valArray = (short*) calloc(sizeof(short), totalRow1); + break; + + case WriteEngine::WR_USHORT: + valArray = (uint16_t*) calloc(sizeof(uint16_t), totalRow1); + break; + + case WriteEngine::WR_LONGLONG: + valArray = (long long*) calloc(sizeof(long long), totalRow1); + break; + + case WriteEngine::WR_ULONGLONG: + valArray = (uint64_t*) calloc(sizeof(uint64_t), totalRow1); + break; + + case WriteEngine::WR_TOKEN: + valArray = (Token*) calloc(sizeof(Token), totalRow1); + break; + + case WriteEngine::WR_BINARY: + //case WriteEngine::WR_INT128: + valArray = calloc(colStructList[i].colWidth, totalRow1); + break; + }*/ + + // convert values to valArray + if (m_opType != DELETE) + { + bExcp = false; + + try + { + convertValArray(totalRow1, cscColTypeList[i], colStructList[i].colType, colValueList[i], valArray); + } + catch (...) + { + bExcp = true; + } + + if (bExcp) + { + if (versioning) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + + return ERR_PARSING; + } + +#ifdef PROFILE + timer.start("writeRow "); +#endif + rc = colOp->writeRow(curCol, totalRow1, rowIdArray, valArray); +#ifdef PROFILE + timer.stop("writeRow "); +#endif + } + else + { +#ifdef PROFILE + timer.start("writeRow "); +#endif + rc = colOp->writeRow(curCol, totalRow1, rowIdArray, valArray, true); +#ifdef PROFILE + timer.stop("writeRow "); +#endif + } + + colOp->clearColumn(curCol); + + if (versioning) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); + + if (valArray != NULL) + free(valArray); + + // check error + if (rc != NO_ERROR) + break; + } + } // end of for (i = 0 + +#ifdef PROFILE + timer.finish(); +#endif + return rc; +} + int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, const ColStructList& colStructList, ColValueList& colValueList, @@ -4828,11 +5760,13 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, } } - //totalRow1 -= totalRow2; + // WIP We can allocate based on column size and not colType // have to init the size here - // nullArray = (bool*) malloc(sizeof(bool) * totalRow); + valArray = calloc(totalRow1, colStructList[i].colWidth); +#if 0 switch (colStructList[i].colType) { + // WIP we don't need type cast here only size case WriteEngine::WR_INT: case WriteEngine::WR_MEDINT: valArray = (int*) calloc(sizeof(int), totalRow1); @@ -4888,18 +5822,24 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, // WIP case WriteEngine::WR_BINARY: - case WriteEngine::WR_INT128: - valArray = calloc(totalRow1, 16); + valArray = calloc(totalRow1, colStructList[i].colWidth); break; } +#endif // convert values to valArray + // WIP + // Is this m_opType ever set to DELETE? if (m_opType != DELETE) { bExcp = false; try { + // WIP We convert values twice!? + // dmlcommandproc converts strings to boost::any and this converts + // into actual type value masked by *void + // It is not clear why we need to convert to boost::any b/c we can convert from the original string here convertValArray(totalRow1, colStructList[i].colType, colValueList[i], valArray); } catch (...) @@ -4916,7 +5856,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, } #ifdef PROFILE - timer.start("writeRow "); + iimer.start("writeRow "); #endif rc = colOp->writeRow(curCol, totalRow1, firstPart, valArray); #ifdef PROFILE @@ -5009,9 +5949,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, } } - //totalRow1 -= totalRow2; // have to init the size here -// nullArray = (bool*) malloc(sizeof(bool) * totalRow); switch (newColStructList[i].colType) { case WriteEngine::WR_INT: @@ -5068,7 +6006,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, break; case WriteEngine::WR_BINARY: - case WriteEngine::WR_INT128: + //case WriteEngine::WR_INT128: // WIP valArray = calloc(totalRow2, 16); break; @@ -5248,7 +6186,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, break; case WriteEngine::WR_BINARY: - case WriteEngine::WR_INT128: + //case WriteEngine::WR_INT128: valArray = calloc(colStructList[i].colWidth, totalRow1); break; } @@ -5314,6 +6252,8 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, return rc; } + + int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, const ColStructList& colStructList, std::vector& colValueList, @@ -5472,7 +6412,8 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, break; case WriteEngine::WR_BINARY: - case WriteEngine::WR_INT128: + // WIP + //case WriteEngine::WR_INT128: ((uint64_t*)valArray)[j] = curValue; //FIXME maybe break; @@ -5621,7 +6562,8 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, break; case WriteEngine::WR_BINARY: - case WriteEngine::WR_INT128: + // WIP + //case WriteEngine::WR_INT128: ((uint64_t*)valArray)[j] = curValue; // FIXME maybe break; } @@ -5657,7 +6599,6 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, return rc; } - int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, const ColStructList& colStructList, const ColValueList& colValueList, @@ -5908,8 +6849,340 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, valArray = (Token*) calloc(sizeof(Token), 1); break; case WriteEngine::WR_BINARY: - case WriteEngine::WR_INT128: - valArray = (char*) calloc(sizeof(char), curColStruct.colWidth); //FIXME maybe + //case WriteEngine::WR_INT128: + valArray = calloc(sizeof(char), curColStruct.colWidth); //FIXME maybe + break; + } + + // convert values to valArray + if (m_opType != DELETE) + { + bExcp = false; + ColTuple curTuple; + curTuple = curTupleList[0]; + + try + { + convertValue(curColStruct.colType, valArray, curTuple.data); + } + catch (...) + { + bExcp = true; + } + + if (bExcp) + { + if (rangeListTot.size() > 0) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeListTot); + + return ERR_PARSING; + } + +#ifdef PROFILE + timer.start("writeRow "); +#endif + rc = colOp->writeRows(curCol, totalRow, ridList, valArray); +#ifdef PROFILE + timer.stop("writeRow "); +#endif + } + else + { +#ifdef PROFILE + timer.start("writeRows "); +#endif + rc = colOp->writeRows(curCol, totalRow, ridList, valArray, 0, true); +#ifdef PROFILE + timer.stop("writeRows "); +#endif + } + +// colOldValueList.push_back(oldValArray); +//timer.start("Delete:closefile"); + colOp->clearColumn(curCol); + +//timer.stop("Delete:closefile"); + if (valArray != NULL) + free(valArray); + + // check error + if (rc != NO_ERROR) + break; + + } // end of for (i = 0) + +// timer.start("Delete:purgePrimProcFdCache"); + if ((idbdatafile::IDBPolicy::useHdfs()) && (files.size() > 0)) + cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); + +//if (idbdatafile::IDBPolicy::useHdfs()) +// cacheutils::dropPrimProcFdCache(); +//timer.stop("Delete:purgePrimProcFdCache"); + if (rangeListTot.size() > 0) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeListTot); + +//timer.stop("Delete:writecolrec"); +//#ifdef PROFILE +//timer.finish(); +//#endif + return rc; +} + + + +int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, + const CSCTypesList& cscColTypes, + const ColStructList& colStructList, + const ColValueList& colValueList, + vector& colOldValueList, + const RIDList& ridList, + const int32_t tableOid, + bool convertStructFlag, + ColTupleList::size_type nRows) +{ + bool bExcp; + int rc = 0; + void* valArray = NULL; + Column curCol; + ColStruct curColStruct; + ColTupleList curTupleList, oldTupleList; + ColStructList::size_type totalColumn; + ColStructList::size_type i; + ColTupleList::size_type totalRow; + + setTransId(txnid); + colOldValueList.clear(); + totalColumn = colStructList.size(); + totalRow = nRows; + +#ifdef PROFILE + StopWatch timer; +#endif + + vector rangeListTot; + std::vector freeList; + vector > fboLists; + vector > rangeLists; + rc = processBeginVBCopy(txnid, colStructList, ridList, freeList, fboLists, rangeLists, rangeListTot); + + if (rc != NO_ERROR) + { + if (rangeListTot.size() > 0) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeListTot); + + switch (rc) + { + case BRM::ERR_DEADLOCK: + return ERR_BRM_DEAD_LOCK; + + case BRM::ERR_VBBM_OVERFLOW: + return ERR_BRM_VB_OVERFLOW; + + case BRM::ERR_NETWORK: + return ERR_BRM_NETWORK; + + case BRM::ERR_READONLY: + return ERR_BRM_READONLY; + + default: + return ERR_BRM_BEGIN_COPY; + } + } + + VBRange aRange; + uint32_t blocksProcessedThisOid = 0; + uint32_t blocksProcessed = 0; + std::vector files; + TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); + + for (i = 0; i < totalColumn; i++) + { + valArray = NULL; + curColStruct = colStructList[i]; + curTupleList = colValueList[i]; //same value for all rows + ColumnOp* colOp = m_colOp[op(curColStruct.fCompressionType)]; + + // convert column data type + if (convertStructFlag) + Convertor::convertColType(&curColStruct); + + // set params + colOp->initColumn(curCol); + colOp->setColParam(curCol, 0, curColStruct.colWidth, + curColStruct.colDataType, curColStruct.colType, curColStruct.dataOid, + curColStruct.fCompressionType, curColStruct.fColDbRoot, + curColStruct.fColPartition, curColStruct.fColSegment); + + + ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(curColStruct.dataOid); + ColExtsInfo::iterator it = aColExtsInfo.begin(); + + while (it != aColExtsInfo.end()) + { + if ((it->dbRoot == curColStruct.fColDbRoot) && (it->partNum == curColStruct.fColPartition) && (it->segNum == curColStruct.fColSegment)) + break; + + it++; + } + + if (it == aColExtsInfo.end()) //add this one to the list + { + ColExtInfo aExt; + aExt.dbRoot = curColStruct.fColDbRoot; + aExt.partNum = curColStruct.fColPartition; + aExt.segNum = curColStruct.fColSegment; + aExt.compType = curColStruct.fCompressionType; + aColExtsInfo.push_back(aExt); + aTbaleMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); + } + + string segFile; + rc = colOp->openColumnFile(curCol, segFile, true, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file + + if (rc != NO_ERROR) + break; + + if (curColStruct.fCompressionType == 0) + { + BRM::FileInfo aFile; + aFile.oid = curColStruct.dataOid; + aFile.partitionNum = curColStruct.fColPartition; + aFile.dbRoot = curColStruct.fColDbRoot;; + aFile.segmentNum = curColStruct.fColSegment; + aFile.compType = curColStruct.fCompressionType; + files.push_back(aFile); + } + + // handling versioning + //cout << " pass to processVersionBuffer rid " << rowIdArray[0] << endl; + //cout << "dataOid:fColPartition = " << curColStruct.dataOid << ":" << curColStruct.fColPartition << endl; +//timer.start("processVersionBuffers"); + //vector rangeList; + // rc = processVersionBuffers(curCol.dataFile.pFile, txnid, curColStruct, curColStruct.colWidth, totalRow, ridList, rangeList); + std::vector curFreeList; + uint32_t blockUsed = 0; + + if (!idbdatafile::IDBPolicy::useHdfs()) + { + if (rangeListTot.size() > 0) + { + if (freeList[0].size >= (blocksProcessed + rangeLists[i].size())) + { + aRange.vbOID = freeList[0].vbOID; + aRange.vbFBO = freeList[0].vbFBO + blocksProcessed; + aRange.size = rangeLists[i].size(); + curFreeList.push_back(aRange); + //cout << "range size = " << aRange.size <<" and blocksProcessed = " << blocksProcessed<< endl; + } + else + { + aRange.vbOID = freeList[0].vbOID; + aRange.vbFBO = freeList[0].vbFBO + blocksProcessed; + aRange.size = freeList[0].size - blocksProcessed; + blockUsed = aRange.size; + curFreeList.push_back(aRange); + + if (freeList.size() > 1) + { + aRange.vbOID = freeList[1].vbOID; + aRange.vbFBO = freeList[1].vbFBO + blocksProcessedThisOid; + aRange.size = rangeLists[i].size() - blockUsed; + curFreeList.push_back(aRange); + blocksProcessedThisOid += aRange.size; + } + else + { + rc = 1; + break; + } + + //cout << "curFreeList size = " << curFreeList.size() << endl; + + } + + blocksProcessed += rangeLists[i].size(); + + //timer.start("Delete:writeVB"); + rc = BRMWrapper::getInstance()-> + writeVB(curCol.dataFile.pFile, (BRM::VER_t)txnid, + curColStruct.dataOid, fboLists[i], rangeLists[i], + colOp, curFreeList, curColStruct.fColDbRoot, true); + } + } + + //timer.stop("Delete:writeVB"); +//timer.stop("processVersionBuffers"); + // cout << " rc for processVersionBuffer is " << rc << endl; + if (rc != NO_ERROR) + { + if (curColStruct.fCompressionType == 0) + { + curCol.dataFile.pFile->flush(); + } + + if (rangeListTot.size() > 0) + BRMWrapper::getInstance()->writeVBEnd(txnid, rangeListTot); + + break; + } + + switch (curColStruct.colType) + { + case WriteEngine::WR_INT: + case WriteEngine::WR_MEDINT: + valArray = (int*) calloc(sizeof(int), 1); + break; + + case WriteEngine::WR_UINT: + case WriteEngine::WR_UMEDINT: + valArray = (uint32_t*) calloc(sizeof(uint32_t), 1); + break; + + case WriteEngine::WR_VARBINARY : // treat same as char for now + case WriteEngine::WR_CHAR: + case WriteEngine::WR_BLOB: + case WriteEngine::WR_TEXT: + valArray = (char*) calloc(sizeof(char), 1 * MAX_COLUMN_BOUNDARY); + break; + + case WriteEngine::WR_FLOAT: + valArray = (float*) calloc(sizeof(float), 1); + break; + + case WriteEngine::WR_DOUBLE: + valArray = (double*) calloc(sizeof(double), 1); + break; + + case WriteEngine::WR_BYTE: + valArray = (char*) calloc(sizeof(char), 1); + break; + + case WriteEngine::WR_UBYTE: + valArray = (uint8_t*) calloc(sizeof(uint8_t), 1); + break; + + case WriteEngine::WR_SHORT: + valArray = (short*) calloc(sizeof(short), 1); + break; + + case WriteEngine::WR_USHORT: + valArray = (uint16_t*) calloc(sizeof(uint16_t), 1); + break; + + case WriteEngine::WR_LONGLONG: + valArray = (long long*) calloc(sizeof(long long), 1); + break; + + case WriteEngine::WR_ULONGLONG: + valArray = (uint64_t*) calloc(sizeof(uint64_t), 1); + break; + + case WriteEngine::WR_TOKEN: + valArray = (Token*) calloc(sizeof(Token), 1); + break; + case WriteEngine::WR_BINARY: + //case WriteEngine::WR_INT128: + valArray = calloc(sizeof(char), curColStruct.colWidth); //FIXME maybe break; } @@ -6472,7 +7745,6 @@ int WriteEngineWrapper::updateNextValue(const TxnID txnId, const OID& columnoid, colTuple.data = nextVal; colTuples.push_back(colTuple); colValueList.push_back(colTuples); - //TxnID txnid; rc = writeColumnRecords(txnId, colStructList, colValueList, ridList, SYSCOLUMN_BASE, false); if (rc != NO_ERROR) diff --git a/writeengine/wrapper/writeengine.h b/writeengine/wrapper/writeengine.h index 229d8c76e..f3ca7ec4e 100644 --- a/writeengine/wrapper/writeengine.h +++ b/writeengine/wrapper/writeengine.h @@ -157,7 +157,15 @@ public: /** * @brief Convert interface value list to internal value array */ - EXPORT void convertValArray(size_t totalRow, const ColType colType, + EXPORT void convertValArray(const size_t totalRow, + const execplan::CalpontSystemCatalog::ColType& cscColType, + const ColType colType, + ColTupleList& curTupleList, void* valArray, + bool bFromList = true) ; + + // WIP legacy + EXPORT void convertValArray(const size_t totalRow, + const ColType colType, ColTupleList& curTupleList, void* valArray, bool bFromList = true) ; @@ -364,6 +372,7 @@ public: * @param dicStringListt dictionary values list */ EXPORT int insertColumnRec_Single(const TxnID& txnid, + CSCTypesList& cscColTypesList, ColStructList& colStructList, ColValueList& colValueList, DctnryStructList& dctnryStructList, @@ -650,10 +659,8 @@ private: /** * @brief Convert interface column type to a internal column type */ - // void convertColType(void* curStruct, const FuncType curType = FUNC_WRITE_ENGINE) const; - + void convertValue(const execplan::CalpontSystemCatalog::ColType &fullColType, ColType colType, void* valArray, size_t pos, boost::any& data, bool fromList = true); void convertValue(const ColType colType, void* valArray, size_t pos, boost::any& data, bool fromList = true); - /** * @brief Convert column value to its internal representation * @@ -661,6 +668,7 @@ private: * @param value Memory pointer for storing output value. Should be pre-allocated * @param data Column data */ + void convertValue(const execplan::CalpontSystemCatalog::ColType &fullColType, const ColType colType, void* value, boost::any& data); void convertValue(const ColType colType, void* value, boost::any& data); /** @@ -690,11 +698,21 @@ private: /** * @brief Common methods to write values to a column */ - int writeColumnRec(const TxnID& txnid, const ColStructList& colStructList, + int writeColumnRec(const TxnID& txnid, + const CSCTypesList& cscColTypes, + const ColStructList& colStructList, ColValueList& colValueList, RID* rowIdArray, const ColStructList& newColStructList, ColValueList& newColValueList, const int32_t tableOid, bool useTmpSuffix, bool versioning = true); + // WIP + int writeColumnRec(const TxnID& txnid, + const ColStructList& colStructList, + ColValueList& colValueList, + RID* rowIdArray, const ColStructList& newColStructList, + ColValueList& newColValueList, const int32_t tableOid, + bool useTmpSuffix, bool versioning = true); + int writeColumnRecBinary(const TxnID& txnid, const ColStructList& colStructList, std::vector& colValueList, @@ -705,10 +723,19 @@ private: //@Bug 1886,2870 pass the address of ridList vector - int writeColumnRec(const TxnID& txnid, const ColStructList& colStructList, + int writeColumnRec(const TxnID& txnid, + const CSCTypesList& cscColTypes, + const ColStructList& colStructList, const ColValueList& colValueList, std::vector& colOldValueList, const RIDList& ridList, const int32_t tableOid, bool convertStructFlag = true, ColTupleList::size_type nRows = 0); + // WIP legacy + int writeColumnRec(const TxnID& txnid, + const ColStructList& colStructList, + const ColValueList& colValueList, std::vector& colOldValueList, + const RIDList& ridList, const int32_t tableOid, + bool convertStructFlag = true, ColTupleList::size_type nRows = 0); + //For update column from column to use int writeColumnRecords(const TxnID& txnid, std::vector& colStructList, From 54c152d6c8e93dadb0544ffce0eadfe5adc39c92 Mon Sep 17 00:00:00 2001 From: drrtuy Date: Tue, 21 Jan 2020 12:57:31 +0300 Subject: [PATCH 08/78] MCOL-641 This commit introduces templates for DataConvert and RowGroup methods. --- dbcon/execplan/simplecolumn.cpp | 6 ++++ dbcon/execplan/treenode.h | 9 ++++-- dbcon/mysql/ha_mcs_execplan.cpp | 9 +++--- dbcon/mysql/ha_mcs_impl.cpp | 34 +++++++++++++++++++---- primitives/primproc/columncommand.cpp | 17 ++++++++---- utils/dataconvert/dataconvert.cpp | 40 +++++++++++++++++++-------- utils/dataconvert/dataconvert.h | 7 +++-- utils/rowgroup/rowgroup.cpp | 5 ++++ utils/rowgroup/rowgroup.h | 21 +++++++++++--- 9 files changed, 111 insertions(+), 37 deletions(-) diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index a58ad66b7..921d4d180 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -625,6 +625,12 @@ void SimpleColumn::evaluate(Row& row, bool& isNull) { switch (fResultType.colWidth) { + case 16: + { + fResult.decimalVal.value = row.getIntField<16>(fInputIndex); + fResult.decimalVal.scale = (unsigned)fResultType.scale; + break; + } case 1: { fResult.decimalVal.value = row.getIntField<1>(fInputIndex); diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index e50c3f30f..fcc92018d 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -36,6 +36,8 @@ #include "exceptclasses.h" #include "dataconvert.h" +using uint128_t = unsigned __int128; + namespace messageqcpp { class ByteStream; @@ -61,7 +63,7 @@ typedef execplan::CalpontSystemCatalog::ColType Type; */ struct IDB_Decimal { - IDB_Decimal(): value(0), scale(0), precision(0) {} + IDB_Decimal(): val(0), value(0), scale(0), precision(0) {} IDB_Decimal(int64_t val, int8_t s, uint8_t p) : value (val), scale(s), @@ -149,9 +151,10 @@ struct IDB_Decimal return (decimalComp(rhs) != 0); } + uint128_t val; int64_t value; - int8_t scale; // 0~18 - uint8_t precision; // 1~18 + int8_t scale; // 0~38 + uint8_t precision; // 1~38 }; typedef IDB_Decimal CNX_Decimal; diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 8a78fbddd..1e4557a71 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -3106,13 +3106,14 @@ CalpontSystemCatalog::ColType colType_MysqlToIDB (const Item* item) { Item_decimal* idp = (Item_decimal*)item; ct.colDataType = CalpontSystemCatalog::DECIMAL; - ct.colWidth = 8; + // MCOL-641 WIP Make this dynamic + ct.colWidth = (idp->max_length >= 18) ? 16 : 8; ct.scale = idp->decimals; if (ct.scale == 0) - ct.precision = idp->max_length - 1; + ct.precision = (idp->max_length > 38) ? 38 : idp->max_length - 1; else - ct.precision = idp->max_length - idp->decimals; + ct.precision = (idp->max_length > 38) ? 38 : idp->max_length - idp->decimals; break; } @@ -3599,10 +3600,8 @@ ArithmeticColumn* buildArithmeticColumn( pt->right(rhs); } - //aop->resultType(colType_MysqlToIDB(item)); // @bug5715. Use InfiniDB adjusted coltype for result type. // decimal arithmetic operation gives double result when the session variable is set. - //idbassert(pt->left() && pt->right() && pt->left()->data() && pt->right()->data()); CalpontSystemCatalog::ColType mysql_type = colType_MysqlToIDB(item); if (get_double_for_decimal_math(current_thd) == true) diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 4d26ab5a0..84d49ebe4 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -145,6 +145,8 @@ extern bool nonConstFunc(Item_func* ifp); namespace { + using int128_t = __int128; + using uint128_t = unsigned __int128; // Calpont vtable non-support error message const string infinidb_autoswitch_warning = "The query includes syntax that is not supported by MariaDB Columnstore distributed mode. The execution was switched to standard mode with downgraded performance."; @@ -247,9 +249,6 @@ void force_close_fep_conn(THD *thd, cal_connection_info* ci, bool check_prev_rc ci->cal_conn_hndl = 0; } -// WIP MCOL-641 -using uint128_t = unsigned __int128; - void storeNumericField(Field** f, int64_t value, CalpontSystemCatalog::ColType& ct) { // unset null bit first @@ -268,6 +267,8 @@ void storeNumericField(Field** f, int64_t value, CalpontSystemCatalog::ColType& //if (f2->dec < ct.scale) // f2->dec = ct.scale; + // WIP MCOL-641 + // This is too much char buf[256]; dataconvert::DataConvert::decimalToString(value, (unsigned)ct.scale, buf, 256, ct.colDataType); (*f)->store(buf, strlen(buf), (*f)->charset()); @@ -808,12 +809,33 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h if (row.getPrecision(s) > 18) { // unset null bit first + // Might be redundant if ((*f)->null_ptr) *(*f)->null_ptr &= ~(*f)->null_bit; - const uint128_t val = *reinterpret_cast(row.getBinaryField2(s)); - char buf[256]; - dataconvert::DataConvert::decimalToString(val, (unsigned)colType.scale, buf, 256, colType.colDataType); + uint128_t* udec; + int128_t* dec; + // We won't have more than 38 digits + sign + dp + // Make this precision based + char buf[41]; + + // This C-style cast doesn't look appropriate. + // Is there a way to use decltype instead of if? + if (colType.colDataType == CalpontSystemCatalog::DECIMAL) + { + dec = row.getBinaryField(s); + dataconvert::DataConvert::decimalToString(dec, + (unsigned)colType.scale, buf, + sizeof(buf), colType.colDataType); + } + else + { + udec = row.getBinaryField(s); + dataconvert::DataConvert::decimalToString(udec, + (unsigned)colType.scale, buf, + sizeof(buf), colType.colDataType); + } + Field_new_decimal* f2 = (Field_new_decimal*)*f; f2->store(buf, strlen(buf), f2->charset()); } diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index 7f24f6931..30d76d0f2 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -281,6 +281,7 @@ void ColumnCommand::process_OT_BOTH() bpp->relRids[i] = *((uint16_t*) &bpp->outputMsg[pos]); pos += 2; + // WIP // values[i] is 8 Bytes wide so coping the pointer to bpp->outputMsg[pos] and crossing fingers // I dont know the liveness of bpp->outputMsg but also I dont know if there is other memory area I can use values[i] = (int64_t) &bpp->outputMsg[pos]; @@ -591,6 +592,7 @@ void ColumnCommand::prep(int8_t outputType, bool absRids) mask = 0x01; break; case 16: + // WIP MCOL-641 cout << __FILE__<< ":" <<__LINE__ << " Fix shift and mask for 16 Bytes ?"<< endl; shift = 1; mask = 0x01; @@ -778,10 +780,8 @@ void ColumnCommand::projectResultRG(RowGroup& rg, uint32_t pos) r.setUintField_offset<4>(*((uint32_t*) msg8), offset); r.nextRow(rowSize); } - break; } - case 8: { for (i = 0; i < outMsg->NVALS; ++i, msg8 += gapSize) @@ -789,12 +789,19 @@ void ColumnCommand::projectResultRG(RowGroup& rg, uint32_t pos) r.setUintField_offset<8>(*((uint64_t*) msg8), offset); r.nextRow(rowSize); } - break; } - case 16: - cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + { + cout << __FILE__<< ":" <<__LINE__ << " ColumnCommand::projectResultRG " << endl; + for (i = 0; i < outMsg->NVALS; ++i, msg8 += gapSize) + { + r.setBinaryField(msg8, colType.colWidth, offset); + r.nextRow(rowSize); + } + break; + } + } } diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index e90471b97..f9510c311 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1174,21 +1174,28 @@ struct uint128_pod }; // WIP MCOL-641 -void DataConvert::toString(unsigned __int128 i, char *p) +// Check for overflows with buflen +template +void DataConvert::toString(T* dec, char *p, size_t buflen) { uint64_t div = 10000000000000000000ULL; size_t div_log = 19; - uint128_t high = i; + // template this + uint128_t high = *dec; uint128_t low; low = high % div; high /= div; uint128_t mid; mid = high % div; high /= div; - + + // WIP How to treat PODs here ? + // use typeof + // Or a templated structure uint128_pod *high_pod = reinterpret_cast(&high); uint128_pod *mid_pod = reinterpret_cast(&mid); uint128_pod *low_pod = reinterpret_cast(&low); + char* original_p = p; int printed_chars = 0; // WIP replace snprintf with streams @@ -1202,9 +1209,9 @@ void DataConvert::toString(unsigned __int128 i, char *p) p += printed_chars; } snprintf(p, div_log+1, "%019lu", low_pod->lo); + if (buflen <= p-original_p) + std::cout << "DataConvert::toString char buffer overflow" << std::endl; } - - // WIP MCOL-641 // Template this // result must be calloc-ed @@ -1213,7 +1220,7 @@ void DataConvert::toString(unsigned __int128 i, char *p) void atoi128(const string& arg, int128_t& res) { // WIP - //char buf[40]; + //char buf[41]; //int128_t *res_ptr = reinterpret_cast(result); res = 0; for (size_t j = 0; j < arg.size(); j++) @@ -1228,10 +1235,14 @@ void atoi128(const string& arg, int128_t& res) } // WIP MCOL-641 -void DataConvert::decimalToString(unsigned __int128 int_val, uint8_t scale, char* buf, unsigned int buflen, - execplan::CalpontSystemCatalog::ColDataType colDataType) +template +void DataConvert::decimalToString(T* valuePtr, + uint8_t scale, + char* buf, + unsigned int buflen, + execplan::CalpontSystemCatalog::ColDataType colDataType) { - toString(int_val, buf); + toString(valuePtr, buf, buflen); // Biggest ColumnStore supports is DECIMAL(38,x), or 38 total digits+dp+sign for column @@ -1243,7 +1254,7 @@ void DataConvert::decimalToString(unsigned __int128 int_val, uint8_t scale, char size_t l1 = strlen(buf); char* ptr = &buf[0]; - if (int_val < 0) + if (*valuePtr < 0) { ptr++; idbassert(l1 >= 2); @@ -1259,7 +1270,7 @@ void DataConvert::decimalToString(unsigned __int128 int_val, uint8_t scale, char const char* zeros = "00000000000000000000000000000000000000"; //38 0's size_t diff = 0; - if (int_val != 0) + if (*valuePtr != 0) diff = scale - l1; //this will always be > 0 else diff = scale; @@ -1267,7 +1278,7 @@ void DataConvert::decimalToString(unsigned __int128 int_val, uint8_t scale, char memmove((ptr + diff), ptr, l1 + 1); //also move null memcpy(ptr, zeros, diff); - if (int_val != 0) + if (*valuePtr != 0) l1 = 0; else l1 = 1; @@ -1289,6 +1300,11 @@ void DataConvert::decimalToString(unsigned __int128 int_val, uint8_t scale, char *(ptr + l1) = '.'; } +// Explicit instantiation +template +void DataConvert::decimalToString(int128_t* value, uint8_t scale, char* buf, unsigned int buflen, execplan::CalpontSystemCatalog::ColDataType colDataType); +template +void DataConvert::decimalToString(uint128_t* value, uint8_t scale, char* buf, unsigned int buflen, execplan::CalpontSystemCatalog::ColDataType colDataType); boost::any DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 18a079fbb..05cc27777 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -1011,8 +1011,11 @@ public: EXPORT static bool isNullData(execplan::ColumnResult* cr, int rownum, execplan::CalpontSystemCatalog::ColType colType); static inline std::string decimalToString(int64_t value, uint8_t scale, execplan::CalpontSystemCatalog::ColDataType colDataType); static inline void decimalToString(int64_t value, uint8_t scale, char* buf, unsigned int buflen, execplan::CalpontSystemCatalog::ColDataType colDataType); - EXPORT static void decimalToString(unsigned __int128 value, uint8_t scale, char* buf, unsigned int buflen, execplan::CalpontSystemCatalog::ColDataType colDataType); - EXPORT static void toString(unsigned __int128 i, char *p); + template + EXPORT static void decimalToString(T* value, uint8_t scale, char* buf, unsigned int buflen, execplan::CalpontSystemCatalog::ColDataType colDataType); + + template + EXPORT static void toString(T* dec, char *p, size_t buflen); static inline std::string constructRegexp(const std::string& str); static inline void trimWhitespace(int64_t& charData); static inline bool isEscapedChar(char c) diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index df10ed108..9e55810d6 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -953,6 +953,11 @@ bool Row::isNullValue(uint32_t colIndex) const switch (len) { + // MCOL-641 WIP + case 16: + return (*((int64_t*) &data[offsets[colIndex]]) == static_cast(joblist::BIGINTNULL)); + break; + case 1 : return (data[offsets[colIndex]] == joblist::TINYINTNULL); diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 03c087cb4..534b9b67a 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -435,7 +435,10 @@ public: inline void setVarBinaryField(const uint8_t* val, uint32_t len, uint32_t colIndex); inline std::string getBinaryField(uint32_t colIndex) const; - inline const uint8_t* getBinaryField2(uint32_t colIndex) const; + template + inline T* getBinaryField(uint32_t colIndex) const; + template + inline T* getBinaryField_offset(uint32_t offset) const; inline boost::shared_ptr getUserData(uint32_t colIndex) const; inline void setUserData(mcsv1sdk::mcsv1Context& context, @@ -792,7 +795,7 @@ inline uint32_t Row::getStringLength(uint32_t colIndex) const return strnlen((char*) &data[offsets[colIndex]], getColumnWidth(colIndex)); } - +// Check whether memcpy affects perf here inline void Row::setBinaryField(const uint8_t* strdata, uint32_t length, uint32_t offset) { memcpy(&data[offset], strdata, length); @@ -837,11 +840,19 @@ inline std::string Row::getBinaryField(uint32_t colIndex) const } // WIP MCOL-641 -inline const uint8_t* Row::getBinaryField2(uint32_t colIndex) const +template +inline T* Row::getBinaryField(uint32_t colIndex) const { - return &data[offsets[colIndex]]; + return reinterpret_cast(&data[offsets[colIndex]]); } +template +inline T* Row::getBinaryField_offset(uint32_t offset) const +{ + return reinterpret_cast(&data[offset]); +} + + inline std::string Row::getVarBinaryStringField(uint32_t colIndex) const { if (inStringTable(colIndex)) @@ -961,10 +972,12 @@ inline void Row::setUintField_offset(uint64_t val, uint32_t offset) case 8: *((uint64_t*) &data[offset]) = val; break; + /* This doesn't look like appropriate place case 16: std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; *((uint64_t*) &data[offset]) = val; break; + */ default: idbassert(0); throw std::logic_error("Row::setUintField called on a non-uint32_t field"); From 98213c0094d10de184e8f5422b96f98b48872b64 Mon Sep 17 00:00:00 2001 From: drrtuy Date: Sun, 26 Jan 2020 11:07:45 +0300 Subject: [PATCH 09/78] MCOL-641 Addition now works for DECIMAL columns with precision > 18. --- dbcon/execplan/arithmeticoperator.h | 19 +++- dbcon/execplan/returnedcolumn.cpp | 6 ++ dbcon/execplan/returnedcolumn.h | 1 + dbcon/execplan/simplecolumn.cpp | 19 +++- dbcon/execplan/treenode.h | 17 +++- .../primproc/batchprimitiveprocessor.cpp | 2 +- primitives/primproc/columncommand.cpp | 2 +- utils/dataconvert/dataconvert.cpp | 2 +- utils/funcexp/funcexp.cpp | 22 ++++- utils/rowgroup/rowgroup.cpp | 96 +++++++++++++++++++ utils/rowgroup/rowgroup.h | 3 + 11 files changed, 175 insertions(+), 14 deletions(-) diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index ddf0b7ead..4660387fe 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -43,6 +43,7 @@ namespace execplan class ArithmeticOperator : public Operator { +using cscType = execplan::CalpontSystemCatalog::ColType; public: ArithmeticOperator(); @@ -200,6 +201,7 @@ private: template inline result_t execute(result_t op1, result_t op2, bool& isNull); inline void execute(IDB_Decimal& result, IDB_Decimal op1, IDB_Decimal op2, bool& isNull); + inline void execute(IDB_Decimal& result, IDB_Decimal op1, IDB_Decimal op2, bool& isNull, cscType& resultCscType); std::string fTimeZone; }; @@ -236,12 +238,11 @@ inline void ArithmeticOperator::evaluate(rowgroup::Row& row, bool& isNull, Parse case execplan::CalpontSystemCatalog::LONGDOUBLE: fResult.longDoubleVal = execute(lop->getLongDoubleVal(row, isNull), rop->getLongDoubleVal(row, isNull), isNull); break; - + // WIP MCOL-641 case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: - execute (fResult.decimalVal, lop->getDecimalVal(row, isNull), rop->getDecimalVal(row, isNull), isNull); + execute(fResult.decimalVal, lop->getDecimalVal(row, isNull), rop->getDecimalVal(row, isNull), isNull, fOperationType); break; - default: { std::ostringstream oss; @@ -282,11 +283,21 @@ inline result_t ArithmeticOperator::execute(result_t op1, result_t op2, bool& is } } -inline void ArithmeticOperator::execute(IDB_Decimal& result, IDB_Decimal op1, IDB_Decimal op2, bool& isNull) +inline void ArithmeticOperator::execute(IDB_Decimal& result, IDB_Decimal op1, IDB_Decimal op2, bool& isNull, cscType& resultCscType) { switch (fOp) { case OP_ADD: + if (resultCscType.precision > 18) + { + // WIP make this a separate function w and w/o overflow check + if (resultCscType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) + result.__v.__s128 = op1.__v.__s128 + op2.__v.__s128; + else + result.__v.__u128 = op1.__v.__u128 + op2.__v.__u128; + break; + } + if (result.scale == op1.scale && result.scale == op2.scale) { result.value = op1.value + op2.value; diff --git a/dbcon/execplan/returnedcolumn.cpp b/dbcon/execplan/returnedcolumn.cpp index 44278424f..5900f077f 100644 --- a/dbcon/execplan/returnedcolumn.cpp +++ b/dbcon/execplan/returnedcolumn.cpp @@ -49,6 +49,7 @@ ReturnedColumn::ReturnedColumn(): fReturnAll (false), fColPosition(-1), fHasAggregate(false), fInputIndex(-1), + fInputOffset(-1), fOutputIndex(-1), fExpressionId ((uint32_t) - 1) { @@ -69,6 +70,7 @@ ReturnedColumn::ReturnedColumn(const string& sql): fHasAggregate(false), fData(sql), fInputIndex(-1), + fInputOffset(-1), fOutputIndex(-1), fExpressionId ((uint32_t) - 1) { @@ -88,6 +90,7 @@ ReturnedColumn::ReturnedColumn(const uint32_t sessionID, const bool returnAll): fColPosition(-1), fHasAggregate(false), fInputIndex(-1), + fInputOffset(-1), fOutputIndex(-1), fExpressionId ((uint32_t) - 1) { @@ -109,6 +112,7 @@ ReturnedColumn::ReturnedColumn(const ReturnedColumn& rhs, const uint32_t session fHasAggregate(rhs.fHasAggregate), fData(rhs.fData), fInputIndex(rhs.fInputIndex), + fInputOffset(rhs.fInputOffset), fOutputIndex(rhs.fOutputIndex), fExpressionId (rhs.fExpressionId) { @@ -141,6 +145,7 @@ void ReturnedColumn::serialize(messageqcpp::ByteStream& b) const b << (uint64_t)fColSource; b << (int64_t)fColPosition; b << (uint32_t)fInputIndex; + b << (uint32_t)fInputOffset; b << (uint32_t)fOutputIndex; b << (int32_t)fSequence; b << (uint8_t)fReturnAll; @@ -163,6 +168,7 @@ void ReturnedColumn::unserialize(messageqcpp::ByteStream& b) b >> (uint64_t&)fColSource; b >> (int64_t&)fColPosition; b >> (uint32_t&)fInputIndex; + b >> (uint32_t&)fInputOffset; b >> (uint32_t&)fOutputIndex; b >> (int32_t&)fSequence; b >> (uint8_t&)fReturnAll; diff --git a/dbcon/execplan/returnedcolumn.h b/dbcon/execplan/returnedcolumn.h index 45f23c047..6c07f5694 100644 --- a/dbcon/execplan/returnedcolumn.h +++ b/dbcon/execplan/returnedcolumn.h @@ -374,6 +374,7 @@ public: protected: std::string fErrMsg; /// error occured in evaluation uint32_t fInputIndex; /// index to the input rowgroup + uint32_t fInputOffset; /// index to the input rowgroup uint32_t fOutputIndex; /// index to the output rowgroup uint32_t fExpressionId; /// unique id for this expression }; diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index 921d4d180..96466403a 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -51,6 +51,7 @@ using namespace joblist; #include "aggregatecolumn.h" #include "constantfilter.h" #include "../../utils/windowfunction/windowfunction.h" +#include "utils/common/branchpred.h" namespace execplan { @@ -482,6 +483,7 @@ void SimpleColumn::setDerivedTable() // @todo make aggregate filter to having clause. not optimize it for now if (fDerivedRefCol && + // WIP replace with typeid() (dynamic_cast(fDerivedRefCol) || dynamic_cast(fDerivedRefCol))) fDerivedTable = ""; @@ -500,6 +502,12 @@ bool SimpleColumn::singleTable(CalpontSystemCatalog::TableAliasName& tan) // @todo move to inline void SimpleColumn::evaluate(Row& row, bool& isNull) { + // WIP Move this block into an appropriate place + if (UNLIKELY(fInputOffset == -1)) + { + fInputOffset = row.getOffset(fInputIndex); + } + bool isNull2 = row.isNullValue(fInputIndex); if (isNull2) @@ -627,7 +635,16 @@ void SimpleColumn::evaluate(Row& row, bool& isNull) { case 16: { - fResult.decimalVal.value = row.getIntField<16>(fInputIndex); + if (fResultType.colDataType == CalpontSystemCatalog::UDECIMAL) + { + fResult.decimalVal.__v.__u128 = + *row.getBinaryField_offset(fInputOffset); + } + else + { + fResult.decimalVal.__v.__s128 = + *row.getBinaryField_offset(fInputOffset); + } fResult.decimalVal.scale = (unsigned)fResultType.scale; break; } diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index fcc92018d..760760a9d 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -36,6 +36,7 @@ #include "exceptclasses.h" #include "dataconvert.h" +using int128_t = __int128; using uint128_t = unsigned __int128; namespace messageqcpp @@ -63,11 +64,17 @@ typedef execplan::CalpontSystemCatalog::ColType Type; */ struct IDB_Decimal { - IDB_Decimal(): val(0), value(0), scale(0), precision(0) {} + IDB_Decimal(): value(0), scale(0), precision(0) + { + __v.__s128 = 0; + } IDB_Decimal(int64_t val, int8_t s, uint8_t p) : value (val), scale(s), - precision(p) {} + precision(p) + { + __v.__s128 = 0; + } int decimalComp(const IDB_Decimal& d) const { @@ -151,7 +158,11 @@ struct IDB_Decimal return (decimalComp(rhs) != 0); } - uint128_t val; + union { + uint128_t __u128; + int128_t __s128; + int64_t __s64[2]; + } __v; int64_t value; int8_t scale; // 0~38 uint8_t precision; // 1~38 diff --git a/primitives/primproc/batchprimitiveprocessor.cpp b/primitives/primproc/batchprimitiveprocessor.cpp index 3750c0cde..54fdcf482 100644 --- a/primitives/primproc/batchprimitiveprocessor.cpp +++ b/primitives/primproc/batchprimitiveprocessor.cpp @@ -1726,7 +1726,7 @@ void BatchPrimitiveProcessor::execute() if (fe2->evaluate(&fe2In)) { applyMapping(fe2Mapping, fe2In, &fe2Out); - //cerr << " passed. output row: " << fe2Out.toString() << endl; + cerr << " passed. output row: " << fe2Out.toString() << endl; fe2Out.setRid (fe2In.getRelRid()); fe2Output.incRowCount(); fe2Out.nextRow(); diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index 30d76d0f2..3b35f7c77 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -593,7 +593,7 @@ void ColumnCommand::prep(int8_t outputType, bool absRids) break; case 16: // WIP MCOL-641 - cout << __FILE__<< ":" <<__LINE__ << " Fix shift and mask for 16 Bytes ?"<< endl; + cout << __FILE__<< ":" <<__LINE__ << " Set shift and mask for 16 Bytes"<< endl; shift = 1; mask = 0x01; break; diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index f9510c311..96146a71a 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1198,7 +1198,7 @@ void DataConvert::toString(T* dec, char *p, size_t buflen) char* original_p = p; int printed_chars = 0; - // WIP replace snprintf with streams + // WIP replace snprintf with streams if (high_pod->lo != 0) { printed_chars = snprintf(p, div_log+1, "%lu", high_pod->lo); p += printed_chars; diff --git a/utils/funcexp/funcexp.cpp b/utils/funcexp/funcexp.cpp index 79039c923..cce824b47 100644 --- a/utils/funcexp/funcexp.cpp +++ b/utils/funcexp/funcexp.cpp @@ -472,10 +472,26 @@ void FuncExp::evaluate(rowgroup::Row& row, std::vector& expressi { IDB_Decimal val = expression[i]->getDecimalVal(row, isNull); - if (isNull) - row.setIntField<8>(BIGINTNULL, expression[i]->outputIndex()); + // WIP + if (expression[i]->resultType().precision > 18) + { + // WIP make this a separate function w and w/o overflow check + if (expression[i]->resultType().colDataType == execplan::CalpontSystemCatalog::DECIMAL) + row.setBinaryField(reinterpret_cast(&val.__v.__s128), + expression[i]->resultType().colWidth, + row.getOffset(expression[i]->outputIndex())); + else + row.setBinaryField(reinterpret_cast(&val.__v.__u128), + expression[i]->resultType().colWidth, + row.getOffset(expression[i]->outputIndex())); + } else - row.setIntField<8>(val.value, expression[i]->outputIndex()); + { + if (isNull) + row.setIntField<8>(BIGINTNULL, expression[i]->outputIndex()); + else + row.setIntField<8>(val.value, expression[i]->outputIndex()); + } break; } diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 9e55810d6..e756e1c5e 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -42,12 +42,15 @@ using namespace execplan; #include "nullvaluemanip.h" #include "rowgroup.h" +#include "dataconvert.h" #include "collation.h" namespace rowgroup { +using cscType = execplan::CalpontSystemCatalog::ColDataType; + StringStore::StringStore() : empty(true), fUseStoreStringMutex(false) { } StringStore::StringStore(const StringStore&) @@ -630,6 +633,19 @@ string Row::toString() const } case CalpontSystemCatalog::BINARY: std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; + break; + // WIP + case CalpontSystemCatalog::DECIMAL: + if (precision[i] > 18) + { + char *buf = (char*)alloca(precision[i] + 3); + // empty the buffer + dataconvert::DataConvert::toString(getBinaryField(i), + buf, precision[i]+3); + os << buf << " "; + break; + } + // fallback if the precision < 18 default: os << getIntField(i) << " "; break; @@ -866,6 +882,79 @@ void Row::initToNull() } } +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + ostringstream os; + os << "Row::isNullValue(): got bad column type at offset("; + os << offset; + os << "). Width="; + os << width << endl; + throw logic_error(os.str()); +} +/* +// WIP how to make if that enables explicit template for two cscTypes? +// Method template resolution could impose some perf degradation +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) == static_cast(joblist::BIGINTNULL)); +} + +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) == static_cast(joblist::BIGINTNULL)); +} + +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) == static_cast(joblist::BIGINTNULL)); +} + +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) == static_cast(joblist::BIGINTNULL)); +} + +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (data[offset] == joblist::TINYINTNULL); +} + +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (data[offset] == joblist::TINYINTNULL); +} + +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) == static_cast(joblist::SMALLINTNULL)); +} + +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) == static_cast(joblist::SMALLINTNULL)); +} + +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) == static_cast(joblist::INTNULL)); +} + +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) == static_cast(joblist::INTNULL)); +} +*/ bool Row::isNullValue(uint32_t colIndex) const { switch (types[colIndex]) @@ -1533,6 +1622,13 @@ void applyMapping(const int* mapping, const Row& in, Row* out) out->setUintField(in.getUintField(i), mapping[i]); else if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::LONGDOUBLE)) out->setLongDoubleField(in.getLongDoubleField(i), mapping[i]); + // WIP this doesn't look right b/c we can pushdown colType + // Migrate to offset based methods here + // code precision 2 width convertor + else if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::DECIMAL && + in.getColumnWidth(i) == 16)) + out->setBinaryField(in.getBinaryField(i), 16, + out->getOffset(mapping[i])); else if (in.isUnsigned(i)) out->setUintField(in.getUintField(i), mapping[i]); else diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 534b9b67a..ed02c6ee7 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -69,6 +69,7 @@ typedef const struct charset_info_st CHARSET_INFO; namespace rowgroup { +//using cscType = execplan::CalpontSystemCatalog::ColDataType; const int16_t rgCommonSize = 8192; /* @@ -447,6 +448,8 @@ public: uint64_t getNullValue(uint32_t colIndex) const; bool isNullValue(uint32_t colIndex) const; + template + inline bool isNullValue_offset(uint32_t offset) const; // when NULLs are pulled out via getIntField(), they come out with these values. // Ex: the 1-byte int null value is 0x80. When it gets cast to an int64_t From 0ff0472842dc8c0d4293f52f2d64b03aabb56745 Mon Sep 17 00:00:00 2001 From: drrtuy Date: Thu, 30 Jan 2020 17:27:36 +0300 Subject: [PATCH 10/78] MCOL-641 sum() now works with DECIMAL(38) columns. TupleAggregateStep class method and buildAggregateColumn() now properly set result data type. doSum() now handles DECIMAL(38) in approprate manner. Low-level null related methods for new binary-based datatypes now handles magic values for binary-based DT. --- dbcon/joblist/tupleaggregatestep.cpp | 31 +++-- dbcon/mysql/ha_mcs_execplan.cpp | 17 ++- .../primproc/batchprimitiveprocessor.cpp | 2 +- utils/rowgroup/rowaggregation.cpp | 115 ++++++++++++++---- utils/rowgroup/rowgroup.cpp | 16 ++- utils/rowgroup/rowgroup.h | 21 ++++ 6 files changed, 164 insertions(+), 38 deletions(-) diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index fd256f36b..789932ff7 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -3129,13 +3129,30 @@ void TupleAggregateStep::prep2PhasesAggregate( throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); } - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - typeAggPm.push_back(CalpontSystemCatalog::LONGDOUBLE); - csNumAggPm.push_back(8); - scaleAggPm.push_back(0); - precisionAggPm.push_back(-1); - widthAggPm.push_back(sizeof(long double)); + // WIP MCOL-641 Replace condition with a + // dynamic one + if (typeProj[colProj] == CalpontSystemCatalog::DECIMAL + && width[colProj] == 16) + { + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + typeAggPm.push_back(CalpontSystemCatalog::DECIMAL); + scaleAggPm.push_back(0); + // WIP makes this dynamic + precisionAggPm.push_back(38); + widthAggPm.push_back(width[colProj]); + csNumAggPm.push_back(8); + } + else + { + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); + typeAggPm.push_back(CalpontSystemCatalog::LONGDOUBLE); + scaleAggPm.push_back(0); + csNumAggPm.push_back(8); + precisionAggPm.push_back(-1); + widthAggPm.push_back(sizeof(long double)); + } colAggPm++; } diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 1e4557a71..bba566d7b 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -4589,7 +4589,6 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) Item_sum* isp = reinterpret_cast(item); Item** sfitempp = isp->get_orig_args(); -// Item** sfitempp = isp->arguments(); SRCP parm; // @bug4756 @@ -4809,7 +4808,6 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) if ((fc && fc->functionParms().empty()) || !fc) { - //ac->aggOp(AggregateColumn::COUNT_ASTERISK); ReturnedColumn* rc = buildReturnedColumn(sfitemp, gwi, gwi.fatalParseError); if (dynamic_cast(rc)) @@ -4896,6 +4894,7 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) // use the first parm for result type. parm = ac->aggParms()[0]; + // WIP why do we use LONGDOUBLE for AVG? if (isp->sum_func() == Item_sum::AVG_FUNC || isp->sum_func() == Item_sum::AVG_DISTINCT_FUNC) { @@ -4918,10 +4917,20 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) else if (isp->sum_func() == Item_sum::SUM_FUNC || isp->sum_func() == Item_sum::SUM_DISTINCT_FUNC) { - CalpontSystemCatalog::ColType ct = parm->resultType(); + // WIP MCOL-641 This fast hack breaks aggregates for + // all float DT's + // UPD it doesn't break b/c actual DT for result type + // is set during JobList creation. + /*CalpontSystemCatalog::ColType ct = parm->resultType(); ct.colDataType = CalpontSystemCatalog::LONGDOUBLE; ct.colWidth = sizeof(long double); - ct.precision = -1; + ct.precision = -1;*/ + CalpontSystemCatalog::ColType ct = parm->resultType(); + ct.colDataType = CalpontSystemCatalog::DECIMAL; + ct.colWidth = 16; + ct.precision = 38; + // WIP set the scale if argument is a float-based DT + ct.scale = 0; ac->resultType(ct); } else if (isp->sum_func() == Item_sum::STD_FUNC || diff --git a/primitives/primproc/batchprimitiveprocessor.cpp b/primitives/primproc/batchprimitiveprocessor.cpp index 54fdcf482..3750c0cde 100644 --- a/primitives/primproc/batchprimitiveprocessor.cpp +++ b/primitives/primproc/batchprimitiveprocessor.cpp @@ -1726,7 +1726,7 @@ void BatchPrimitiveProcessor::execute() if (fe2->evaluate(&fe2In)) { applyMapping(fe2Mapping, fe2In, &fe2Out); - cerr << " passed. output row: " << fe2Out.toString() << endl; + //cerr << " passed. output row: " << fe2Out.toString() << endl; fe2Out.setRid (fe2In.getRelRid()); fe2Output.incRowCount(); fe2Out.nextRow(); diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index c73e751af..7d62c4a78 100755 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -222,6 +222,11 @@ inline string getStringNullValue() return joblist::CPNULLSTRMARK; } +inline uint64_t getBinaryNullValue() +{ + return joblist::BINARYNULL; +} + } @@ -413,6 +418,7 @@ void RowAggregation::updateStringMinMax(string val1, string val2, int64_t col, i inline bool RowAggregation::isNull(const RowGroup* pRowGroup, const Row& row, int64_t col) { /* TODO: Can we replace all of this with a call to row.isNullValue(col)? */ + // WIP MCOL-641 Yes. We can bool ret = false; int colDataType = (pRowGroup->getColTypes())[col]; @@ -536,18 +542,7 @@ inline bool RowAggregation::isNull(const RowGroup* pRowGroup, const Row& row, in case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - int colWidth = pRowGroup->getColumnWidth(col); - int64_t val = row.getIntField(col); - - if (colWidth == 1) - ret = ((uint8_t)val == joblist::TINYINTNULL); - else if (colWidth == 2) - ret = ((uint16_t)val == joblist::SMALLINTNULL); - else if (colWidth == 4) - ret = ((uint32_t)val == joblist::INTNULL); - else - ret = ((uint64_t)val == joblist::BIGINTNULL); - + row.isNullValue(col); break; } @@ -1170,7 +1165,20 @@ void RowAggregation::makeAggFieldsNull(Row& row) case execplan::CalpontSystemCatalog::UDECIMAL: { int colWidth = fRowGroupOut->getColumnWidth(colOut); - row.setIntField(getUintNullValue(colDataType, colWidth), colOut); + if (colWidth <= 8) + { + row.setIntField(getUintNullValue(colDataType, colWidth), colOut); + } + else + { + // WIP This is only 1st part of the value + uint64_t nullValue = getBinaryNullValue(); + uint32_t offset = row.getOffset(colOut); + row.setBinaryField_offset(&nullValue, sizeof(nullValue), + offset); + row.setBinaryField_offset(&nullValue, sizeof(nullValue), + offset+sizeof(nullValue)); + } break; } @@ -1339,11 +1347,18 @@ void RowAggregation::doMinMax(const Row& rowIn, int64_t colIn, int64_t colOut, i // Note: NULL value check must be done on UM & PM // UM may receive NULL values, too. //------------------------------------------------------------------------------ +// WIP MCOL-641. This and other methods must be type based to avoid needless mem +// allocation for wide DTs void RowAggregation::doSum(const Row& rowIn, int64_t colIn, int64_t colOut, int funcType) { int colDataType = (fRowGroupIn.getColTypes())[colIn]; long double valIn = 0; - long double valOut = fRow.getLongDoubleField(colOut); + bool isWideDataType = false; + void *wideValInPtr = NULL; + // WIP MCOL-641 Probably the width must be taken + // from colOut + uint32_t width = fRowGroupOut->getColumnWidth(colOut); + if (isNull(&fRowGroupIn, rowIn, colIn) == true) return; @@ -1372,12 +1387,31 @@ void RowAggregation::doSum(const Row& rowIn, int64_t colIn, int64_t colOut, int case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - valIn = rowIn.getIntField(colIn); - double scale = (double)(fRowGroupIn.getScale())[colIn]; - if (valIn != 0 && scale > 0) + // WIP MCOL-641 make the size dynamic and use branch prediction cond + isWideDataType = (width) > 8 ? true : false; + if (!isWideDataType) { - valIn /= pow(10.0, scale); + valIn = rowIn.getIntField(colIn); + double scale = (double)(fRowGroupIn.getScale())[colIn]; + if (valIn != 0 && scale > 0) + { + valIn /= pow(10.0, scale); + } } + else + { + if (colDataType == execplan::CalpontSystemCatalog::DECIMAL) + { + int128_t *dec = rowIn.getBinaryField(colIn); + wideValInPtr = reinterpret_cast(dec); + } + else + { + uint128_t *dec = rowIn.getBinaryField(colIn); + wideValInPtr = reinterpret_cast(dec); + } + } + break; } @@ -1428,14 +1462,51 @@ void RowAggregation::doSum(const Row& rowIn, int64_t colIn, int64_t colOut, int break; } } - if (isNull(fRowGroupOut, fRow, colOut)) + // WIP MCOL-641 + if (!isWideDataType) { - fRow.setLongDoubleField(valIn, colOut); + if (isNull(fRowGroupOut, fRow, colOut)) + { + fRow.setLongDoubleField(valIn, colOut); + } + else + { + long double valOut = fRow.getLongDoubleField(colOut); + fRow.setLongDoubleField(valIn+valOut, colOut); + } } else { - fRow.setLongDoubleField(valIn+valOut, colOut); - } + if (colDataType == execplan::CalpontSystemCatalog::DECIMAL) + { + int128_t *dec = reinterpret_cast(wideValInPtr); + // WIP MCOL-641 Replace Row::setBinaryField1 + if (isNull(fRowGroupOut, fRow, colOut)) + { + fRow.setBinaryField1(dec, width, colOut); + } + else + { + int128_t *valOutPtr = fRow.getBinaryField(colOut); + int128_t sum = *valOutPtr + *dec; + fRow.setBinaryField1(&sum, width, colOut); + } + } + else + { + uint128_t *dec = reinterpret_cast(wideValInPtr); + if (isNull(fRowGroupOut, fRow, colOut)) + { + fRow.setBinaryField1(dec, width, colOut); + } + else + { + uint128_t *valOutPtr = fRow.getBinaryField(colOut); + uint128_t sum = *valOutPtr + *dec; + fRow.setBinaryField1(&sum, width, colOut); + } + } + } // end-of isWideDataType block } //------------------------------------------------------------------------------ diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index e756e1c5e..ec229d8b5 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -815,7 +815,6 @@ void Row::initToNull() default: *((uint64_t*) &data[offsets[i]]) = *((uint64_t*) joblist::CPNULLSTRMARK.c_str()); memset(&data[offsets[i] + 8], 0, len - 8); - //strcpy((char *) &data[offsets[i]], joblist::CPNULLSTRMARK.c_str()); break; } @@ -846,6 +845,13 @@ void Row::initToNull() *((int32_t*) &data[offsets[i]]) = static_cast(joblist::INTNULL); break; + case 16 : + // WIP MCOL-641 + uint64_t *dec = reinterpret_cast(&data[offsets[i]]); ++ dec[0] = joblist::BINARYNULL; ++ dec[1] = joblist::BINARYNULL; + break; + default: *((int64_t*) &data[offsets[i]]) = static_cast(joblist::BIGINTNULL); break; @@ -1039,13 +1045,15 @@ bool Row::isNullValue(uint32_t colIndex) const case CalpontSystemCatalog::UDECIMAL: { uint32_t len = getColumnWidth(colIndex); + const uint64_t *dec; switch (len) { - // MCOL-641 WIP + // MCOL-641 case 16: - return (*((int64_t*) &data[offsets[colIndex]]) == static_cast(joblist::BIGINTNULL)); - break; + dec = reinterpret_cast(&data[offsets[colIndex]]); + return ((dec[0] == joblist::BINARYNULL) + && (dec[1] == joblist::BINARYNULL)); case 1 : return (data[offsets[colIndex]] == joblist::TINYINTNULL); diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index ed02c6ee7..37f1ad3e4 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -66,6 +66,9 @@ typedef const struct charset_info_st CHARSET_INFO; // Workaround for my_global.h #define of isnan(X) causing a std::std namespace +using int128_t = __int128; +using uint128_t = unsigned __int128; + namespace rowgroup { @@ -424,6 +427,10 @@ public: void setStringField(const std::string& val, uint32_t colIndex); inline void setStringField(const uint8_t*, uint32_t len, uint32_t colIndex); inline void setBinaryField(const uint8_t* strdata, uint32_t length, uint32_t offset); + template + inline void setBinaryField1(T* strdata, uint32_t width, uint32_t colIndex); + template + inline void setBinaryField_offset(T* strdata, uint32_t width, uint32_t colIndex); // support VARBINARY // Add 2-byte length at the CHARSET_INFO*beginning of the field. NULL and zero length field are // treated the same, could use one of the length bit to distinguish these two cases. @@ -804,6 +811,20 @@ inline void Row::setBinaryField(const uint8_t* strdata, uint32_t length, uint32_ memcpy(&data[offset], strdata, length); } +template +inline void Row::setBinaryField1(T* value, uint32_t width, uint32_t colIndex) +{ + memcpy(&data[offsets[colIndex]], value, width); +} + +template +inline void Row::setBinaryField_offset(T* value, uint32_t width, uint32_t offset) +{ + // WIP + //memcpy(&data[offset], value, width); + *reinterpret_cast(&data[offset]) = *value; +} + inline void Row::setStringField(const uint8_t* strdata, uint32_t length, uint32_t colIndex) { uint64_t offset; From 84f9821720e9e08a81db20c97f7044ef9651c2cf Mon Sep 17 00:00:00 2001 From: drrtuy Date: Tue, 4 Feb 2020 23:02:39 +0300 Subject: [PATCH 11/78] MCOL-641 Switched to DataConvert static methods in joblist code. Replaced BINARYEMPTYROW and BINARYNULL values. We need to have separate magic values for numeric and non-numeric binary types b/c numeric cant tolerate losing 0 used for magics previously. atoi128() now parses minus sign and produces negative values. RowAggregation::isNull() now uses Row::isNull() for DECIMAL. --- dbcon/joblist/jlf_execplantojoblist.cpp | 18 +++------- dbcon/joblist/joblisttypes.h | 4 +-- utils/dataconvert/dataconvert.cpp | 46 ++++++++++++++++--------- utils/dataconvert/dataconvert.h | 21 +++++++---- utils/rowgroup/rowaggregation.cpp | 2 +- utils/rowgroup/rowgroup.cpp | 16 ++++----- 6 files changed, 59 insertions(+), 48 deletions(-) diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 0a226c38d..3460b0491 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -1601,18 +1601,6 @@ bool optimizeIdbPatitionSimpleFilter(SimpleFilter* sf, JobStepVector& jsv, JobIn return true; } - -// WIP MCOL-641 put this in dataconvert -void atoi128(const string& arg, unsigned __int128& res) -{ - res = 0; - for (size_t j = 0; j < arg.size(); j++) - { - if (LIKELY(arg[j]-'0' >= 0)) - res = res*10 + arg[j] - '0'; - } -} - const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) { JobStepVector jsv; @@ -1899,11 +1887,13 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) } #else - // WIP MCOL-641 + // WIP MCOL-641 width check must be a f() not a literal + // make a template from convertValueNum to avoid extra if + // this condition doesn't support UDECIMAL if (ct.colDataType == CalpontSystemCatalog::DECIMAL && ct.colWidth == 16) { - atoi128(constval, val128); + dataconvert::atoi128(constval, val128); } else { diff --git a/dbcon/joblist/joblisttypes.h b/dbcon/joblist/joblisttypes.h index 9ad0e071c..cfd3a72e8 100644 --- a/dbcon/joblist/joblisttypes.h +++ b/dbcon/joblist/joblisttypes.h @@ -83,8 +83,8 @@ const uint16_t NULL_UINT16 = USMALLINTNULL; const uint32_t NULL_UINT32 = UINTNULL; const uint64_t NULL_UINT64 = UBIGINTNULL; -const uint64_t BINARYEMPTYROW = 0; -const uint64_t BINARYNULL = 0; +const uint64_t BINARYEMPTYROW = UBIGINTEMPTYROW; +const uint64_t BINARYNULL = UBIGINTNULL; const std::string CPNULLSTRMARK("_CpNuLl_"); const std::string CPSTRNOTFOUND("_CpNoTf_"); diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 96146a71a..501a2e59a 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -51,6 +51,7 @@ typedef uint32_t ulong; using namespace logging; + namespace { @@ -1162,10 +1163,8 @@ bool stringToTimestampStruct(const string& data, TimeStamp& timeStamp, const str } -// WIP +// WIP MCOL-641 #include -using int128_t = __int128; -using uint128_t = unsigned __int128; struct uint128_pod { @@ -1192,6 +1191,7 @@ void DataConvert::toString(T* dec, char *p, size_t buflen) // WIP How to treat PODs here ? // use typeof // Or a templated structure + // Use uint64* to access parts of uint128 and remove pods uint128_pod *high_pod = reinterpret_cast(&high); uint128_pod *mid_pod = reinterpret_cast(&mid); uint128_pod *low_pod = reinterpret_cast(&low); @@ -1201,7 +1201,7 @@ void DataConvert::toString(T* dec, char *p, size_t buflen) // WIP replace snprintf with streams if (high_pod->lo != 0) { printed_chars = snprintf(p, div_log+1, "%lu", high_pod->lo); - p += printed_chars; + p += printed_chars; printed_chars = snprintf(p, div_log+1, "%019lu", mid_pod->lo); p += printed_chars; } else if (mid_pod->lo != 0) { @@ -1215,32 +1215,42 @@ void DataConvert::toString(T* dec, char *p, size_t buflen) // WIP MCOL-641 // Template this // result must be calloc-ed -//template -//void atoi_(const string &arg, T &res) -void atoi128(const string& arg, int128_t& res) +void atoi128(const std::string& arg, int128_t& res) { - // WIP - //char buf[41]; - //int128_t *res_ptr = reinterpret_cast(result); res = 0; - for (size_t j = 0; j < arg.size(); j++) + size_t idx = (arg[0] == '-') ? 1 : 0; + for (size_t j = idx; j < arg.size(); j++) { // WIP Optimize this if (LIKELY(arg[j]-'0' >= 0)) res = res*10 + arg[j] - '0'; } + // Use bit shift if possible + if (idx) + res *= -1; //toString(res, buf); //std::cerr << "atoi_ " << buf <= 0)) + res = res*10 + arg[j] - '0'; + } +} + // WIP MCOL-641 template void DataConvert::decimalToString(T* valuePtr, uint8_t scale, char* buf, unsigned int buflen, - execplan::CalpontSystemCatalog::ColDataType colDataType) + cscDataType colDataType) { toString(valuePtr, buf, buflen); @@ -1302,19 +1312,23 @@ void DataConvert::decimalToString(T* valuePtr, } // Explicit instantiation template -void DataConvert::decimalToString(int128_t* value, uint8_t scale, char* buf, unsigned int buflen, execplan::CalpontSystemCatalog::ColDataType colDataType); +void DataConvert::decimalToString(int128_t* value, uint8_t scale, + char* buf, unsigned int buflen, cscDataType colDataType); template -void DataConvert::decimalToString(uint128_t* value, uint8_t scale, char* buf, unsigned int buflen, execplan::CalpontSystemCatalog::ColDataType colDataType); +void DataConvert::decimalToString(uint128_t* value, uint8_t scale, + char* buf, unsigned int buflen, cscDataType colDataType); boost::any DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, - const std::string& dataOrig, bool& pushWarning, const std::string& timeZone, bool nulFlag, bool noRoundup, bool isUpdate) + const std::string& dataOrig, bool& pushWarning, + const std::string& timeZone, bool nulFlag, + bool noRoundup, bool isUpdate) { boost::any value; // WIP std::string data( dataOrig ); pushWarning = false; - CalpontSystemCatalog::ColDataType type = colType.colDataType; + cscDataType type = colType.colDataType; //if ( !data.empty() ) if (!nulFlag) diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 05cc27777..6ec934610 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -84,6 +84,7 @@ inline uint64_t uint64ToStr(uint64_t n) return htonll(n); } +using cscDataType = execplan::CalpontSystemCatalog::ColDataType; #if defined(_MSC_VER) && defined(xxxDATACONVERT_DLLEXPORT) #define EXPORT __declspec(dllexport) @@ -114,7 +115,6 @@ const int64_t IDB_pow[19] = 1000000000000000000LL }; - const int32_t SECS_PER_MIN = 60; const int32_t MINS_PER_HOUR = 60; const int32_t HOURS_PER_DAY = 24; @@ -134,6 +134,9 @@ const int32_t MIN_TIMESTAMP_VALUE = 0; namespace dataconvert { +using int128_t = __int128; +using uint128_t = unsigned __int128; + enum CalpontDateTimeFormat { CALPONTDATE_ENUM = 1, // date format is: "YYYY-MM-DD" @@ -157,6 +160,9 @@ struct MySQLTime } }; +void atoi128(const std::string& arg, int128_t& res); +void atoi128(const std::string& arg, uint128_t& res); + /** * This function converts the timezone represented as a string * in the format "+HH:MM" or "-HH:MM" to a signed offset in seconds @@ -1009,10 +1015,10 @@ public: EXPORT static bool isColumnTimeStampValid( int64_t timeStamp ); EXPORT static bool isNullData(execplan::ColumnResult* cr, int rownum, execplan::CalpontSystemCatalog::ColType colType); - static inline std::string decimalToString(int64_t value, uint8_t scale, execplan::CalpontSystemCatalog::ColDataType colDataType); - static inline void decimalToString(int64_t value, uint8_t scale, char* buf, unsigned int buflen, execplan::CalpontSystemCatalog::ColDataType colDataType); + static inline std::string decimalToString(int64_t value, uint8_t scale, cscDataType colDataType); + static inline void decimalToString(int64_t value, uint8_t scale, char* buf, unsigned int buflen, cscDataType colDataType); template - EXPORT static void decimalToString(T* value, uint8_t scale, char* buf, unsigned int buflen, execplan::CalpontSystemCatalog::ColDataType colDataType); + EXPORT static void decimalToString(T* value, uint8_t scale, char* buf, unsigned int buflen, cscDataType colDataType); template EXPORT static void toString(T* dec, char *p, size_t buflen); @@ -1226,15 +1232,16 @@ inline void DataConvert::timeToString1( long long timevalue, char* buf, unsigned #endif } -inline std::string DataConvert::decimalToString(int64_t value, uint8_t scale, execplan::CalpontSystemCatalog::ColDataType colDataType) +inline std::string DataConvert::decimalToString(int64_t value, uint8_t scale, cscDataType colDataType) { + // This is too much char buf[80]; DataConvert::decimalToString(value, scale, buf, 80, colDataType); return std::string(buf); } -inline void DataConvert::decimalToString(int64_t int_val, uint8_t scale, char* buf, unsigned int buflen, - execplan::CalpontSystemCatalog::ColDataType colDataType) +inline void DataConvert::decimalToString(int64_t int_val, uint8_t scale, + char* buf, unsigned int buflen, cscDataType colDataType) { // Need to convert a string with a binary unsigned number in it to a 64-bit signed int diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index 7d62c4a78..0176e87b5 100755 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -542,7 +542,7 @@ inline bool RowAggregation::isNull(const RowGroup* pRowGroup, const Row& row, in case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - row.isNullValue(col); + ret = row.isNullValue(col); break; } diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index ec229d8b5..e3be508ab 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -846,12 +846,12 @@ void Row::initToNull() break; case 16 : - // WIP MCOL-641 + { uint64_t *dec = reinterpret_cast(&data[offsets[i]]); -+ dec[0] = joblist::BINARYNULL; -+ dec[1] = joblist::BINARYNULL; + dec[0] = joblist::BINARYNULL; + dec[1] = joblist::BINARYNULL; break; - + } default: *((int64_t*) &data[offsets[i]]) = static_cast(joblist::BIGINTNULL); break; @@ -1045,15 +1045,15 @@ bool Row::isNullValue(uint32_t colIndex) const case CalpontSystemCatalog::UDECIMAL: { uint32_t len = getColumnWidth(colIndex); - const uint64_t *dec; + const int64_t *dec; switch (len) { // MCOL-641 case 16: - dec = reinterpret_cast(&data[offsets[colIndex]]); - return ((dec[0] == joblist::BINARYNULL) - && (dec[1] == joblist::BINARYNULL)); + dec = reinterpret_cast(&data[offsets[colIndex]]); + return ((dec[0] == static_cast(joblist::BINARYNULL)) + && (dec[1] == static_cast(joblist::BINARYNULL))); case 1 : return (data[offsets[colIndex]] == joblist::TINYINTNULL); From 55afcd88901d7e867d3fe8da746137377695c605 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Sat, 1 Feb 2020 22:16:58 -0500 Subject: [PATCH 12/78] MCOL-641 Basic extent elimination support for Decimal38. --- dbcon/ddlpackageproc/createtableprocessor.cpp | 4 + dbcon/dmlpackageproc/dmlpackageprocessor.cpp | 2 + dbcon/execplan/calpontsystemcatalog.h | 1 + dbcon/joblist/batchprimitiveprocessor-jl.cpp | 33 +- dbcon/joblist/batchprimitiveprocessor-jl.h | 6 +- dbcon/joblist/lbidlist.cpp | 162 ++++++++-- dbcon/joblist/lbidlist.h | 26 +- dbcon/joblist/primitivemsg.h | 4 +- dbcon/joblist/tuple-bps.cpp | 65 +++- dbcon/mysql/ha_mcs_partition.cpp | 120 +++++-- primitives/linux-port/column.cpp | 75 +++-- .../primproc/batchprimitiveprocessor.cpp | 55 +++- primitives/primproc/batchprimitiveprocessor.h | 20 +- primitives/primproc/columncommand.cpp | 19 +- tools/editem/editem.cpp | 51 ++- utils/dataconvert/dataconvert.cpp | 30 +- utils/dataconvert/dataconvert.h | 36 +++ versioning/BRM/brmtypes.h | 44 +++ versioning/BRM/dbrm.cpp | 50 ++- versioning/BRM/dbrm.h | 3 +- versioning/BRM/extentmap.cpp | 297 ++++++++++++++---- versioning/BRM/extentmap.h | 19 +- versioning/BRM/slavecomm.cpp | 27 +- versioning/BRM/slavedbrmnode.cpp | 26 +- writeengine/bulk/we_brmreporter.cpp | 1 + writeengine/bulk/we_bulkloadbuffer.cpp | 105 +++++-- writeengine/bulk/we_bulkloadbuffer.h | 15 + writeengine/bulk/we_colextinf.cpp | 86 +++-- writeengine/bulk/we_colextinf.h | 75 ++++- writeengine/bulk/we_columninfo.h | 17 +- writeengine/shared/we_brm.h | 2 +- writeengine/wrapper/we_colop.cpp | 72 ++++- writeengine/wrapper/writeengine.cpp | 95 ++++-- 33 files changed, 1318 insertions(+), 325 deletions(-) diff --git a/dbcon/ddlpackageproc/createtableprocessor.cpp b/dbcon/ddlpackageproc/createtableprocessor.cpp index a49dda249..e8f615c93 100644 --- a/dbcon/ddlpackageproc/createtableprocessor.cpp +++ b/dbcon/ddlpackageproc/createtableprocessor.cpp @@ -573,6 +573,10 @@ keepGoing: { colDefPtr->fType->fLength = 8; } + else if (colDefPtr->fType->fPrecision > 18 && colDefPtr->fType->fPrecision < 39) + { + colDefPtr->fType->fLength = 16; + } } bytestream << (fStartingColOID + (colNum++) + 1); diff --git a/dbcon/dmlpackageproc/dmlpackageprocessor.cpp b/dbcon/dmlpackageproc/dmlpackageprocessor.cpp index 32684dee5..19f8bc0ce 100644 --- a/dbcon/dmlpackageproc/dmlpackageprocessor.cpp +++ b/dbcon/dmlpackageproc/dmlpackageprocessor.cpp @@ -510,6 +510,8 @@ int DMLPackageProcessor::commitBatchAutoOnTransaction(uint64_t uniqueId, BRM::Tx aInfo.firstLbid = *iter; aInfo.max = numeric_limits::min(); // Not used aInfo.min = numeric_limits::max(); // Not used + dataconvert::DataConvert::int128Min(aInfo.bigMax); // Not used + dataconvert::DataConvert::int128Max(aInfo.bigMin); // Not used aInfo.seqNum = -1; cpInfos.push_back(aInfo); ++iter; diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index b89861b07..0f7b66aa2 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -1016,6 +1016,7 @@ inline bool isUnsigned(const execplan::CalpontSystemCatalog::ColDataType type) case execplan::CalpontSystemCatalog::UMEDINT: case execplan::CalpontSystemCatalog::UINT: case execplan::CalpontSystemCatalog::UBIGINT: + // TODO MCOL-641 add decimal here return true; default: diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.cpp b/dbcon/joblist/batchprimitiveprocessor-jl.cpp index 523e9a44a..28cd1b66b 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.cpp +++ b/dbcon/joblist/batchprimitiveprocessor-jl.cpp @@ -411,6 +411,7 @@ void BatchPrimitiveProcessorJL::addElementType(const StringElementType& et, uint * block touches */ +// TODO MCOL-641 Add support here. Refer to BatchPrimitiveProcessor::makeResponse() void BatchPrimitiveProcessorJL::getElementTypes(ByteStream& in, vector* out, bool* validCPData, uint64_t* lbid, int64_t* min, int64_t* max, uint32_t* cachedIO, uint32_t* physIO, @@ -491,6 +492,7 @@ void BatchPrimitiveProcessorJL::getElementTypes(ByteStream& in, * blocks touched */ +// TODO MCOL-641 Add support here. Refer to BatchPrimitiveProcessor::makeResponse() void BatchPrimitiveProcessorJL::getStringElementTypes(ByteStream& in, vector* out, bool* validCPData, uint64_t* lbid, int64_t* min, int64_t* max, uint32_t* cachedIO, uint32_t* physIO, uint32_t* touchedBlocks) const @@ -547,6 +549,7 @@ void BatchPrimitiveProcessorJL::getStringElementTypes(ByteStream& in, * as when the output type is TableBands */ +// TODO MCOL-641 Add support here. Refer to BatchPrimitiveProcessor::makeResponse() void BatchPrimitiveProcessorJL::getTuples(messageqcpp::ByteStream& in, std::vector* out, bool* validCPData, uint64_t* lbid, int64_t* min, int64_t* max, uint32_t* cachedIO, uint32_t* physIO, uint32_t* touchedBlocks) const @@ -692,9 +695,9 @@ bool BatchPrimitiveProcessorJL::countThisMsg(messageqcpp::ByteStream& in) const if (_hasScan) { if (data[offset] != 0) - offset += 25; // skip the CP data + offset += (data[offset + CP_FLAG_AND_LBID] * 2) + CP_FLAG_AND_LBID + 1; // skip the CP data with wide min/max values (16/32 bytes each) else - offset += 9; // skip only the "valid CP data" & LBID bytes + offset += CP_FLAG_AND_LBID; // skip only the "valid CP data" & LBID bytes } idbassert(in.length() > offset); @@ -718,11 +721,12 @@ void BatchPrimitiveProcessorJL::deserializeAggregateResult(ByteStream* in, } void BatchPrimitiveProcessorJL::getRowGroupData(ByteStream& in, vector* out, - bool* validCPData, uint64_t* lbid, int64_t* min, int64_t* max, + bool* validCPData, uint64_t* lbid, __int128* min, __int128* max, uint32_t* cachedIO, uint32_t* physIO, uint32_t* touchedBlocks, bool* countThis, - uint32_t threadID) const + uint32_t threadID, bool* hasBinaryColumn, const execplan::CalpontSystemCatalog::ColType& colType) const { uint64_t tmp64; + unsigned __int128 tmp128; uint8_t tmp8; RGData rgData; uint32_t rowCount; @@ -754,10 +758,23 @@ void BatchPrimitiveProcessorJL::getRowGroupData(ByteStream& in, vector* if (*validCPData) { in >> *lbid; - in >> tmp64; - *min = (int64_t) tmp64; - in >> tmp64; - *max = (int64_t) tmp64; + in >> tmp8; + *hasBinaryColumn = (tmp8 > 8); + if (*hasBinaryColumn) + { + idbassert(colType.colWidth > 8); + in >> tmp128; + *min = tmp128; + in >> tmp128; + *max = tmp128; + } + else + { + in >> tmp64; + *min = static_cast<__int128>(tmp64); + in >> tmp64; + *max = static_cast<__int128>(tmp64); + } } else in >> *lbid; diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.h b/dbcon/joblist/batchprimitiveprocessor-jl.h index a5f29a02b..b8b7e14c5 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.h +++ b/dbcon/joblist/batchprimitiveprocessor-jl.h @@ -49,6 +49,8 @@ namespace joblist { const uint32_t LOGICAL_BLOCKS_CONVERTER = 23; // 10 + 13. 13 to convert to logical blocks, // 10 to convert to groups of 1024 logical blocks +const uint8_t CP_FLAG_AND_LBID = 9; // # bytes used for CP boolean and the lbid + // used by BatchPrimitiveProcessorJL::countThisMsg() // forward reference struct JobInfo; @@ -166,9 +168,9 @@ public: void deserializeAggregateResults(messageqcpp::ByteStream* in, std::vector* out) const; void getRowGroupData(messageqcpp::ByteStream& in, std::vector* out, - bool* validCPData, uint64_t* lbid, int64_t* min, int64_t* max, + bool* validCPData, uint64_t* lbid, __int128* min, __int128* max, uint32_t* cachedIO, uint32_t* physIO, uint32_t* touchedBlocks, bool* countThis, - uint32_t threadID) const; + uint32_t threadID, bool* hasBinaryColumn, const execplan::CalpontSystemCatalog::ColType& colType) const; void deserializeAggregateResult(messageqcpp::ByteStream* in, std::vector* out) const; bool countThisMsg(messageqcpp::ByteStream& in) const; diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index b6cc575ae..d0f97f6bf 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -180,8 +180,8 @@ void LBIDList::Dump(long Index, int Count) const // Store max/min structure for update later. lbidPartitionVector serves a list of lbid,max,min to update // when the primitive returns with valid max/min values and the brm returns an invalid flag for the // requested lbid -// -bool LBIDList::GetMinMax(int64_t& min, int64_t& max, int64_t& seq, int64_t lbid, +template +bool LBIDList::GetMinMax(T& min, T& max, int64_t& seq, int64_t lbid, const std::vector* pEMEntries, execplan::CalpontSystemCatalog::ColDataType colDataType) { @@ -235,13 +235,29 @@ bool LBIDList::GetMinMax(int64_t& min, int64_t& max, int64_t& seq, int64_t lbid, if (isUnsigned(colDataType)) { - mmp->max = 0; - mmp->min = static_cast(numeric_limits::max()); + if (typeid(T) == typeid(__int128)) + { + mmp->bigMax = 0; + mmp->bigMin = -1; + } + else + { + mmp->max = 0; + mmp->min = static_cast(numeric_limits::max()); + } } else { - mmp->max = numeric_limits::min(); - mmp->min = numeric_limits::max(); + if (typeid(T) == typeid(__int128)) + { + dataconvert::DataConvert::int128Min(mmp->bigMax); + dataconvert::DataConvert::int128Max(mmp->bigMin); + } + else + { + mmp->max = numeric_limits::min(); + mmp->min = numeric_limits::max(); + } } mmp->isValid = retVal; @@ -257,6 +273,7 @@ bool LBIDList::GetMinMax(int64_t& min, int64_t& max, int64_t& seq, int64_t lbid, return false; } +//TODO MCOL-641 Do we need support here? bool LBIDList::GetMinMax(int64_t* min, int64_t* max, int64_t* seq, int64_t lbid, const tr1::unordered_map& entries, execplan::CalpontSystemCatalog::ColDataType colDataType) @@ -302,7 +319,8 @@ bool LBIDList::GetMinMax(int64_t* min, int64_t* max, int64_t* seq, // Get the min, max, and sequence number for the specified LBID by searching // the given vector of ExtentMap entries. -int LBIDList::getMinMaxFromEntries(int64_t& min, int64_t& max, int32_t& seq, +template +int LBIDList::getMinMaxFromEntries(T& min, T& max, int32_t& seq, int64_t lbid, const std::vector& EMEntries) { for (unsigned i = 0; i < EMEntries.size(); i++) @@ -311,9 +329,17 @@ int LBIDList::getMinMaxFromEntries(int64_t& min, int64_t& max, int32_t& seq, if (lbid >= EMEntries[i].range.start && lbid <= lastLBID) { - min = EMEntries[i].partition.cprange.lo_val; - max = EMEntries[i].partition.cprange.hi_val; - seq = EMEntries[i].partition.cprange.sequenceNum; + if (typeid(T) == typeid(__int128)) + { + min = EMEntries[i].partition.cprange.bigLoVal; + max = EMEntries[i].partition.cprange.bigHiVal; + } + else + { + min = EMEntries[i].partition.cprange.lo_val; + max = EMEntries[i].partition.cprange.hi_val; + } + seq = EMEntries[i].partition.cprange.sequenceNum; return EMEntries[i].partition.cprange.isValid; } } @@ -321,8 +347,8 @@ int LBIDList::getMinMaxFromEntries(int64_t& min, int64_t& max, int32_t& seq, return BRM::CP_INVALID; } -// -void LBIDList::UpdateMinMax(int64_t min, int64_t max, int64_t lbid, CalpontSystemCatalog::ColDataType type, +template +void LBIDList::UpdateMinMax(T min, T max, int64_t lbid, CalpontSystemCatalog::ColDataType type, bool validData) { MinMaxPartition* mmp = NULL; @@ -373,19 +399,41 @@ void LBIDList::UpdateMinMax(int64_t min, int64_t max, int64_t lbid, CalpontSyste } else if (execplan::isUnsigned(type)) { - if (static_cast(min) < static_cast(mmp->min)) - mmp->min = min; + if (typeid(T) == typeid(__int128)) + { + if (static_cast(min) < static_cast(mmp->bigMin)) + mmp->bigMin = min; - if (static_cast(max) > static_cast(mmp->max)) - mmp->max = max; + if (static_cast(max) > static_cast(mmp->bigMax)) + mmp->bigMax = max; + } + else + { + if (static_cast(min) < static_cast(mmp->min)) + mmp->min = min; + + if (static_cast(max) > static_cast(mmp->max)) + mmp->max = max; + } } else { - if (min < mmp->min) - mmp->min = min; + if (typeid(T) == typeid(__int128)) + { + if (min < mmp->bigMin) + mmp->bigMin = min; - if (max > mmp->max) - mmp->max = max; + if (max > mmp->bigMax) + mmp->bigMax = max; + } + else + { + if (min < mmp->min) + mmp->min = min; + + if (max > mmp->max) + mmp->max = max; + } } } @@ -405,7 +453,7 @@ void LBIDList::UpdateMinMax(int64_t min, int64_t max, int64_t lbid, CalpontSyste } } -void LBIDList::UpdateAllPartitionInfo() +void LBIDList::UpdateAllPartitionInfo(const execplan::CalpontSystemCatalog::ColType& colType) { MinMaxPartition* mmp = NULL; #ifdef DEBUG @@ -431,8 +479,17 @@ void LBIDList::UpdateAllPartitionInfo() if ((mmp->isValid == BRM::CP_INVALID) && (mmp->blksScanned > 0)) { cpInfo.firstLbid = mmp->lbid; - cpInfo.max = mmp->max; - cpInfo.min = mmp->min; + cpInfo.isBinaryColumn = (colType.colWidth == 16); + if (cpInfo.isBinaryColumn) + { + cpInfo.bigMax = mmp->bigMax; + cpInfo.bigMin = mmp->bigMin; + } + else + { + cpInfo.max = mmp->max; + cpInfo.min = mmp->min; + } cpInfo.seqNum = (int32_t)mmp->seq; vCpInfo.push_back(cpInfo); @@ -638,8 +695,7 @@ bool LBIDList::checkRangeOverlap(int64_t min, int64_t max, int64_t tmin, int64_t } } -bool LBIDList::CasualPartitionPredicate(const int64_t Min, - const int64_t Max, +bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, const messageqcpp::ByteStream* bs, const uint16_t NOPS, const execplan::CalpontSystemCatalog::ColType& ct, @@ -650,6 +706,9 @@ bool LBIDList::CasualPartitionPredicate(const int64_t Min, const char* MsgDataPtr = (const char*) bs->buf(); bool scan = true; int64_t value = 0; + __int128 bigValue = 0; + dataconvert::Int128Pod_t* bigValuePod; + bigValuePod = reinterpret_cast(&bigValue); bool bIsUnsigned = execplan::isUnsigned(ct.colDataType); bool bIsChar = execplan::isCharType(ct.colDataType); @@ -700,7 +759,13 @@ bool LBIDList::CasualPartitionPredicate(const int64_t Min, value = static_cast(val); } case 16: - cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + { + unsigned __int128 val; + bigValuePod = reinterpret_cast(&val); + bigValuePod->lo = *reinterpret_cast(MsgDataPtr); + bigValuePod->hi = *(reinterpret_cast(MsgDataPtr) + 1); + bigValue = static_cast<__int128>(val); + } } } else @@ -734,11 +799,15 @@ bool LBIDList::CasualPartitionPredicate(const int64_t Min, value = val; } case 16: - cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + { + bigValuePod->lo = *reinterpret_cast(MsgDataPtr); + bigValuePod->hi = *(reinterpret_cast(MsgDataPtr) + 1); + } } } // Should we also check for empty here? + // TODO MCOL-641 if (isNull(value, ct)) // This will work even if the data column is unsigned. { continue; @@ -750,8 +819,8 @@ bool LBIDList::CasualPartitionPredicate(const int64_t Min, { // MCOL-1246 Trim trailing whitespace for matching so that we have // the same as InnoDB behaviour - int64_t tMin = Min; - int64_t tMax = Max; + int64_t tMin = cpRange.lo_val; + int64_t tMax = cpRange.hi_val; dataconvert::DataConvert::trimWhitespace(tMin); dataconvert::DataConvert::trimWhitespace(tMax); @@ -761,11 +830,25 @@ bool LBIDList::CasualPartitionPredicate(const int64_t Min, } else if (bIsUnsigned) { - scan = compareVal(static_cast(Min), static_cast(Max), static_cast(value), op, lcf); + if (ct.colWidth <= 8) + { + scan = compareVal(static_cast(cpRange.lo_val), static_cast(cpRange.hi_val), static_cast(value), op, lcf); + } + else if (ct.colWidth == 16) + { + scan = compareVal(static_cast(cpRange.bigLoVal), static_cast(cpRange.bigHiVal), static_cast(bigValue), op, lcf); + } } else { - scan = compareVal(Min, Max, value, op, lcf); + if (ct.colWidth <= 8) + { + scan = compareVal(cpRange.lo_val, cpRange.hi_val, value, op, lcf); + } + else if (ct.colWidth == 16) + { + scan = compareVal(cpRange.bigLoVal, cpRange.bigHiVal, bigValue, op, lcf); + } } if (BOP == BOP_AND && !scan) @@ -820,6 +903,23 @@ void LBIDList::copyLbidList(const LBIDList& rhs) fDebug = rhs.fDebug; } +template +bool LBIDList::GetMinMax<__int128>(__int128& min, __int128& max, int64_t& seq, int64_t lbid, + const std::vector* pEMEntries, + execplan::CalpontSystemCatalog::ColDataType colDataType); +template +bool LBIDList::GetMinMax(int64_t& min, int64_t& max, int64_t& seq, int64_t lbid, + const std::vector* pEMEntries, + execplan::CalpontSystemCatalog::ColDataType colDataType); + +template +void LBIDList::UpdateMinMax<__int128>(__int128 min, __int128 max, int64_t lbid, + execplan::CalpontSystemCatalog::ColDataType type, bool validData = true); + +template +void LBIDList::UpdateMinMax(int64_t min, int64_t max, int64_t lbid, + execplan::CalpontSystemCatalog::ColDataType type, bool validData = true); + } //namespace joblist // vim:ts=4 sw=4: diff --git a/dbcon/joblist/lbidlist.h b/dbcon/joblist/lbidlist.h index 56b26315a..0b2fccca6 100644 --- a/dbcon/joblist/lbidlist.h +++ b/dbcon/joblist/lbidlist.h @@ -50,11 +50,19 @@ struct MinMaxPartition { int64_t lbid; int64_t lbidmax; - int64_t min; - int64_t max; int64_t seq; int isValid; uint32_t blksScanned; + union + { + __int128 bigMin; + int64_t min; + }; + union + { + __int128 bigMax; + int64_t max; + }; }; /** @brief class LBIDList @@ -84,7 +92,8 @@ public: // If pEMEntries is provided, then min/max will be extracted from that // vector, else extents in BRM will be searched. If type is unsigned, caller // should static cast returned min and max to uint64_t - bool GetMinMax(int64_t& min, int64_t& max, int64_t& seq, int64_t lbid, + template + bool GetMinMax(T& min, T& max, int64_t& seq, int64_t lbid, const std::vector* pEMEntries, execplan::CalpontSystemCatalog::ColDataType type); @@ -92,15 +101,15 @@ public: const std::tr1::unordered_map& entries, execplan::CalpontSystemCatalog::ColDataType type); - void UpdateMinMax(int64_t min, int64_t max, int64_t lbid, + template + void UpdateMinMax(T min, T max, int64_t lbid, execplan::CalpontSystemCatalog::ColDataType type, bool validData = true); - void UpdateAllPartitionInfo(); + void UpdateAllPartitionInfo(const execplan::CalpontSystemCatalog::ColType& colType); bool IsRangeBoundary(uint64_t lbid); - bool CasualPartitionPredicate(const int64_t Min, - const int64_t Max, + bool CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, const messageqcpp::ByteStream* MsgDataPtr, const uint16_t NOPS, const execplan::CalpontSystemCatalog::ColType& ct, @@ -135,7 +144,8 @@ private: template inline bool compareVal(const T& Min, const T& Max, const T& value, char op, uint8_t lcf); - int getMinMaxFromEntries(int64_t& min, int64_t& max, int32_t& seq, + template + int getMinMaxFromEntries(T& min, T& max, int32_t& seq, int64_t lbid, const std::vector& EMEntries); boost::shared_ptr em; diff --git a/dbcon/joblist/primitivemsg.h b/dbcon/joblist/primitivemsg.h index f2cad3e76..a75367978 100644 --- a/dbcon/joblist/primitivemsg.h +++ b/dbcon/joblist/primitivemsg.h @@ -704,8 +704,8 @@ struct NewColResultHeader uint16_t NVALS; uint16_t ValidMinMax; // 1 if Min/Max are valid, otherwise 0 uint32_t OutputType; - int64_t Min; // Minimum value in this block for signed data types - int64_t Max; // Maximum value in this block for signed data types + __int128 Min; // Minimum value in this block for signed data types + __int128 Max; // Maximum value in this block for signed data types uint32_t CacheIO; // I/O count from buffer cache uint32_t PhysicalIO; // Physical I/O count from disk // if OutputType was OT_DATAVALUE, what follows is DataType[NVALS] diff --git a/dbcon/joblist/tuple-bps.cpp b/dbcon/joblist/tuple-bps.cpp index 02b479886..4c65f2dc3 100644 --- a/dbcon/joblist/tuple-bps.cpp +++ b/dbcon/joblist/tuple-bps.cpp @@ -838,8 +838,7 @@ void TupleBPS::storeCasualPartitionInfo(const bool estimateRowCounts) scanFlags[idx] = scanFlags[idx] && (ignoreCP || extent.partition.cprange.isValid != BRM::CP_VALID || lbidListVec[i]->CasualPartitionPredicate( - extent.partition.cprange.lo_val, - extent.partition.cprange.hi_val, + extent.partition.cprange, &(colCmd->getFilterString()), colCmd->getFilterCount(), colCmd->getColType(), @@ -928,7 +927,8 @@ void TupleBPS::prepCasualPartitioning() { uint32_t i; int64_t min, max, seq; - boost::mutex::scoped_lock lk(cpMutex); + __int128 bigMin, bigMax; + boost::mutex::scoped_lock lk(cpMutex); for (i = 0; i < scannedExtents.size(); i++) { @@ -938,8 +938,18 @@ void TupleBPS::prepCasualPartitioning() if (scanFlags[i] && lbidList->CasualPartitionDataType(fColType.colDataType, fColType.colWidth)) - lbidList->GetMinMax(min, max, seq, (int64_t) scannedExtents[i].range.start, - &scannedExtents, fColType.colDataType); + { + if (fColType.colWidth <= 8) + { + lbidList->GetMinMax(min, max, seq, (int64_t) scannedExtents[i].range.start, + &scannedExtents, fColType.colDataType); + } + else if (fColType.colWidth == 16) + { + lbidList->GetMinMax(bigMin, bigMax, seq, (int64_t) scannedExtents[i].range.start, + &scannedExtents, fColType.colDataType); + } + } } else scanFlags[i] = true; @@ -1859,8 +1869,17 @@ abort: struct _CPInfo { _CPInfo(int64_t MIN, int64_t MAX, uint64_t l, bool val) : min(MIN), max(MAX), LBID(l), valid(val) { }; - int64_t min; - int64_t max; + _CPInfo(__int128 BIGMIN, __int128 BIGMAX, uint64_t l, bool val) : bigMin(BIGMIN), bigMax(BIGMAX), LBID(l), valid(val) { }; + union + { + __int128 bigMin; + int64_t min; + }; + union + { + __int128 bigMax; + int64_t max; + }; uint64_t LBID; bool valid; }; @@ -1878,8 +1897,9 @@ void TupleBPS::receiveMultiPrimitiveMessages(uint32_t threadID) vector fromPrimProc; bool validCPData; - int64_t min; - int64_t max; + bool hasBinaryColumn; + __int128 min; + __int128 max; uint64_t lbid; vector<_CPInfo> cpv; uint32_t cachedIO; @@ -2154,7 +2174,7 @@ void TupleBPS::receiveMultiPrimitiveMessages(uint32_t threadID) fromPrimProc.clear(); fBPP->getRowGroupData(*bs, &fromPrimProc, &validCPData, &lbid, &min, &max, - &cachedIO, &physIO, &touchedBlocks, &unused, threadID); + &cachedIO, &physIO, &touchedBlocks, &unused, threadID, &hasBinaryColumn, fColType); /* Another layer of messiness. Need to refactor this fcn. */ while (!fromPrimProc.empty() && !cancelled()) @@ -2318,7 +2338,16 @@ void TupleBPS::receiveMultiPrimitiveMessages(uint32_t threadID) touchedBlocks_Thread += touchedBlocks; if (fOid >= 3000 && ffirstStepType == SCAN && bop == BOP_AND) - cpv.push_back(_CPInfo(min, max, lbid, validCPData)); + { + if (fColType.colWidth <= 8) + { + cpv.push_back(_CPInfo((int64_t) min, (int64_t) max, lbid, validCPData)); + } + else if (fColType.colWidth == 16) + { + cpv.push_back(_CPInfo(min, max, lbid, validCPData)); + } + } } // end of the per-rowgroup processing loop // insert the resulting rowgroup data from a single bytestream into dlp @@ -2344,8 +2373,16 @@ void TupleBPS::receiveMultiPrimitiveMessages(uint32_t threadID) for (i = 0; i < size; i++) { - lbidList->UpdateMinMax(cpv[i].min, cpv[i].max, cpv[i].LBID, fColType.colDataType, - cpv[i].valid); + if (fColType.colWidth > 8) + { + lbidList->UpdateMinMax(cpv[i].bigMin, cpv[i].bigMax, cpv[i].LBID, fColType.colDataType, + cpv[i].valid); + } + else + { + lbidList->UpdateMinMax(cpv[i].min, cpv[i].max, cpv[i].LBID, fColType.colDataType, + cpv[i].valid); + } } cpMutex.unlock(); @@ -2619,7 +2656,7 @@ out: if (ffirstStepType == SCAN && bop == BOP_AND && !cancelled()) { cpMutex.lock(); - lbidList->UpdateAllPartitionInfo(); + lbidList->UpdateAllPartitionInfo(fColType); cpMutex.unlock(); } } diff --git a/dbcon/mysql/ha_mcs_partition.cpp b/dbcon/mysql/ha_mcs_partition.cpp index 47dff51f1..edf357df2 100644 --- a/dbcon/mysql/ha_mcs_partition.cpp +++ b/dbcon/mysql/ha_mcs_partition.cpp @@ -235,15 +235,30 @@ struct PartitionInfo { int64_t min; int64_t max; + union + { + __int128 bigMin; + int64_t min_; + }; + union + { + __int128 bigMax; + int64_t max_; + }; uint64_t status; PartitionInfo(): min((uint64_t)0x8000000000000001ULL), max((uint64_t) - 0x8000000000000001LL), - status(0) {}; + status(0) + { + DataConvert::int128Min(bigMin); + DataConvert::int128Max(bigMax); + }; }; typedef map PartitionMap; -const string format(int64_t v, CalpontSystemCatalog::ColType& ct) +template +const string format(T v, CalpontSystemCatalog::ColType& ct) { ostringstream oss; @@ -282,14 +297,23 @@ const string format(int64_t v, CalpontSystemCatalog::ColType& ct) case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - if (ct.scale > 0) + if (ct.colWidth <= 8) { - double d = ((double)(v) / (double)pow((double)10, ct.scale)); - oss << setprecision(ct.scale) << fixed << d; + if (ct.scale > 0) + { + double d = ((double)(v) / (double)pow((double)10, ct.scale)); + oss << setprecision(ct.scale) << fixed << d; + } + else + { + oss << (int64_t) v; + } } else { - oss << v; + char buf[MAX_DECIMAL_STRING_LENGTH]; + DataConvert::decimalToString((__int128*)&v, (unsigned)ct.scale, buf, sizeof(buf), ct.colDataType); + oss << buf; } break; @@ -308,7 +332,7 @@ const string format(int64_t v, CalpontSystemCatalog::ColType& ct) break; default: - oss << v; + oss << (int64_t) v; break; } @@ -1075,7 +1099,13 @@ extern "C" partInfo.status |= ET_DISABLED; mapit = partMap.find(logicalPartNum); - int state = em.getExtentMaxMin(iter->range.start, partInfo.max, partInfo.min, seqNum); + + int state; + + if (ct.colWidth <= 8) + state = em.getExtentMaxMin(iter->range.start, partInfo.max, partInfo.min, seqNum); + else if (ct.colWidth == 16) + state = em.getExtentMaxMin(iter->range.start, partInfo.bigMax, partInfo.bigMin, seqNum); // char column order swap for compare if ((ct.colDataType == CalpontSystemCatalog::CHAR && ct.colWidth <= 8) || @@ -1097,8 +1127,16 @@ extern "C" if (mapit->second.status & CPINVALID) continue; - mapit->second.min = (partInfo.min < mapit->second.min ? partInfo.min : mapit->second.min); - mapit->second.max = (partInfo.max > mapit->second.max ? partInfo.max : mapit->second.max); + if (ct.colWidth <= 8) + { + mapit->second.min = (partInfo.min < mapit->second.min ? partInfo.min : mapit->second.min); + mapit->second.max = (partInfo.max > mapit->second.max ? partInfo.max : mapit->second.max); + } + else if (ct.colWidth == 16) + { + mapit->second.bigMin = (partInfo.bigMin < mapit->second.bigMin ? partInfo.bigMin : mapit->second.bigMin); + mapit->second.bigMax = (partInfo.bigMax > mapit->second.bigMax ? partInfo.bigMax : mapit->second.bigMax); + } } } } @@ -1118,13 +1156,29 @@ extern "C" ostringstream output; output.setf(ios::left, ios::adjustfield); - output << setw(10) << "Part#" - << setw(30) << "Min" - << setw(30) << "Max" << "Status"; + if (ct.colWidth <= 8) + { + output << setw(10) << "Part#" + << setw(30) << "Min" + << setw(30) << "Max" << "Status"; + } + else + { + output << setw(10) << "Part#" + << setw(40) << "Min" + << setw(40) << "Max" << "Status"; + } int64_t maxLimit = numeric_limits::max(); int64_t minLimit = numeric_limits::min(); + __int128 bigMaxLimit, bigMinLimit; + DataConvert::int128Max(bigMaxLimit); + DataConvert::int128Min(bigMinLimit); + unsigned __int128 ubigMaxLimit, ubigMinLimit; + DataConvert::uint128Max(ubigMaxLimit); + ubigMinLimit = 0; + // char column order swap for compare in subsequent loop if ((ct.colDataType == CalpontSystemCatalog::CHAR && ct.colWidth <= 8) || (ct.colDataType == CalpontSystemCatalog::VARCHAR && ct.colWidth <= 7)) @@ -1143,24 +1197,48 @@ extern "C" if (partIt->second.status & CPINVALID) { - output << setw(30) << "N/A" << setw(30) << "N/A"; + if (ct.colWidth <= 8) + output << setw(30) << "N/A" << setw(30) << "N/A"; + else + output << setw(40) << "N/A" << setw(40) << "N/A"; } else { if ((isUnsigned(ct.colDataType))) { - if (static_cast(partIt->second.min) == numeric_limits::max() - && static_cast(partIt->second.max) == numeric_limits::min()) - output << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; + if (ct.colWidth <= 8) + { + if (static_cast(partIt->second.min) == numeric_limits::max() + && static_cast(partIt->second.max) == numeric_limits::min()) + output << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; + else + output << setw(30) << format(partIt->second.min, ct) << setw(30) << format(partIt->second.max, ct); + } else - output << setw(30) << format(partIt->second.min, ct) << setw(30) << format(partIt->second.max, ct); + { + if (static_cast(partIt->second.bigMin) == ubigMaxLimit + && static_cast(partIt->second.bigMax) == ubigMinLimit) + output << setw(40) << "Empty/Null" << setw(40) << "Empty/Null"; + else + output << setw(40) << format(partIt->second.bigMin, ct) << setw(40) << format(partIt->second.bigMax, ct); + } } else { - if (partIt->second.min == maxLimit && partIt->second.max == minLimit) - output << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; + if (ct.colWidth <= 8) + { + if (partIt->second.min == maxLimit && partIt->second.max == minLimit) + output << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; + else + output << setw(30) << format(partIt->second.min, ct) << setw(30) << format(partIt->second.max, ct); + } else - output << setw(30) << format(partIt->second.min, ct) << setw(30) << format(partIt->second.max, ct); + { + if (partIt->second.bigMin == bigMaxLimit && partIt->second.bigMax == bigMinLimit) + output << setw(40) << "Empty/Null" << setw(40) << "Empty/Null"; + else + output << setw(40) << format(partIt->second.bigMin, ct) << setw(40) << format(partIt->second.bigMax, ct); + } } } diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 4f3cea781..4d19940e5 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -616,7 +616,7 @@ inline bool isMinMaxValid(const NewColRequestHeader* in) case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - return (in->DataSize <= 8); + return (in->DataSize <= 16); default: return false; @@ -906,8 +906,8 @@ inline uint8_t* nextBinColValue(int type, *rid = ridArray[(*index)++]; } - *isNull = isNullVal(type, val8); - *isEmpty = isEmptyVal(type, val8); + *isNull = isNullVal(type, &val8[*rid * W]); + *isEmpty = isEmptyVal(type, &val8[*rid * W]); //cout << "nextUnsignedColValue index " << *index << " rowid " << *rid << endl; // at this point, nextRid is the index to return, and index is... // if RIDs are not specified, nextRid + 1, @@ -1540,11 +1540,6 @@ inline void p_Col_ridArray(NewColRequestHeader* in, using uint128_t = unsigned __int128; using int128_t = __int128; -struct uint128_pod { - uint64_t lo; - uint64_t hi; -}; - // for BINARY template inline void p_Col_bin_ridArray(NewColRequestHeader* in, @@ -1560,9 +1555,9 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, placeholderRegex.used = false; //FIXME: pCol is setting it to 8192 cause logicalBlockMode is true - if(itemsPerBlk == BLOCK_SIZE){ + /*if(itemsPerBlk == BLOCK_SIZE){ itemsPerBlk = BLOCK_SIZE/W; - } + }*/ if (in->NVALS > 0) ridArray = reinterpret_cast(&in8[sizeof(NewColRequestHeader) + @@ -1588,13 +1583,13 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, { if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType)) { - out->Min = static_cast(numeric_limits::max()); + out->Min = -1; out->Max = 0; } else { - out->Min = numeric_limits::max(); - out->Max = numeric_limits::min(); + dataconvert::DataConvert::int128Max(out->Min); + dataconvert::DataConvert::int128Min(out->Max); } } else @@ -1606,7 +1601,6 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, typedef char binWtype [W]; const ColArgs* args = NULL; - int64_t val = 0; binWtype* bval; int nextRidIndex = 0, argIndex = 0; bool done = false, cmp = false, isNull = false, isEmpty = false; @@ -1655,16 +1649,14 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, bval = (binWtype*)nextBinColValue(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull, &isEmpty, &rid, in->OutputType, reinterpret_cast(block), itemsPerBlk); + uint128_t val; + dataconvert::Int128Pod_t *valPod; + valPod = reinterpret_cast(&val); + while (!done) { -// if((*((uint64_t *) (bval))) != 0) -// { -// cout << "rid "<< rid << " value "; -// if(W > 16) printf("%016X%016X ",( *(((uint64_t *) (bval)) +3)),(*(((uint64_t *) (bval)) +2))); -// printf("%016X%016X ",( *(((uint64_t *) (bval)) +1)),(*((uint64_t *) (bval))) ); -// -// cout << endl; -// } + valPod->lo = *reinterpret_cast(*bval); + valPod->hi = *(reinterpret_cast(*bval) + 1); if (cops == NULL) // implies parsedColumnFilter && columnFilterMode == SET { @@ -1701,13 +1693,9 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, // if((*((uint64_t *) (uval))) != 0) cout << "comparing " << dec << (*((uint64_t *) (uval))) << " to " << (*((uint64_t *) (argVals[argIndex]))) << endl; // WIP MCOL-641 - uint128_t val, filterVal; - uint128_pod *valPod, *filterValPod; - valPod = reinterpret_cast(&val); - filterValPod = reinterpret_cast(&filterVal); - - valPod->lo = *reinterpret_cast(*bval); - valPod->hi = *(reinterpret_cast(*bval) + 1); + uint128_t filterVal; + dataconvert::Int128Pod_t *filterValPod; + filterValPod = reinterpret_cast(&filterVal); filterValPod->lo = *reinterpret_cast(argVals[argIndex]); filterValPod->hi = *(reinterpret_cast(argVals[argIndex]) + 1); @@ -1767,6 +1755,35 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, } } + // Set the min and max if necessary. Ignore nulls. + if (out->ValidMinMax && !isNull && !isEmpty) + { + if (in->DataType == CalpontSystemCatalog::CHAR || in->DataType == CalpontSystemCatalog::VARCHAR) + { + if (colCompare(out->Min, val, COMPARE_GT, false, in->DataType, W, placeholderRegex)) + out->Min = val; + + if (colCompare(out->Max, val, COMPARE_LT, false, in->DataType, W, placeholderRegex)) + out->Max = val; + } + else if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType)) + { + if (static_cast(out->Min) > val) + out->Min = static_cast(val); + + if (static_cast(out->Max) < val) + out->Max = static_cast(val);; + } + else + { + if (out->Min > (__int128) val) + out->Min = (__int128) val; + + if (out->Max < (__int128) val) + out->Max = (__int128) val; + } + } + bval = (binWtype*)nextBinColValue(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull, &isEmpty, &rid, in->OutputType, reinterpret_cast(block), itemsPerBlk); diff --git a/primitives/primproc/batchprimitiveprocessor.cpp b/primitives/primproc/batchprimitiveprocessor.cpp index 3750c0cde..99bda331a 100644 --- a/primitives/primproc/batchprimitiveprocessor.cpp +++ b/primitives/primproc/batchprimitiveprocessor.cpp @@ -111,6 +111,7 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor() : minVal(MAX64), maxVal(MIN64), lbidForCP(0), + hasBinaryColumn(false), busyLoaderCount(0), physIO(0), cachedIO(0), @@ -155,6 +156,7 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor(ByteStream& b, double prefetch, minVal(MAX64), maxVal(MIN64), lbidForCP(0), + hasBinaryColumn(false), busyLoaderCount(0), physIO(0), cachedIO(0), @@ -574,6 +576,7 @@ void BatchPrimitiveProcessor::addToJoiner(ByteStream& bs) /* skip the header */ bs.advance(sizeof(ISMPacketHeader) + 3 * sizeof(uint32_t)); +// TODO MCOL-641 bs >> count; bs >> startPos; @@ -1082,8 +1085,16 @@ void BatchPrimitiveProcessor::initProcessor() fAggregator->setInputOutput(fe2 ? fe2Output : outputRG, &fAggregateRG); } - minVal = MAX64; - maxVal = MIN64; + if (!hasBinaryColumn) + { + minVal = MAX64; + maxVal = MIN64; + } + else + { + dataconvert::DataConvert::int128Min(bigMaxVal); + dataconvert::DataConvert::int128Max(bigMinVal); + } // @bug 1269, initialize data used by execute() for async loading blocks // +1 for the scan filter step with no predicate, if any @@ -1970,8 +1981,18 @@ void BatchPrimitiveProcessor::writeProjectionPreamble() { *serialized << (uint8_t) 1; *serialized << lbidForCP; - *serialized << (uint64_t) minVal; - *serialized << (uint64_t) maxVal; + if (hasBinaryColumn) + { + *serialized << (uint8_t) 16; // width of min/max value + *serialized << (unsigned __int128) bigMinVal; + *serialized << (unsigned __int128) bigMaxVal; + } + else + { + *serialized << (uint8_t) 8; // width of min/max value + *serialized << (uint64_t) minVal; + *serialized << (uint64_t) maxVal; + } } else { @@ -2060,8 +2081,18 @@ void BatchPrimitiveProcessor::makeResponse() { *serialized << (uint8_t) 1; *serialized << lbidForCP; - *serialized << (uint64_t) minVal; - *serialized << (uint64_t) maxVal; + if (hasBinaryColumn) + { + *serialized << (uint8_t) 16; // width of min/max value + *serialized << (unsigned __int128) bigMinVal; + *serialized << (unsigned __int128) bigMaxVal; + } + else + { + *serialized << (uint8_t) 8; // width of min/max value + *serialized << (uint64_t) minVal; + *serialized << (uint64_t) maxVal; + } } else { @@ -2164,8 +2195,16 @@ int BatchPrimitiveProcessor::operator()() } allocLargeBuffers(); - minVal = MAX64; - maxVal = MIN64; + if (!hasBinaryColumn) + { + minVal = MAX64; + maxVal = MIN64; + } + else + { + dataconvert::DataConvert::int128Min(bigMaxVal); + dataconvert::DataConvert::int128Max(bigMinVal); + } validCPData = false; #ifdef PRIMPROC_STOPWATCH stopwatch->start("BPP() execute"); diff --git a/primitives/primproc/batchprimitiveprocessor.h b/primitives/primproc/batchprimitiveprocessor.h index d56481a5f..e8a9b9384 100644 --- a/primitives/primproc/batchprimitiveprocessor.h +++ b/primitives/primproc/batchprimitiveprocessor.h @@ -211,8 +211,8 @@ private: bool needStrValues; /* Common space for primitive data */ - static const uint32_t BUFFER_SIZE = 65536; - uint8_t blockData[BLOCK_SIZE * 8]; + static const uint32_t BUFFER_SIZE = 131072; + uint8_t blockData[BLOCK_SIZE * 16]; boost::scoped_array outputMsg; uint32_t outMsgSize; @@ -228,9 +228,21 @@ private: bool hasScan; bool validCPData; - int64_t minVal, maxVal; // CP data from a scanned column - uint64_t lbidForCP; + // CP data from a scanned column + union + { + __int128 bigMinVal; + int64_t minVal; + }; + union + { + __int128 bigMaxVal; + int64_t maxVal; + }; + uint64_t lbidForCP; + // MCOL-641 + bool hasBinaryColumn; // IO counters boost::mutex counterLock; uint32_t busyLoaderCount; diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index 3b35f7c77..ac33e47a3 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -206,6 +206,12 @@ void ColumnCommand::loadData() ByteStream::octbyte o = getEmptyRowValue(colType.colDataType, colType.colWidth); oPtr[idx] = o; } + else if (colType.colWidth == 16) + { + uint64_t *ptr = reinterpret_cast(&bpp->blockData[i * BLOCK_SIZE] + (idx*16) ); + *ptr = joblist::BINARYEMPTYROW; + *(ptr + 1) = joblist::BINARYEMPTYROW; + } } }// else @@ -256,8 +262,17 @@ void ColumnCommand::issuePrimitive() //if (wasVersioned && outMsg->ValidMinMax) // cout << "CC: versioning overriding min max data\n"; bpp->lbidForCP = lbid; - bpp->maxVal = outMsg->Max; - bpp->minVal = outMsg->Min; + if (primMsg->DataSize > 8) + { + bpp->hasBinaryColumn = true; + bpp->bigMaxVal = outMsg->Max; + bpp->bigMinVal = outMsg->Min; + } + else + { + bpp->maxVal = static_cast(outMsg->Max); + bpp->minVal = static_cast(outMsg->Min); + } } } // issuePrimitive() diff --git a/tools/editem/editem.cpp b/tools/editem/editem.cpp index 15d4d2a5f..f2b392dad 100644 --- a/tools/editem/editem.cpp +++ b/tools/editem/editem.cpp @@ -408,11 +408,31 @@ int clearAllCPData() if (err == 0 && ranges.size() > 0) { + // Get the extents for a given OID to determine it's column width + std::vector entries; + CHECK(emp->getExtents(oid, entries, false, false, true)); + + if (entries.empty()) + continue; + + bool isBinaryColumn = entries[0].colWid > 8; + BRM::CPInfo cpInfo; BRM::CPInfoList_t vCpInfo; - cpInfo.max = numeric_limits::min(); - cpInfo.min = numeric_limits::max(); + + if (!isBinaryColumn) + { + cpInfo.max = numeric_limits::min(); + cpInfo.min = numeric_limits::max(); + } + else + { + dataconvert::DataConvert::int128Min(cpInfo.bigMax); + dataconvert::DataConvert::int128Max(cpInfo.bigMin); + } + cpInfo.seqNum = -1; + cpInfo.isBinaryColumn = isBinaryColumn; for (uint32_t i = 0; i < ranges.size(); i++) { @@ -433,17 +453,38 @@ int clearAllCPData() //------------------------------------------------------------------------------ int clearmm(OID_t oid) { - BRM::LBIDRange_v ranges; CHECK(emp->lookup(oid, ranges)); BRM::LBIDRange_v::size_type rcount = ranges.size(); + // Get the extents for a given OID to determine it's column width + std::vector entries; + CHECK(emp->getExtents(oid, entries, false, false, true)); + if (entries.empty()) + { + cerr << "There are no entries in the Extent Map for OID: " << oid << endl; + return 1; + } + + bool isBinaryColumn = entries[0].colWid > 8; + // @bug 2280. Changed to use the batch interface to clear the CP info to make the clear option faster. BRM::CPInfo cpInfo; BRM::CPInfoList_t vCpInfo; - cpInfo.max = numeric_limits::min(); - cpInfo.min = numeric_limits::max(); + + if (!isBinaryColumn) + { + cpInfo.max = numeric_limits::min(); + cpInfo.min = numeric_limits::max(); + } + else + { + dataconvert::DataConvert::int128Min(cpInfo.bigMax); + dataconvert::DataConvert::int128Max(cpInfo.bigMin); + } + cpInfo.seqNum = -1; + cpInfo.isBinaryColumn = isBinaryColumn; for (unsigned i = 0; i < rcount; i++) { diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 501a2e59a..5016f478c 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1163,22 +1163,12 @@ bool stringToTimestampStruct(const string& data, TimeStamp& timeStamp, const str } -// WIP MCOL-641 -#include - -struct uint128_pod -{ - uint64_t lo; - uint64_t hi; -}; - // WIP MCOL-641 // Check for overflows with buflen template void DataConvert::toString(T* dec, char *p, size_t buflen) { uint64_t div = 10000000000000000000ULL; - size_t div_log = 19; // template this uint128_t high = *dec; uint128_t low; @@ -1192,23 +1182,27 @@ void DataConvert::toString(T* dec, char *p, size_t buflen) // use typeof // Or a templated structure // Use uint64* to access parts of uint128 and remove pods - uint128_pod *high_pod = reinterpret_cast(&high); - uint128_pod *mid_pod = reinterpret_cast(&mid); - uint128_pod *low_pod = reinterpret_cast(&low); + Int128Pod_t *high_pod = reinterpret_cast(&high); + Int128Pod_t *mid_pod = reinterpret_cast(&mid); + Int128Pod_t *low_pod = reinterpret_cast(&low); char* original_p = p; int printed_chars = 0; // WIP replace snprintf with streams if (high_pod->lo != 0) { - printed_chars = snprintf(p, div_log+1, "%lu", high_pod->lo); + printed_chars = sprintf(p, "%lu", high_pod->lo); + p += printed_chars; + printed_chars = sprintf(p, "%019lu", mid_pod->lo); p += printed_chars; - printed_chars = snprintf(p, div_log+1, "%019lu", mid_pod->lo); - p += printed_chars; + sprintf(p, "%019lu", low_pod->lo); } else if (mid_pod->lo != 0) { - printed_chars = snprintf(p, div_log+1, "%lu", mid_pod->lo); + printed_chars = sprintf(p, "%lu", mid_pod->lo); p += printed_chars; + sprintf(p, "%019lu", low_pod->lo); + } + else { + sprintf(p, "%lu", low_pod->lo); } - snprintf(p, div_log+1, "%019lu", low_pod->lo); if (buflen <= p-original_p) std::cout << "DataConvert::toString char buffer overflow" << std::endl; } diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 6ec934610..4bd30ca33 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -134,9 +134,23 @@ const int32_t MIN_TIMESTAMP_VALUE = 0; namespace dataconvert { +// Decimal has maximum 38 digits with 3 extra chars for dot(.), minus(-), null character(\0) +const int MAX_DECIMAL_STRING_LENGTH = 41; + +// WIP MCOL-641 using int128_t = __int128; using uint128_t = unsigned __int128; +struct Int128Pod_struct +{ + uint64_t lo; + uint64_t hi; +}; + +typedef Int128Pod_struct Int128Pod_t; + +void atoi128(const std::string& arg, int128_t& res); + enum CalpontDateTimeFormat { CALPONTDATE_ENUM = 1, // date format is: "YYYY-MM-DD" @@ -1022,6 +1036,28 @@ public: template EXPORT static void toString(T* dec, char *p, size_t buflen); + + static inline void int128Max(int128_t& i) + { + Int128Pod_t *pod = reinterpret_cast(&i); + pod->lo = 0xFFFFFFFFFFFFFFFF; + pod->hi = 0x7FFFFFFFFFFFFFFF; + } + + static inline void int128Min(int128_t& i) + { + Int128Pod_t *pod = reinterpret_cast(&i); + pod->lo = 0; + pod->hi = 0x8000000000000000; + } + + static inline void uint128Max(uint128_t& i) + { + Int128Pod_t *pod = reinterpret_cast(&i); + pod->lo = 0xFFFFFFFFFFFFFFFF; + pod->hi = 0xFFFFFFFFFFFFFFFF; + } + static inline std::string constructRegexp(const std::string& str); static inline void trimWhitespace(int64_t& charData); static inline bool isEscapedChar(char c) diff --git a/versioning/BRM/brmtypes.h b/versioning/BRM/brmtypes.h index e0e3f64e7..373b1a2eb 100644 --- a/versioning/BRM/brmtypes.h +++ b/versioning/BRM/brmtypes.h @@ -151,6 +151,17 @@ struct CPInfo int64_t max; int64_t min; int32_t seqNum; + union + { + __int128 bigMax; + int64_t max_; + }; + union + { + __int128 bigMin; + int64_t min_; + }; + bool isBinaryColumn; }; typedef std::vector CPInfoList_t; @@ -160,6 +171,17 @@ struct CPMaxMin int64_t max; int64_t min; int32_t seqNum; + union + { + __int128 bigMax; + int64_t max_; + }; + union + { + __int128 bigMin; + int64_t min_; + }; + bool isBinaryColumn; }; typedef std::tr1::unordered_map CPMaxMinMap_t; @@ -172,7 +194,18 @@ struct CPInfoMerge int64_t min; // min value to be merged with current min value int32_t seqNum; // sequence number (not currently used) execplan::CalpontSystemCatalog::ColDataType type; + int32_t colWidth; bool newExtent; // is this to be treated as a new extent + union + { + __int128 bigMax; + int64_t max_; + }; + union + { + __int128 bigMin; + int64_t min_; + }; }; typedef std::vector CPInfoMergeList_t; @@ -184,7 +217,18 @@ struct CPMaxMinMerge int64_t min; int32_t seqNum; execplan::CalpontSystemCatalog::ColDataType type; + int32_t colWidth; bool newExtent; + union + { + __int128 bigMax; + int64_t max_; + }; + union + { + __int128 bigMin; + int64_t min_; + }; }; typedef std::tr1::unordered_map CPMaxMinMergeMap_t; diff --git a/versioning/BRM/dbrm.cpp b/versioning/BRM/dbrm.cpp index e6079f252..028efad55 100644 --- a/versioning/BRM/dbrm.cpp +++ b/versioning/BRM/dbrm.cpp @@ -30,6 +30,7 @@ //#define NDEBUG #include +#include "dataconvert.h" #include "oamcache.h" #include "rwlock.h" #include "mastersegmenttable.h" @@ -461,7 +462,8 @@ int DBRM::markExtentsInvalid(const vector& lbids, return err; } -int DBRM::getExtentMaxMin(const LBID_t lbid, int64_t& max, int64_t& min, int32_t& seqNum) throw() +template +int DBRM::getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) throw() { #ifdef BRM_INFO @@ -556,7 +558,14 @@ int DBRM::setExtentsMaxMin(const CPInfoList_t& cpInfos) DBRM_THROW for (it = cpInfos.begin(); it != cpInfos.end(); it++) { - command << (uint64_t)it->firstLbid << (uint64_t)it->max << (uint64_t)it->min << (uint32_t)it->seqNum; + if (it->isBinaryColumn) + { + command << (uint8_t)1 << (uint64_t)it->firstLbid << (unsigned __int128)it->bigMax << (unsigned __int128)it->bigMin << (uint32_t)it->seqNum; + } + else + { + command << (uint8_t)0 << (uint64_t)it->firstLbid << (uint64_t)it->max << (uint64_t)it->min << (uint32_t)it->seqNum; + } } err = send_recv(command, response); @@ -4526,15 +4535,34 @@ void DBRM::invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN // lookup the column oid for that lbid (all we care about is oid here) if (em->lookupLocal(lbid, oid, dbRoot, partitionNum, segmentNum, fileBlockOffset) == 0) { - if (execplan::isUnsigned(csc->colType(oid).colDataType)) + execplan::CalpontSystemCatalog::ColType colType = csc->colType(oid); + bool isBinaryColumn = colType.colWidth > 8; + aInfo.isBinaryColumn = isBinaryColumn; + if (!isBinaryColumn) { - aInfo.max = 0; - aInfo.min = numeric_limits::max(); + if (execplan::isUnsigned(colType.colDataType)) + { + aInfo.max = 0; + aInfo.min = numeric_limits::max(); + } + else + { + aInfo.max = numeric_limits::min(); + aInfo.min = numeric_limits::max(); + } } else { - aInfo.max = numeric_limits::min(); - aInfo.min = numeric_limits::max(); + if (execplan::isUnsigned(colType.colDataType)) + { + aInfo.bigMax = 0; + aInfo.bigMin = -1; + } + else + { + dataconvert::DataConvert::int128Min(aInfo.bigMax); + dataconvert::DataConvert::int128Max(aInfo.bigMin); + } } } else @@ -4542,6 +4570,8 @@ void DBRM::invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN // We have a problem, but we need to put something in. This should never happen. aInfo.max = numeric_limits::min(); aInfo.min = numeric_limits::max(); + // MCOL-641 is this correct? + aInfo.isBinaryColumn = false; } aInfo.seqNum = -2; @@ -4552,4 +4582,10 @@ void DBRM::invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN setExtentsMaxMin(cpInfos); } +template +int DBRM::getExtentMaxMin<__int128>(const LBID_t lbid, __int128& max, __int128& min, int32_t& seqNum) throw(); + +template +int DBRM::getExtentMaxMin(const LBID_t lbid, int64_t& max, int64_t& min, int32_t& seqNum) throw(); + } //namespace diff --git a/versioning/BRM/dbrm.h b/versioning/BRM/dbrm.h index 00165f496..90b24d203 100644 --- a/versioning/BRM/dbrm.h +++ b/versioning/BRM/dbrm.h @@ -822,7 +822,8 @@ public: execplan::CalpontSystemCatalog::ColDataType colDataType) DBRM_THROW; EXPORT int markExtentsInvalid(const std::vector& lbids, const std::vector& colDataTypes) DBRM_THROW; - EXPORT int getExtentMaxMin(const LBID_t lbid, int64_t& max, int64_t& min, int32_t& seqNum) throw(); + template + EXPORT int getExtentMaxMin(const LBID_t lbid, T& max, T& min, int32_t& seqNum) throw(); EXPORT int setExtentMaxMin(const LBID_t lbid, const int64_t max, const int64_t min, const int32_t seqNum) DBRM_THROW; diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index 2b79fc6ea..d549da71f 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -116,8 +116,10 @@ namespace BRM EMCasualPartition_struct::EMCasualPartition_struct() { - lo_val = numeric_limits::min(); - hi_val = numeric_limits::max(); + lo_val = numeric_limits::max(); + hi_val = numeric_limits::min(); + dataconvert::DataConvert::int128Max(bigLoVal); + dataconvert::DataConvert::int128Min(bigHiVal); sequenceNum = 0; isValid = CP_INVALID; } @@ -130,10 +132,20 @@ EMCasualPartition_struct::EMCasualPartition_struct(const int64_t lo, const int64 isValid = CP_INVALID; } +EMCasualPartition_struct::EMCasualPartition_struct(const __int128 bigLo, const __int128 bigHi, const int32_t seqNum) +{ + bigLoVal = bigLo; + bigHiVal = bigHi; + sequenceNum = seqNum; + isValid = CP_INVALID; +} + EMCasualPartition_struct::EMCasualPartition_struct(const EMCasualPartition_struct& em) { lo_val = em.lo_val; hi_val = em.hi_val; + bigLoVal = em.bigLoVal; + bigHiVal = em.bigHiVal; sequenceNum = em.sequenceNum; isValid = em.isValid; } @@ -142,6 +154,8 @@ EMCasualPartition_struct& EMCasualPartition_struct::operator= (const EMCasualPar { lo_val = em.lo_val; hi_val = em.hi_val; + bigLoVal = em.bigLoVal; + bigHiVal = em.bigHiVal; sequenceNum = em.sequenceNum; isValid = em.isValid; return *this; @@ -339,11 +353,15 @@ int ExtentMap::_markInvalid(const LBID_t lbid, const execplan::CalpontSystemCata { fExtentMap[i].partition.cprange.lo_val = numeric_limits::max(); fExtentMap[i].partition.cprange.hi_val = 0; + fExtentMap[i].partition.cprange.bigLoVal = -1; + fExtentMap[i].partition.cprange.bigHiVal = 0; } else { fExtentMap[i].partition.cprange.lo_val = numeric_limits::max(); fExtentMap[i].partition.cprange.hi_val = numeric_limits::min(); + dataconvert::DataConvert::int128Max(fExtentMap[i].partition.cprange.bigLoVal); + dataconvert::DataConvert::int128Min(fExtentMap[i].partition.cprange.bigHiVal); } incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); @@ -460,6 +478,8 @@ int ExtentMap::markInvalid(const vector& lbids, **/ +// TODO MCOL-641 Not adding support here since this function appears to be unused anywhere. + int ExtentMap::setMaxMin(const LBID_t lbid, const int64_t max, const int64_t min, @@ -626,8 +646,16 @@ void ExtentMap::setExtentsMaxMin(const CPMaxMinMap_t& cpMap, bool firstNode, boo fExtentMap[i].partition.cprange.isValid == CP_INVALID) { makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); - fExtentMap[i].partition.cprange.hi_val = it->second.max; - fExtentMap[i].partition.cprange.lo_val = it->second.min; + if (it->second.isBinaryColumn) + { + fExtentMap[i].partition.cprange.bigHiVal = it->second.bigMax; + fExtentMap[i].partition.cprange.bigLoVal = it->second.bigMin; + } + else + { + fExtentMap[i].partition.cprange.hi_val = it->second.max; + fExtentMap[i].partition.cprange.lo_val = it->second.min; + } fExtentMap[i].partition.cprange.isValid = CP_VALID; incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); extentsUpdated++; @@ -663,8 +691,16 @@ void ExtentMap::setExtentsMaxMin(const CPMaxMinMap_t& cpMap, bool firstNode, boo else if (it->second.seqNum == -2) { makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); - fExtentMap[i].partition.cprange.hi_val = it->second.max; - fExtentMap[i].partition.cprange.lo_val = it->second.min; + if (it->second.isBinaryColumn) + { + fExtentMap[i].partition.cprange.bigHiVal = it->second.bigMax; + fExtentMap[i].partition.cprange.bigLoVal = it->second.bigMin; + } + else + { + fExtentMap[i].partition.cprange.hi_val = it->second.max; + fExtentMap[i].partition.cprange.lo_val = it->second.min; + } fExtentMap[i].partition.cprange.isValid = CP_INVALID; incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); extentsUpdated++; @@ -707,6 +743,7 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) { CPMaxMinMergeMap_t::const_iterator it; + // TODO MCOL-641 Add support in the debugging outputs here. #ifdef BRM_DEBUG log("ExtentMap::mergeExtentsMaxMin()", logging::LOG_TYPE_DEBUG); @@ -784,13 +821,15 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) log(os.str(), logging::LOG_TYPE_DEBUG); #endif + bool isBinaryColumn = it->second.colWidth > 8; + switch (fExtentMap[i].partition.cprange.isValid) { // Merge input min/max with current min/max case CP_VALID: { - if (!isValidCPRange( it->second.max, - it->second.min, + if (!isValidCPRange( !isBinaryColumn ? it->second.max : it->second.bigMax, + !isBinaryColumn ? it->second.min : it->second.bigMin, it->second.type )) { break; @@ -804,8 +843,8 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) // min/max needs to be set instead of merged. if (isValidCPRange( - fExtentMap[i].partition.cprange.hi_val, - fExtentMap[i].partition.cprange.lo_val, + !isBinaryColumn ? fExtentMap[i].partition.cprange.hi_val : fExtentMap[i].partition.cprange.bigHiVal, + !isBinaryColumn ? fExtentMap[i].partition.cprange.lo_val : fExtentMap[i].partition.cprange.bigLoVal, it->second.type)) { // Swap byte order to do binary string comparison @@ -836,39 +875,83 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) } else if (isUnsigned(it->second.type)) { - if (static_cast(it->second.min) < - static_cast(fExtentMap[i].partition.cprange.lo_val)) + if (!isBinaryColumn) { - fExtentMap[i].partition.cprange.lo_val = - it->second.min; - } + if (static_cast(it->second.min) < + static_cast(fExtentMap[i].partition.cprange.lo_val)) + { + fExtentMap[i].partition.cprange.lo_val = + it->second.min; + } - if (static_cast(it->second.max) > - static_cast(fExtentMap[i].partition.cprange.hi_val)) + if (static_cast(it->second.max) > + static_cast(fExtentMap[i].partition.cprange.hi_val)) + { + fExtentMap[i].partition.cprange.hi_val = + it->second.max; + } + } + else { - fExtentMap[i].partition.cprange.hi_val = - it->second.max; + if (static_cast(it->second.bigMin) < + static_cast(fExtentMap[i].partition.cprange.bigLoVal)) + { + fExtentMap[i].partition.cprange.bigLoVal = + it->second.bigMin; + } + + if (static_cast(it->second.bigMax) > + static_cast(fExtentMap[i].partition.cprange.bigHiVal)) + { + fExtentMap[i].partition.cprange.bigHiVal = + it->second.bigMax; + } } } else { - if (it->second.min < - fExtentMap[i].partition.cprange.lo_val) - fExtentMap[i].partition.cprange.lo_val = - it->second.min; + if (!isBinaryColumn) + { + if (it->second.min < + fExtentMap[i].partition.cprange.lo_val) + fExtentMap[i].partition.cprange.lo_val = + it->second.min; - if (it->second.max > - fExtentMap[i].partition.cprange.hi_val) - fExtentMap[i].partition.cprange.hi_val = - it->second.max; + if (it->second.max > + fExtentMap[i].partition.cprange.hi_val) + fExtentMap[i].partition.cprange.hi_val = + it->second.max; + } + else + { + if (it->second.bigMin < + fExtentMap[i].partition.cprange.bigLoVal) + fExtentMap[i].partition.cprange.bigLoVal = + it->second.bigMin; + + if (it->second.bigMax > + fExtentMap[i].partition.cprange.bigHiVal) + fExtentMap[i].partition.cprange.bigHiVal = + it->second.bigMax; + } } } else { - fExtentMap[i].partition.cprange.lo_val = - it->second.min; - fExtentMap[i].partition.cprange.hi_val = - it->second.max; + if (!isBinaryColumn) + { + fExtentMap[i].partition.cprange.lo_val = + it->second.min; + fExtentMap[i].partition.cprange.hi_val = + it->second.max; + } + else + { + fExtentMap[i].partition.cprange.bigLoVal = + it->second.bigMin; + fExtentMap[i].partition.cprange.bigHiVal = + it->second.bigMax; + } } incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); @@ -897,14 +980,24 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) if (it->second.newExtent) { - if (isValidCPRange( it->second.max, - it->second.min, + if (isValidCPRange( !isBinaryColumn ? it->second.max : it->second.bigMax, + !isBinaryColumn ? it->second.min : it->second.bigMin, it->second.type )) { - fExtentMap[i].partition.cprange.lo_val = - it->second.min; - fExtentMap[i].partition.cprange.hi_val = - it->second.max; + if (!isBinaryColumn) + { + fExtentMap[i].partition.cprange.lo_val = + it->second.min; + fExtentMap[i].partition.cprange.hi_val = + it->second.max; + } + else + { + fExtentMap[i].partition.cprange.bigLoVal = + it->second.bigMin; + fExtentMap[i].partition.cprange.bigHiVal = + it->second.bigMax; + } } // Even if invalid range; we set state to CP_VALID, @@ -939,22 +1032,51 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) // Range is considered invalid if min or max, are NULL (min()), or EMPTY // (min()+1). For unsigned types NULL is max() and EMPTY is max()-1. //------------------------------------------------------------------------------ -bool ExtentMap::isValidCPRange(int64_t max, int64_t min, execplan::CalpontSystemCatalog::ColDataType type) const +template +bool ExtentMap::isValidCPRange(const T& max, const T& min, execplan::CalpontSystemCatalog::ColDataType type) const { if (isUnsigned(type)) { - if ( (static_cast(min) >= (numeric_limits::max() - 1)) || - (static_cast(max) >= (numeric_limits::max() - 1)) ) + if (typeid(T) != typeid(__int128)) { - return false; + if ( (static_cast(min) >= (numeric_limits::max() - 1)) || + (static_cast(max) >= (numeric_limits::max() - 1)) ) + { + return false; + } + } + else + { + unsigned __int128 temp; + dataconvert::DataConvert::uint128Max(temp); + + if ( (static_cast(min) >= (temp - 1)) || + (static_cast(max) >= (temp - 1)) ) + { + return false; + } } } else { - if ( (min <= (numeric_limits::min() + 1)) || - (max <= (numeric_limits::min() + 1)) ) + if (typeid(T) != typeid(__int128)) { - return false; + if ( (min <= (numeric_limits::min() + 1)) || + (max <= (numeric_limits::min() + 1)) ) + { + return false; + } + } + else + { + __int128 temp; + dataconvert::DataConvert::int128Min(temp); + + if ( (min <= (temp + 1)) || + (max <= (temp + 1)) ) + { + return false; + } } } @@ -969,9 +1091,9 @@ bool ExtentMap::isValidCPRange(int64_t max, int64_t min, execplan::CalpontSystem * return the sequenceNum of the extent and the max/min values as -1. **/ +template int ExtentMap::getMaxMin(const LBID_t lbid, - int64_t& max, - int64_t& min, + T& max, T& min, int32_t& seqNum) { #ifdef BRM_INFO @@ -987,8 +1109,19 @@ int ExtentMap::getMaxMin(const LBID_t lbid, } #endif - max = numeric_limits::max(); - min = 0; + if (typeid(T) == typeid(__int128)) + { + __int128 tmpMax, tmpMin; + dataconvert::DataConvert::int128Min(tmpMax); + dataconvert::DataConvert::int128Max(tmpMin); + max = tmpMax; + min = tmpMin; + } + else + { + max = numeric_limits::min(); + min = numeric_limits::max(); + } seqNum *= (-1); int entries; int i; @@ -1014,8 +1147,16 @@ int ExtentMap::getMaxMin(const LBID_t lbid, if (lbid >= fExtentMap[i].range.start && lbid <= lastBlock) { - max = fExtentMap[i].partition.cprange.hi_val; - min = fExtentMap[i].partition.cprange.lo_val; + if (typeid(T) == typeid(__int128)) + { + max = fExtentMap[i].partition.cprange.bigHiVal; + min = fExtentMap[i].partition.cprange.bigLoVal; + } + else + { + max = fExtentMap[i].partition.cprange.hi_val; + min = fExtentMap[i].partition.cprange.lo_val; + } seqNum = fExtentMap[i].partition.cprange.sequenceNum; isValid = fExtentMap[i].partition.cprange.isValid; releaseEMEntryTable(READ); @@ -2569,13 +2710,29 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, if (isUnsigned(colDataType)) { - e->partition.cprange.lo_val = numeric_limits::max(); - e->partition.cprange.hi_val = 0; + if (colWidth <= 8) + { + e->partition.cprange.lo_val = numeric_limits::max(); + e->partition.cprange.hi_val = 0; + } + else + { + e->partition.cprange.bigLoVal = -1; + e->partition.cprange.bigHiVal = 0; + } } else { - e->partition.cprange.lo_val = numeric_limits::max(); - e->partition.cprange.hi_val = numeric_limits::min(); + if (colWidth <= 8) + { + e->partition.cprange.lo_val = numeric_limits::max(); + e->partition.cprange.hi_val = numeric_limits::min(); + } + else + { + dataconvert::DataConvert::int128Max(e->partition.cprange.bigLoVal); + dataconvert::DataConvert::int128Min(e->partition.cprange.bigHiVal); + } } e->partition.cprange.sequenceNum = 0; @@ -2764,13 +2921,29 @@ LBID_t ExtentMap::_createColumnExtentExactFile(uint32_t size, int OID, if (isUnsigned(colDataType)) { - e->partition.cprange.lo_val = numeric_limits::max(); - e->partition.cprange.hi_val = 0; + if (colWidth <= 8) + { + e->partition.cprange.lo_val = numeric_limits::max(); + e->partition.cprange.hi_val = 0; + } + else + { + e->partition.cprange.bigLoVal = -1; + e->partition.cprange.bigHiVal = 0; + } } else { - e->partition.cprange.lo_val = numeric_limits::max(); - e->partition.cprange.hi_val = numeric_limits::min(); + if (colWidth <= 8) + { + e->partition.cprange.lo_val = numeric_limits::max(); + e->partition.cprange.hi_val = numeric_limits::min(); + } + else + { + dataconvert::DataConvert::int128Max(e->partition.cprange.bigLoVal); + dataconvert::DataConvert::int128Min(e->partition.cprange.bigHiVal); + } } e->partition.cprange.sequenceNum = 0; @@ -2955,6 +3128,8 @@ LBID_t ExtentMap::_createDictStoreExtent(uint32_t size, int OID, e->status = EXTENTUNAVAILABLE;// @bug 1911 mark extent as in process e->partition.cprange.lo_val = numeric_limits::max(); e->partition.cprange.hi_val = numeric_limits::min(); + dataconvert::DataConvert::int128Max(e->partition.cprange.bigLoVal); + dataconvert::DataConvert::int128Min(e->partition.cprange.bigHiVal); e->partition.cprange.sequenceNum = 0; e->partition.cprange.isValid = CP_INVALID; @@ -5840,6 +6015,12 @@ void ExtentMap::dumpTo(ostream& os) } */ +template +int ExtentMap::getMaxMin<__int128>(const LBID_t lbidRange, __int128& max, __int128& min, int32_t& seqNum); + +template +int ExtentMap::getMaxMin(const LBID_t lbidRange, int64_t& max, int64_t& min, int32_t& seqNum); + } //namespace // vim:ts=4 sw=4: diff --git a/versioning/BRM/extentmap.h b/versioning/BRM/extentmap.h index 6cd628a37..e3d4634d8 100644 --- a/versioning/BRM/extentmap.h +++ b/versioning/BRM/extentmap.h @@ -102,14 +102,25 @@ const char CP_VALID = 2; struct EMCasualPartition_struct { - RangePartitionData_t hi_val; // This needs to be reinterpreted as unsigned for uint64_t column types. + RangePartitionData_t hi_val; // This needs to be reinterpreted as unsigned for uint64_t column types. RangePartitionData_t lo_val; int32_t sequenceNum; char isValid; //CP_INVALID - No min/max and no DML in progress. CP_UPDATING - Update in progress. CP_VALID- min/max is valid EXPORT EMCasualPartition_struct(); EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum); + EXPORT EMCasualPartition_struct(const __int128 bigLo, const __int128 bigHi, const int32_t seqNum); EXPORT EMCasualPartition_struct(const EMCasualPartition_struct& em); EXPORT EMCasualPartition_struct& operator= (const EMCasualPartition_struct& em); + union + { + __int128 bigLoVal; + int64_t loVal; + }; + union + { + __int128 bigHiVal; + int64_t hiVal; + }; }; typedef EMCasualPartition_struct EMCasualPartition_t; @@ -854,7 +865,8 @@ public: */ void mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock = true); - EXPORT int getMaxMin(const LBID_t lbidRange, int64_t& max, int64_t& min, int32_t& seqNum); + template + EXPORT int getMaxMin(const LBID_t lbidRange, T& max, T& min, int32_t& seqNum); inline bool empty() { @@ -934,7 +946,8 @@ private: uint16_t dbRoot, uint32_t partitionNum, uint16_t segmentNum); - bool isValidCPRange(int64_t max, int64_t min, execplan::CalpontSystemCatalog::ColDataType type) const; + template + bool isValidCPRange(const T& max, const T& min, execplan::CalpontSystemCatalog::ColDataType type) const; void deleteExtent(int emIndex); LBID_t getLBIDsFromFreeList(uint32_t size); void reserveLBIDRange(LBID_t start, uint8_t size); // used by load() to allocate pre-existing LBIDs diff --git a/versioning/BRM/slavecomm.cpp b/versioning/BRM/slavecomm.cpp index f43a8c351..56b9d5d49 100644 --- a/versioning/BRM/slavecomm.cpp +++ b/versioning/BRM/slavecomm.cpp @@ -1329,9 +1329,12 @@ void SlaveComm::do_setExtentsMaxMin(ByteStream& msg) LBID_t lbid; uint64_t tmp64; uint32_t tmp32; + uint8_t tmp8; + unsigned __int128 tmp128; int err; ByteStream reply; int32_t updateCount; + bool isBinaryColumn = false; #ifdef BRM_VERBOSE cerr << "WorkerComm: do_setExtentsMaxMin()" << endl; @@ -1348,14 +1351,30 @@ void SlaveComm::do_setExtentsMaxMin(ByteStream& msg) // Loop through extents and add each one to a map. for (int64_t i = 0; i < updateCount; i++) { + msg >> tmp8; + isBinaryColumn = (tmp8 != 0); + msg >> tmp64; lbid = tmp64; - msg >> tmp64; - cpMaxMin.max = tmp64; + cpMaxMin.isBinaryColumn = isBinaryColumn; - msg >> tmp64; - cpMaxMin.min = tmp64; + if (isBinaryColumn) + { + msg >> tmp128; + cpMaxMin.bigMax = tmp128; + + msg >> tmp128; + cpMaxMin.bigMin = tmp128; + } + else + { + msg >> tmp64; + cpMaxMin.max = tmp64; + + msg >> tmp64; + cpMaxMin.min = tmp64; + } msg >> tmp32; cpMaxMin.seqNum = tmp32; diff --git a/versioning/BRM/slavedbrmnode.cpp b/versioning/BRM/slavedbrmnode.cpp index 36d4b0cb7..f163ea31f 100644 --- a/versioning/BRM/slavedbrmnode.cpp +++ b/versioning/BRM/slavedbrmnode.cpp @@ -413,8 +413,17 @@ int SlaveDBRMNode::bulkSetHWMAndCP(const vector& hwmArgs, { for (i = 0; i < setCPDataArgs.size(); i++) { - setCPEntry.max = setCPDataArgs[i].max; - setCPEntry.min = setCPDataArgs[i].min; + setCPEntry.isBinaryColumn = setCPDataArgs[i].isBinaryColumn; + if (!setCPEntry.isBinaryColumn) + { + setCPEntry.max = setCPDataArgs[i].max; + setCPEntry.min = setCPDataArgs[i].min; + } + else + { + setCPEntry.bigMax = setCPDataArgs[i].bigMax; + setCPEntry.bigMin = setCPDataArgs[i].bigMin; + } setCPEntry.seqNum = setCPDataArgs[i].seqNum; bulkSetCPMap[setCPDataArgs[i].firstLbid] = setCPEntry; } @@ -428,8 +437,17 @@ int SlaveDBRMNode::bulkSetHWMAndCP(const vector& hwmArgs, for (i = 0; i < mergeCPDataArgs.size(); i++) { mergeCPEntry.type = mergeCPDataArgs[i].type; - mergeCPEntry.max = mergeCPDataArgs[i].max; - mergeCPEntry.min = mergeCPDataArgs[i].min; + mergeCPEntry.colWidth = mergeCPDataArgs[i].colWidth; + if (mergeCPDataArgs[i].colWidth <= 8) + { + mergeCPEntry.max = mergeCPDataArgs[i].max; + mergeCPEntry.min = mergeCPDataArgs[i].min; + } + else + { + mergeCPEntry.bigMax = mergeCPDataArgs[i].bigMax; + mergeCPEntry.bigMin = mergeCPDataArgs[i].bigMin; + } mergeCPEntry.newExtent = mergeCPDataArgs[i].newExtent; mergeCPEntry.seqNum = mergeCPDataArgs[i].seqNum; bulkMergeCPMap[mergeCPDataArgs[i].startLbid] = mergeCPEntry; diff --git a/writeengine/bulk/we_brmreporter.cpp b/writeengine/bulk/we_brmreporter.cpp index fd9691ef9..0787700be 100644 --- a/writeengine/bulk/we_brmreporter.cpp +++ b/writeengine/bulk/we_brmreporter.cpp @@ -294,6 +294,7 @@ void BRMReporter::sendHWMToFile( ) //------------------------------------------------------------------------------ // Send Casual Partition update information to BRM //------------------------------------------------------------------------------ +// TODO MCOL-641 void BRMReporter::sendCPToFile( ) { if (fCPInfo.size() > 0) diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index 069aa0c64..1f199fac8 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -301,6 +301,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, int32_t iDate; char charTmpBuf[MAX_COLUMN_BOUNDARY + 1] = {0}; long long llVal = 0, llDate = 0; + __int128 bigllVal = 0; uint64_t tmp64; uint32_t tmp32; uint8_t ubiVal; @@ -938,7 +939,9 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, // BIG INT //---------------------------------------------------------------------- case WriteEngine::WR_LONGLONG: + case WriteEngine::WR_BINARY: { + // TODO MCOL-641 Add full support here. bool bSatVal = false; if ( column.dataType != CalpontSystemCatalog::DATETIME && @@ -983,9 +986,16 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, if ( (column.dataType == CalpontSystemCatalog::DECIMAL) || (column.dataType == CalpontSystemCatalog::UDECIMAL)) { - // errno is initialized and set in convertDecimalString - llVal = Convertor::convertDecimalString( - field, fieldLength, column.scale ); + if (width <= 8) + { + // errno is initialized and set in convertDecimalString + llVal = Convertor::convertDecimalString( + field, fieldLength, column.scale ); + } + else + { + dataconvert::atoi128(string(field), bigllVal); + } } else { @@ -1016,13 +1026,26 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, bufStats.satCount++; // Update min/max range - if (llVal < bufStats.minBufferVal) - bufStats.minBufferVal = llVal; + if (width <= 8) + { + if (llVal < bufStats.minBufferVal) + bufStats.minBufferVal = llVal; - if (llVal > bufStats.maxBufferVal) - bufStats.maxBufferVal = llVal; + if (llVal > bufStats.maxBufferVal) + bufStats.maxBufferVal = llVal; - pVal = &llVal; + pVal = &llVal; + } + else + { + if (bigllVal < bufStats.bigMinBufferVal) + bufStats.bigMinBufferVal = bigllVal; + + if (bigllVal > bufStats.bigMaxBufferVal) + bufStats.bigMaxBufferVal = bigllVal; + + pVal = &bigllVal; + } } else if (column.dataType == CalpontSystemCatalog::TIME) { @@ -1651,11 +1674,24 @@ int BulkLoadBuffer::parseCol(ColumnInfo& columnInfo) // Update CP min/max if this is last row in this extent if ( (fStartRowParser + i) == lastInputRowInExtent ) { - columnInfo.updateCPInfo( lastInputRowInExtent, - bufStats.minBufferVal, - bufStats.maxBufferVal, - columnInfo.column.dataType ); + if (columnInfo.column.width <= 8) + { + columnInfo.updateCPInfo( lastInputRowInExtent, + bufStats.minBufferVal, + bufStats.maxBufferVal, + columnInfo.column.dataType, + columnInfo.column.width ); + } + else + { + columnInfo.updateCPInfo( lastInputRowInExtent, + bufStats.bigMinBufferVal, + bufStats.bigMaxBufferVal, + columnInfo.column.dataType, + columnInfo.column.width ); + } + // TODO MCOL-641 Add support here. if (fLog->isDebug( DEBUG_2 )) { ostringstream oss; @@ -1675,14 +1711,30 @@ int BulkLoadBuffer::parseCol(ColumnInfo& columnInfo) if (isUnsigned(columnInfo.column.dataType) || isCharType(columnInfo.column.dataType)) { - bufStats.minBufferVal = static_cast(MAX_UBIGINT); - bufStats.maxBufferVal = static_cast(MIN_UBIGINT); + if (columnInfo.column.width <= 8) + { + bufStats.minBufferVal = static_cast(MAX_UBIGINT); + bufStats.maxBufferVal = static_cast(MIN_UBIGINT); + } + else + { + bufStats.bigMinBufferVal = -1; + bufStats.bigMaxBufferVal = 0; + } updateCPInfoPendingFlag = false; } else { - bufStats.minBufferVal = MAX_BIGINT; - bufStats.maxBufferVal = MIN_BIGINT; + if (columnInfo.column.width <= 8) + { + bufStats.minBufferVal = MAX_BIGINT; + bufStats.maxBufferVal = MIN_BIGINT; + } + else + { + dataconvert::DataConvert::int128Max(bufStats.bigMinBufferVal); + dataconvert::DataConvert::int128Min(bufStats.bigMaxBufferVal); + } updateCPInfoPendingFlag = false; } } @@ -1690,10 +1742,22 @@ int BulkLoadBuffer::parseCol(ColumnInfo& columnInfo) if (updateCPInfoPendingFlag) { - columnInfo.updateCPInfo( lastInputRowInExtent, - bufStats.minBufferVal, - bufStats.maxBufferVal, - columnInfo.column.dataType ); + if (columnInfo.column.width <= 8) + { + columnInfo.updateCPInfo( lastInputRowInExtent, + bufStats.minBufferVal, + bufStats.maxBufferVal, + columnInfo.column.dataType, + columnInfo.column.width ); + } + else + { + columnInfo.updateCPInfo( lastInputRowInExtent, + bufStats.bigMinBufferVal, + bufStats.bigMaxBufferVal, + columnInfo.column.dataType, + columnInfo.column.width ); + } } if (bufStats.satCount) // @bug 3504: increment row saturation count @@ -1724,6 +1788,7 @@ int BulkLoadBuffer::parseCol(ColumnInfo& columnInfo) Stats::stopParseEvent(WE_STATS_PARSE_COL); #endif + // TODO MCOL-641 Add support here. if (fLog->isDebug( DEBUG_2 )) { ostringstream oss; diff --git a/writeengine/bulk/we_bulkloadbuffer.h b/writeengine/bulk/we_bulkloadbuffer.h index b5e817741..3932f39aa 100644 --- a/writeengine/bulk/we_bulkloadbuffer.h +++ b/writeengine/bulk/we_bulkloadbuffer.h @@ -30,6 +30,7 @@ #include "boost/ptr_container/ptr_vector.hpp" #include "we_columninfo.h" #include "calpontsystemcatalog.h" +#include "dataconvert.h" namespace WriteEngine { @@ -42,17 +43,31 @@ public: int64_t minBufferVal; int64_t maxBufferVal; int64_t satCount; + union + { + __int128 bigMinBufferVal; + int64_t minBufferVal_; + }; + union + { + __int128 bigMaxBufferVal; + int64_t maxBufferVal_; + }; BLBufferStats(ColDataType colDataType) : satCount(0) { if (isUnsigned(colDataType) || isCharType(colDataType)) { minBufferVal = static_cast(MAX_UBIGINT); maxBufferVal = static_cast(MIN_UBIGINT); + bigMinBufferVal = -1; + bigMaxBufferVal = 0; } else { minBufferVal = MAX_BIGINT; maxBufferVal = MIN_BIGINT; + dataconvert::DataConvert::int128Max(bigMinBufferVal); + dataconvert::DataConvert::int128Min(bigMaxBufferVal); } } }; diff --git a/writeengine/bulk/we_colextinf.cpp b/writeengine/bulk/we_colextinf.cpp index 2e98dd63d..a020c7362 100644 --- a/writeengine/bulk/we_colextinf.cpp +++ b/writeengine/bulk/we_colextinf.cpp @@ -70,10 +70,11 @@ void ColExtInf::addFirstEntry( RID lastInputRow, // buffer is flushed), so we will not have an LBID for the 1st buffer for this // extent. //------------------------------------------------------------------------------ -void ColExtInf::addOrUpdateEntry( RID lastInputRow, - int64_t minVal, - int64_t maxVal, - ColDataType colDataType ) +template +void ColExtInf::addOrUpdateEntryTemplate( RID lastInputRow, + T minVal, T maxVal, + ColDataType colDataType, + int width ) { boost::mutex::scoped_lock lock(fMapMutex); @@ -91,30 +92,65 @@ void ColExtInf::addOrUpdateEntry( RID lastInputRow, // If all rows had null value for this column, then minVal will be // MAX_INT and maxVal will be MIN_INT (see getCPInfoForBRM()). - if (iter->second.fMinVal == LLONG_MIN) // init the range + __int128 bigMinValInit; + dataconvert::DataConvert::int128Max(bigMinValInit); + if ((iter->second.fMinVal == LLONG_MIN && width <= 8) || + (iter->second.fbigMinVal == bigMinValInit && width > 8)) // init the range { - iter->second.fMinVal = minVal; - iter->second.fMaxVal = maxVal; + if (width <= 8) + { + iter->second.fMinVal = minVal; + iter->second.fMaxVal = maxVal; + } + else + { + iter->second.fbigMinVal = minVal; + iter->second.fbigMaxVal = maxVal; + } } else // Update the range { if (isUnsigned(colDataType) || isCharType(colDataType)) { - if (static_cast(minVal) - < static_cast(iter->second.fMinVal)) - iter->second.fMinVal = minVal; + if (width <= 8) + { + if (static_cast(minVal) + < static_cast(iter->second.fMinVal)) + iter->second.fMinVal = minVal; - if (static_cast(maxVal) - > static_cast(iter->second.fMaxVal)) - iter->second.fMaxVal = maxVal; + if (static_cast(maxVal) + > static_cast(iter->second.fMaxVal)) + iter->second.fMaxVal = maxVal; + } + else + { + if (static_cast(minVal) + < static_cast(iter->second.fbigMinVal)) + iter->second.fbigMinVal = minVal; + + if (static_cast(maxVal) + > static_cast(iter->second.fbigMaxVal)) + iter->second.fbigMaxVal = maxVal; + } } else { - if (minVal < iter->second.fMinVal) - iter->second.fMinVal = minVal; + if (width <= 8) + { + if (minVal < iter->second.fMinVal) + iter->second.fMinVal = minVal; - if (maxVal > iter->second.fMaxVal) - iter->second.fMaxVal = maxVal; + if (maxVal > iter->second.fMaxVal) + iter->second.fMaxVal = maxVal; + } + else + { + if (minVal < iter->second.fbigMinVal) + iter->second.fbigMinVal = minVal; + + if (maxVal > iter->second.fbigMaxVal) + iter->second.fbigMaxVal = maxVal; + } } } } @@ -178,6 +214,8 @@ void ColExtInf::getCPInfoForBRM( JobColumn column, BRMReporter& brmReporter ) // if applicable (indicating an extent with no non-NULL values). int64_t minVal = iter->second.fMinVal; int64_t maxVal = iter->second.fMaxVal; + __int128 bigMinVal = iter->second.fbigMinVal; + __int128 bigMaxVal = iter->second.fbigMaxVal; if ( bIsChar ) { @@ -198,6 +236,7 @@ void ColExtInf::getCPInfoForBRM( JobColumn column, BRMReporter& brmReporter ) // Log for now; may control with debug flag later //if (fLog->isDebug( DEBUG_1 )) + // TODO MCOL-641 Add support here. { std::ostringstream oss; oss << "Saving CP update for OID-" << fColOid << @@ -232,11 +271,20 @@ void ColExtInf::getCPInfoForBRM( JobColumn column, BRMReporter& brmReporter ) BRM::CPInfoMerge cpInfoMerge; cpInfoMerge.startLbid = iter->second.fLbid; - cpInfoMerge.max = maxVal; - cpInfoMerge.min = minVal; + if (column.width <= 8) + { + cpInfoMerge.max = maxVal; + cpInfoMerge.min = minVal; + } + else + { + cpInfoMerge.bigMax = bigMaxVal; + cpInfoMerge.bigMin = bigMinVal; + } cpInfoMerge.seqNum = -1; // Not used by mergeExtentsMaxMin cpInfoMerge.type = column.dataType; cpInfoMerge.newExtent = iter->second.fNewExtent; + cpInfoMerge.colWidth = column.width; brmReporter.addToCPInfo( cpInfoMerge ); ++iter; diff --git a/writeengine/bulk/we_colextinf.h b/writeengine/bulk/we_colextinf.h index acc1a59bd..e7215c5c4 100644 --- a/writeengine/bulk/we_colextinf.h +++ b/writeengine/bulk/we_colextinf.h @@ -40,6 +40,7 @@ #include "brmtypes.h" #include "we_type.h" +#include "dataconvert.h" namespace WriteEngine { @@ -62,14 +63,22 @@ public: ColExtInfEntry() : fLbid(INVALID_LBID), fMinVal(LLONG_MIN), fMaxVal(LLONG_MIN), - fNewExtent(true) { } + fNewExtent(true) + { + dataconvert::DataConvert::int128Min(fbigMaxVal); + dataconvert::DataConvert::int128Max(fbigMinVal); + } // Used to create entry for an existing extent we are going to add data to. ColExtInfEntry(BRM::LBID_t lbid, bool bIsNewExtent) : fLbid(lbid), fMinVal(LLONG_MIN), fMaxVal(LLONG_MIN), - fNewExtent(bIsNewExtent) { } + fNewExtent(bIsNewExtent) + { + dataconvert::DataConvert::int128Min(fbigMaxVal); + dataconvert::DataConvert::int128Max(fbigMinVal); + } // Used to create entry for a new extent, with LBID not yet allocated ColExtInfEntry(int64_t minVal, int64_t maxVal) : @@ -78,6 +87,13 @@ public: fMaxVal(maxVal), fNewExtent(true) { } + // Used to create entry for a new extent, with LBID not yet allocated + ColExtInfEntry(__int128 bigMinVal, __int128 bigMaxVal) : + fLbid(INVALID_LBID), + fNewExtent(true), + fbigMinVal(bigMinVal), + fbigMaxVal(bigMaxVal) { } + // Used to create entry for a new extent, with LBID not yet allocated ColExtInfEntry(uint64_t minVal, uint64_t maxVal) : fLbid(INVALID_LBID), @@ -85,10 +101,27 @@ public: fMaxVal(static_cast(maxVal)), fNewExtent(true) { } + // Used to create entry for a new extent, with LBID not yet allocated + ColExtInfEntry(unsigned __int128 bigMinVal, unsigned __int128 bigMaxVal) : + fLbid(INVALID_LBID), + fNewExtent(true), + fbigMinVal(static_cast<__int128>(bigMinVal)), + fbigMaxVal(static_cast<__int128>(bigMaxVal)) { } + BRM::LBID_t fLbid; // LBID for an extent; should be the starting LBID int64_t fMinVal; // minimum value for extent associated with LBID int64_t fMaxVal; // maximum value for extent associated with LBID bool fNewExtent;// is this a new extent + union + { + __int128 fbigMinVal; + int64_t fMinVal_; + }; + union + { + __int128 fbigMaxVal; + int64_t fMaxVal_; + }; }; //------------------------------------------------------------------------------ @@ -122,7 +155,14 @@ public: virtual void addOrUpdateEntry( RID lastInputRow, int64_t minVal, int64_t maxVal, - ColDataType colDataType ) { } + ColDataType colDataType, + int width ) { } + + virtual void addOrUpdateEntry( RID lastInputRow, + __int128 minVal, + __int128 maxVal, + ColDataType colDataType, + int width ) { } virtual void getCPInfoForBRM ( JobColumn column, BRMReporter& brmReporter) { } @@ -173,10 +213,33 @@ public: * @param minVal Minimum value for the latest buffer read * @param maxVal Maximum value for the latest buffer read */ + template + void addOrUpdateEntryTemplate( RID lastInputRow, + T minVal, T maxVal, + ColDataType colDataType, + int width ); + virtual void addOrUpdateEntry( RID lastInputRow, - int64_t minVal, - int64_t maxVal, - ColDataType colDataType ); + int64_t minVal, int64_t maxVal, + ColDataType colDataType, + int width ) + { + addOrUpdateEntryTemplate( lastInputRow, + minVal, maxVal, + colDataType, + width ); + } + + virtual void addOrUpdateEntry( RID lastInputRow, + __int128 minVal, __int128 maxVal, + ColDataType colDataType, + int width ) + { + addOrUpdateEntryTemplate( lastInputRow, + minVal, maxVal, + colDataType, + width ); + } /** @brief Send updated Casual Partition (CP) info to BRM. */ diff --git a/writeengine/bulk/we_columninfo.h b/writeengine/bulk/we_columninfo.h index 15ded5310..f3bb8f23e 100644 --- a/writeengine/bulk/we_columninfo.h +++ b/writeengine/bulk/we_columninfo.h @@ -294,10 +294,11 @@ public: /** @brief Update extent CP information */ + template void updateCPInfo( RID lastInputRow, - int64_t minVal, - int64_t maxVal, - ColDataType colDataType ); + T minVal, T maxVal, + ColDataType colDataType, + int width ); /** @brief Setup initial extent we will begin loading at start of import. * @param dbRoot DBRoot of starting extent @@ -567,13 +568,15 @@ inline unsigned ColumnInfo::rowsPerExtent() return fRowsPerExtent; } +template inline void ColumnInfo::updateCPInfo( RID lastInputRow, - int64_t minVal, - int64_t maxVal, - ColDataType colDataType ) + T minVal, + T maxVal, + ColDataType colDataType, + int width ) { - fColExtInf->addOrUpdateEntry( lastInputRow, minVal, maxVal, colDataType ); + fColExtInf->addOrUpdateEntry( lastInputRow, minVal, maxVal, colDataType, width ); } } // end of namespace diff --git a/writeengine/shared/we_brm.h b/writeengine/shared/we_brm.h index 23dfe16ce..897c961c4 100644 --- a/writeengine/shared/we_brm.h +++ b/writeengine/shared/we_brm.h @@ -649,7 +649,7 @@ inline int BRMWrapper::bulkSetHWMAndCP( inline int BRMWrapper::setExtentsMaxMin(const BRM::CPInfoList_t& cpinfoList) { - int rc = blockRsltnMgrPtr->setExtentsMaxMin( cpinfoList ); + int rc = blockRsltnMgrPtr->setExtentsMaxMin(cpinfoList); return getRC( rc, ERR_BRM_SET_EXTENTS_CP ); } diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index 8a92b9d61..24ce85d0f 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -39,6 +39,7 @@ using namespace std; using namespace execplan; +#include "dataconvert.h" #include "IDBDataFile.h" #include "IDBPolicy.h" @@ -1036,8 +1037,18 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi } } - cpInfo.max = nextValStart + nexValNeeded - 1; - cpInfo.min = nextValStart; + cpInfo.isBinaryColumn = column.colWidth > 8; + + if (!cpInfo.isBinaryColumn) + { + cpInfo.max = nextValStart + nexValNeeded - 1; + cpInfo.min = nextValStart; + } + else + { + cpInfo.bigMax = nextValStart + nexValNeeded - 1; + cpInfo.bigMin = nextValStart; + } cpInfo.seqNum = 0; } @@ -1092,15 +1103,33 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi } } - if (isUnsigned(column.colDataType)) + cpInfo.isBinaryColumn = column.colWidth > 8; + + if (!cpInfo.isBinaryColumn) { - cpInfo.max = 0; - cpInfo.min = static_cast(numeric_limits::max()); + if (isUnsigned(column.colDataType)) + { + cpInfo.max = 0; + cpInfo.min = static_cast(numeric_limits::max()); + } + else + { + cpInfo.max = numeric_limits::min(); + cpInfo.min = numeric_limits::max(); + } } else { - cpInfo.max = numeric_limits::min(); - cpInfo.min = numeric_limits::max(); + if (isUnsigned(column.colDataType)) + { + cpInfo.bigMax = 0; + cpInfo.min = -1; + } + else + { + dataconvert::DataConvert::int128Min(cpInfo.bigMax); + dataconvert::DataConvert::int128Max(cpInfo.bigMin); + } } cpInfo.seqNum = -1; @@ -1127,16 +1156,33 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi if (autoincrement) //@Bug 4074. Mark it invalid first to set later { BRM::CPInfo cpInfo1; + cpInfo1.isBinaryColumn = column.colWidth > 8; - if (isUnsigned(column.colDataType)) + if (!cpInfo1.isBinaryColumn) { - cpInfo1.max = 0; - cpInfo1.min = static_cast(numeric_limits::max()); + if (isUnsigned(column.colDataType)) + { + cpInfo1.max = 0; + cpInfo1.min = static_cast(numeric_limits::max()); + } + else + { + cpInfo1.max = numeric_limits::min(); + cpInfo1.min = numeric_limits::max(); + } } else { - cpInfo1.max = numeric_limits::min(); - cpInfo1.min = numeric_limits::max(); + if (isUnsigned(column.colDataType)) + { + cpInfo1.bigMax = 0; + cpInfo1.bigMin = -1; + } + else + { + dataconvert::DataConvert::int128Min(cpInfo1.bigMax); + dataconvert::DataConvert::int128Max(cpInfo1.bigMin); + } } cpInfo1.seqNum = -1; @@ -1152,13 +1198,11 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi BRM::CPInfoList_t cpinfoList; cpInfo.firstLbid = startLbid; cpinfoList.push_back(cpInfo); - //cout << "calling setExtentsMaxMin for startLbid = " << startLbid << endl; rc = BRMWrapper::getInstance()->setExtentsMaxMin(cpinfoList); if ( rc != NO_ERROR) return rc; - //cout << "calling setLocalHWM for oid:hwm = " << column.dataFile.fid <<":"<setLocalHWM((OID)column.dataFile.fid, column.dataFile.fPartition, column.dataFile.fSegment, colHwm); diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 7296654b6..29edee9d4 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -57,6 +57,7 @@ using namespace execplan; #include "IDBPolicy.h" #include "MonitorProcMem.h" using namespace idbdatafile; +#include "dataconvert.h" #ifdef _MSC_VER #define isnan _isnan @@ -1505,8 +1506,8 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, //-------------------------------------------------------------------------- if (isFirstBatchPm) { - currentDBrootIdx = dbRootExtentTrackers[colId]->getCurrentDBRootIdx(); - extentInfo = dbRootExtentTrackers[colId]->getDBRootExtentList(); + currentDBrootIdx = dbRootExtentTrackers[colId]->getCurrentDBRootIdx(); + extentInfo = dbRootExtentTrackers[colId]->getDBRootExtentList(); dbRoot = extentInfo[currentDBrootIdx].fDbRoot; partitionNum = extentInfo[currentDBrootIdx].fPartition; @@ -1536,24 +1537,11 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, BRM::CPInfoList_t cpinfoList; BRM::CPInfo cpInfo; - if (isUnsigned(colStructList[i].colDataType)) - { - cpInfo.max = 0; - cpInfo.min = static_cast(numeric_limits::max()); - } - else - { - cpInfo.max = numeric_limits::min(); - cpInfo.min = numeric_limits::max(); - } - - cpInfo.seqNum = -1; - for ( i = 0; i < extents.size(); i++) { colOp = m_colOp[op(colStructList[i].fCompressionType)]; colOp->initColumn(curCol); - colOp->setColParam(curCol, colId, colStructList[i].colWidth, colStructList[i].colDataType, + colOp->setColParam(curCol, colId, colStructList[i].colWidth, colStructList[i].colDataType, colStructList[i].colType, colStructList[i].dataOid, colStructList[i].fCompressionType, dbRoot, partitionNum, segmentNum); rc = colOp->extendColumn(curCol, false, extents[i].startBlkOffset, extents[i].startLbid, extents[i].allocSize, dbRoot, @@ -1562,6 +1550,37 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, if (rc != NO_ERROR) return rc; + cpInfo.isBinaryColumn = colStructList[i].colWidth > 8; + + if (!cpInfo.isBinaryColumn) + { + if (isUnsigned(colStructList[i].colDataType)) + { + cpInfo.max = 0; + cpInfo.min = static_cast(numeric_limits::max()); + } + else + { + cpInfo.max = numeric_limits::min(); + cpInfo.min = numeric_limits::max(); + } + } + else + { + if (isUnsigned(colStructList[i].colDataType)) + { + cpInfo.bigMax = 0; + cpInfo.bigMin = -1; + } + else + { + dataconvert::DataConvert::int128Min(cpInfo.bigMax); + dataconvert::DataConvert::int128Max(cpInfo.bigMin); + } + } + + cpInfo.seqNum = -1; + //mark the extents to invalid cpInfo.firstLbid = extents[i].startLbid; cpinfoList.push_back(cpInfo); @@ -2278,19 +2297,6 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, BRM::CPInfoList_t cpinfoList; BRM::CPInfo cpInfo; - if (isUnsigned(colStructList[i].colDataType)) - { - cpInfo.max = 0; - cpInfo.min = static_cast(numeric_limits::max()); - } - else - { - cpInfo.max = numeric_limits::min(); - cpInfo.min = numeric_limits::max(); - } - - cpInfo.seqNum = -1; - for ( i = 0; i < extents.size(); i++) { colOp = m_colOp[op(colStructList[i].fCompressionType)]; @@ -2304,6 +2310,37 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, if (rc != NO_ERROR) return rc; + cpInfo.isBinaryColumn = colStructList[i].colWidth > 8; + + if (!cpInfo.isBinaryColumn) + { + if (isUnsigned(colStructList[i].colDataType)) + { + cpInfo.max = 0; + cpInfo.min = static_cast(numeric_limits::max()); + } + else + { + cpInfo.max = numeric_limits::min(); + cpInfo.min = numeric_limits::max(); + } + } + else + { + if (isUnsigned(colStructList[i].colDataType)) + { + cpInfo.bigMax = 0; + cpInfo.bigMin = -1; + } + else + { + dataconvert::DataConvert::int128Min(cpInfo.bigMax); + dataconvert::DataConvert::int128Max(cpInfo.bigMin); + } + } + + cpInfo.seqNum = -1; + //mark the extents to invalid cpInfo.firstLbid = extents[i].startLbid; cpinfoList.push_back(cpInfo); From 93170c3b316e3a85aed2f10983467db7e9a55a8a Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Mon, 17 Feb 2020 19:52:05 -0500 Subject: [PATCH 13/78] MCOL-641 Basic support for multi-value inserts, and deletes. --- dbcon/ddlpackageproc/ddlpackageprocessor.cpp | 13 +- dbcon/dmlpackageproc/dmlpackageprocessor.h | 2 +- primitives/linux-port/column.cpp | 4 +- writeengine/server/we_ddlcommandproc.cpp | 117 +- writeengine/server/we_dmlcommandproc.cpp | 36 +- writeengine/server/we_dmlcommandproc.h | 4 +- writeengine/shared/we_blockop.cpp | 5 +- writeengine/shared/we_convertor.cpp | 2 +- writeengine/wrapper/we_colop.cpp | 32 +- writeengine/wrapper/writeengine.cpp | 1554 +----------------- writeengine/wrapper/writeengine.h | 48 +- 11 files changed, 266 insertions(+), 1551 deletions(-) diff --git a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp index b26392b0c..97712bf38 100644 --- a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp +++ b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp @@ -1550,17 +1550,20 @@ void DDLPackageProcessor::updateSyscolumns(execplan::CalpontSystemCatalog::SCN t if (result.result != NO_ERROR) return; - WriteEngine::ColStructList colStructs; + WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; //std::vector colStructs; WriteEngine::ColStruct colStruct; + execplan::CalpontSystemCatalog::ColType colType; WriteEngine::DctnryStructList dctnryStructList; WriteEngine::DctnryValueList dctnryValueList; //Build column structure for COLUMNPOS_COL - colStruct.dataOid = OID_SYSCOLUMN_COLUMNPOS; - colStruct.colWidth = 4; + colType.columnOID = colStruct.dataOid = OID_SYSCOLUMN_COLUMNPOS; + colType.colWidth = colStruct.colWidth = 4; colStruct.tokenFlag = false; - colStruct.colDataType = CalpontSystemCatalog::INT; + colType.colDataType = colStruct.colDataType = CalpontSystemCatalog::INT; colStructs.push_back(colStruct); + cscColTypeList.push_back(colType); int error; std::string err; std::vector colOldValuesList1; @@ -1568,7 +1571,7 @@ void DDLPackageProcessor::updateSyscolumns(execplan::CalpontSystemCatalog::SCN t try { //@Bug 3051 use updateColumnRecs instead of updateColumnRec to use different value for diffrent rows. - if (NO_ERROR != (error = fWriteEngine.updateColumnRecs( txnID, colStructs, colValuesList, ridList ))) + if (NO_ERROR != (error = fWriteEngine.updateColumnRecs( txnID, cscColTypeList, colStructs, colValuesList, ridList ))) { // build the logging message WErrorCodes ec; diff --git a/dbcon/dmlpackageproc/dmlpackageprocessor.h b/dbcon/dmlpackageproc/dmlpackageprocessor.h index cc213c347..05c428781 100644 --- a/dbcon/dmlpackageproc/dmlpackageprocessor.h +++ b/dbcon/dmlpackageproc/dmlpackageprocessor.h @@ -478,7 +478,7 @@ protected: { if (((colType.colDataType == execplan::CalpontSystemCatalog::CHAR) && (colType.colWidth > 8)) || ((colType.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7)) - || ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 18)) + || ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 38)) || (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY) || (colType.colDataType == execplan::CalpontSystemCatalog::BLOB) || (colType.colDataType == execplan::CalpontSystemCatalog::TEXT)) diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 4d19940e5..c8fd6673d 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -286,9 +286,7 @@ inline bool isEmptyVal<16>(uint8_t type, const uint8_t* ival) // For BINARY { const uint64_t* val = reinterpret_cast(ival); // WIP ugly speed hack - return (((val[0] == joblist::BINARYEMPTYROW) && (val[1] == joblist::BINARYEMPTYROW)) - || ((val[0] == joblist::BIGINTEMPTYROW) && (val[1] == joblist::BIGINTEMPTYROW))) -; + return ((val[0] == joblist::BINARYEMPTYROW) && (val[1] == joblist::BINARYEMPTYROW)); } diff --git a/writeengine/server/we_ddlcommandproc.cpp b/writeengine/server/we_ddlcommandproc.cpp index 988bb340a..8356ffc22 100644 --- a/writeengine/server/we_ddlcommandproc.cpp +++ b/writeengine/server/we_ddlcommandproc.cpp @@ -140,6 +140,7 @@ uint8_t WE_DDLCommandProc::writeSystable(ByteStream& bs, std::string& err) WriteEngine::ColTuple colTuple; WriteEngine::ColStruct colStruct; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; WriteEngine::ColTupleList colTuples; WriteEngine::dictStr dctColTuples; WriteEngine::DctnryStruct dctnryStruct; @@ -255,6 +256,7 @@ uint8_t WE_DDLCommandProc::writeSystable(ByteStream& bs, std::string& err) } colStructs.push_back(colStruct); + cscColTypeList.push_back(column.colType); oids[colStruct.dataOid] = colStruct.dataOid; //oidsToFlush.push_back(colStruct.dataOid); @@ -293,7 +295,7 @@ uint8_t WE_DDLCommandProc::writeSystable(ByteStream& bs, std::string& err) // TODO: This may be redundant static boost::mutex dbrmMutex; boost::mutex::scoped_lock lk(dbrmMutex); - error = fWEWrapper.insertColumnRec_SYS(txnID, colStructs, colValuesList, + error = fWEWrapper.insertColumnRec_SYS(txnID, cscColTypeList, colStructs, colValuesList, dctnryStructList, dctnryValueList, SYSCOLUMN_BASE); if (error != WriteEngine::NO_ERROR) @@ -392,6 +394,7 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err WriteEngine::ColTuple colTuple; WriteEngine::ColStruct colStruct; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; WriteEngine::ColTupleList colTuples; WriteEngine::dictStr dctColTuples; WriteEngine::DctnryStruct dctnryStruct; @@ -702,6 +705,7 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err { colStructs.push_back(colStruct); dctnryStructList.push_back (dctnryStruct); + cscColTypeList.push_back(column.colType); } colList[i].push_back(colTuple); @@ -733,7 +737,7 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err } //fWEWrapper.setDebugLevel(WriteEngine::DEBUG_3); - error = fWEWrapper.insertColumnRec_SYS(txnID, colStructs, colValuesList, + error = fWEWrapper.insertColumnRec_SYS(txnID, cscColTypeList, colStructs, colValuesList, dctnryStructList, dctnryValueList, SYSCOLUMN_BASE); if (idbdatafile::IDBPolicy::useHdfs()) @@ -819,6 +823,7 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string& err) WriteEngine::ColStruct colStruct; WriteEngine::ColTuple colTuple; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; WriteEngine::ColTupleList colTuples; WriteEngine::DctColTupleList dctColTuples; WriteEngine::ColValueList colValuesList; @@ -1091,6 +1096,7 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string& err) colStructs.push_back(colStruct); dctnryStructList.push_back (dctnryStruct); + cscColTypeList.push_back(column.colType); colList[i].push_back(colTuple); //colList.push_back(WriteEngine::ColTupleList()); //colList.back().push_back(colTuple); @@ -1118,7 +1124,7 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string& err) fWEWrapper.startTransaction(txnID); int rc1 = 0; - error = fWEWrapper.insertColumnRec_SYS(txnID, colStructs, colValuesList, + error = fWEWrapper.insertColumnRec_SYS(txnID, cscColTypeList, colStructs, colValuesList, dctnryStructList, dctnryValueList, SYSCOLUMN_BASE); if (idbdatafile::IDBPolicy::useHdfs()) @@ -1368,7 +1374,9 @@ uint8_t WE_DDLCommandProc::deleteSyscolumn(ByteStream& bs, std::string& err) WriteEngine::ColStruct colStruct; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colExtentsStruct; + std::vector colExtentsColType; std::vector colValuesList; WriteEngine::RIDList ridList; std::vector ridLists; @@ -1403,17 +1411,19 @@ uint8_t WE_DDLCommandProc::deleteSyscolumn(ByteStream& bs, std::string& err) oids[colStruct.dataOid] = colStruct.dataOid; //oidsToFlush.push_back(colStruct.dataOid); colStructs.push_back(colStruct); + cscColTypeList.push_back(column.colType); ++column_iterator; } colExtentsStruct.push_back(colStructs); + colExtentsColType.push_back(cscColTypeList); ridLists.push_back(ridList); if (0 != colStructs.size() && 0 != ridLists[0].size()) { - int error = fWEWrapper.deleteRow(txnID, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); + int error = fWEWrapper.deleteRow(txnID, colExtentsColType, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); int rc1 = 0; @@ -1508,7 +1518,9 @@ uint8_t WE_DDLCommandProc::deleteSyscolumnRow(ByteStream& bs, std::string& err) WriteEngine::ColStruct colStruct; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colExtentsStruct; + std::vector colExtentsColType; std::vector colValuesList; WriteEngine::RIDList ridList; std::vector ridLists; @@ -1537,17 +1549,20 @@ uint8_t WE_DDLCommandProc::deleteSyscolumnRow(ByteStream& bs, std::string& err) oids[colStruct.dataOid] = colStruct.dataOid; //oidsToFlush.push_back(colStruct.dataOid); colStructs.push_back(colStruct); + cscColTypeList.push_back(column.colType); ++column_iterator; } colExtentsStruct.push_back(colStructs); + colExtentsColType.push_back(cscColTypeList); ridLists.push_back(ridList); if (0 != colStructs.size() && 0 != ridLists[0].size()) { - int error = fWEWrapper.deleteRow(txnID, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); + int error = fWEWrapper.deleteRow(txnID, colExtentsColType, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); + int rc1 = 0; if (idbdatafile::IDBPolicy::useHdfs()) @@ -1644,7 +1659,9 @@ uint8_t WE_DDLCommandProc::deleteSystable(ByteStream& bs, std::string& err) WriteEngine::ColStruct colStruct; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colExtentsStruct; + std::vector colExtentsColType; std::vector colValuesList; WriteEngine::RIDList ridList; std::vector ridLists; @@ -1672,17 +1689,20 @@ uint8_t WE_DDLCommandProc::deleteSystable(ByteStream& bs, std::string& err) oids[colStruct.dataOid] = colStruct.dataOid; //oidsToFlush.push_back(colStruct.dataOid); colStructs.push_back(colStruct); + cscColTypeList.push_back(column.colType); ++column_iterator; } colExtentsStruct.push_back(colStructs); + colExtentsColType.push_back(cscColTypeList); ridLists.push_back(ridList); if (0 != colStructs.size() && 0 != ridLists[0].size()) { - int error = fWEWrapper.deleteRow(txnID, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); + int error = fWEWrapper.deleteRow(txnID, colExtentsColType, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); + int rc1 = 0; if (idbdatafile::IDBPolicy::useHdfs()) @@ -1755,7 +1775,9 @@ uint8_t WE_DDLCommandProc::deleteSystables(ByteStream& bs, std::string& err) systemCatalogPtr->identity(CalpontSystemCatalog::EC); WriteEngine::ColStruct colStruct; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colExtentsStruct; + std::vector colExtentsColType; std::vector colValuesList; WriteEngine::RIDList ridList; std::vector ridLists; @@ -1804,16 +1826,19 @@ uint8_t WE_DDLCommandProc::deleteSystables(ByteStream& bs, std::string& err) oids[colStruct.dataOid] = colStruct.dataOid; //oidsToFlush.push_back(colStruct.dataOid); colStructs.push_back(colStruct); + cscColTypeList.push_back(column.colType); ++column_iterator; } colExtentsStruct.push_back(colStructs); + colExtentsColType.push_back(cscColTypeList); ridLists.push_back(ridList); { - int error = fWEWrapper.deleteRow(txnID, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); + int error = fWEWrapper.deleteRow(txnID, colExtentsColType, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); + int rc1 = 0; if (idbdatafile::IDBPolicy::useHdfs()) @@ -1867,7 +1892,9 @@ uint8_t WE_DDLCommandProc::deleteSystables(ByteStream& bs, std::string& err) CalpontSystemCatalog::RIDList colRidList = systemCatalogPtr->columnRIDs(userTableName); colStructs.clear(); + cscColTypeList.clear(); colExtentsStruct.clear(); + colExtentsColType.clear(); colValuesList.clear(); ridList.clear(); ridLists.clear(); @@ -1902,17 +1929,20 @@ uint8_t WE_DDLCommandProc::deleteSystables(ByteStream& bs, std::string& err) colStructs.push_back(colStruct); oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column.colType); //oidsToFlush.push_back(colStruct.dataOid); ++column_iterator; } colExtentsStruct.push_back(colStructs); + colExtentsColType.push_back(cscColTypeList); ridLists.push_back(ridList); if (0 != colStructs.size() && 0 != ridLists[0].size()) { - int error = fWEWrapper.deleteRow(txnID, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); + int error = fWEWrapper.deleteRow(txnID, colExtentsColType, colExtentsStruct, colValuesList, ridLists, SYSCOLUMN_BASE); + int rc1 = 0; if (idbdatafile::IDBPolicy::useHdfs()) @@ -2035,6 +2065,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnAuto(ByteStream& bs, std::string& err) WriteEngine::ColValueList colValuesList; WriteEngine::ColTupleList aColList; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colOldValuesList; std::map oids; //std::vector oidsToFlush; @@ -2075,6 +2106,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnAuto(ByteStream& bs, std::string& err) oids[colStruct.dataOid] = colStruct.dataOid; //oidsToFlush.push_back(colStruct.dataOid); dctnryStructList.push_back(dctnryStruct); + cscColTypeList.push_back(column.colType); for (unsigned int i = 0; i < roList.size(); i++) { @@ -2083,6 +2115,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnAuto(ByteStream& bs, std::string& err) colValuesList.push_back(aColList); std::vector colExtentsStruct; + std::vector colExtentsColType; std::vector dctnryExtentsStruct; std::vector extentsinfo; extentInfo aExtentinfo; @@ -2125,13 +2158,14 @@ uint8_t WE_DDLCommandProc::updateSyscolumnAuto(ByteStream& bs, std::string& err) colExtentsStruct.push_back(colStructs); dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); } // call the write engine to update the row if (idbdatafile::IDBPolicy::useHdfs()) fWEWrapper.startTransaction(txnID); - rc = fWEWrapper.updateColumnRec(txnID, colExtentsStruct, colValuesList, colOldValuesList, + rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); if (rc != NO_ERROR) @@ -2226,6 +2260,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnNextvalCol(ByteStream& bs, std::string WriteEngine::ColValueList colValuesList; WriteEngine::ColTupleList aColList; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colOldValuesList; std::map oids; //std::vector oidsToFlush; @@ -2259,6 +2294,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnNextvalCol(ByteStream& bs, std::string //oidsToFlush.push_back(colStruct.dataOid); colStructs.push_back(colStruct); dctnryStructList.push_back(dctnryStruct); + cscColTypeList.push_back(column.colType); for (unsigned int i = 0; i < roList.size(); i++) { @@ -2296,6 +2332,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnNextvalCol(ByteStream& bs, std::string std::vector ridLists; std::vector colExtentsStruct; + std::vector colExtentsColType; std::vector dctnryExtentsStruct; ridLists.push_back(ridList); @@ -2314,13 +2351,14 @@ uint8_t WE_DDLCommandProc::updateSyscolumnNextvalCol(ByteStream& bs, std::string colExtentsStruct.push_back(colStructs); dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); } // call the write engine to update the row fWEWrapper.setTransId(txnID); fWEWrapper.startTransaction(txnID); - rc = fWEWrapper.updateColumnRec(txnID, colExtentsStruct, colValuesList, colOldValuesList, + rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); if (rc != NO_ERROR) @@ -2396,6 +2434,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnTablename(ByteStream& bs, std::string& WriteEngine::ColValueList colValuesList; WriteEngine::ColTupleList aColList; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colOldValuesList; tableName.schema = CALPONT_SCHEMA; tableName.table = SYSCOLUMN_TABLE; @@ -2463,6 +2502,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnTablename(ByteStream& bs, std::string& colStructs.push_back(colStruct); dctnryStructList.push_back(dctnryStruct); + cscColTypeList.push_back(column.colType); for (unsigned int i = 0; i < roList.size(); i++) { @@ -2485,6 +2525,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnTablename(ByteStream& bs, std::string& std::vector extentsinfo; extentInfo aExtentinfo; std::vector colExtentsStruct; + std::vector colExtentsColType; std::vector dctnryExtentsStruct; for (unsigned int i = 0; i < roList.size(); i++) @@ -2521,6 +2562,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnTablename(ByteStream& bs, std::string& colExtentsStruct.push_back(colStructs); dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); } // call the write engine to update the row @@ -2529,7 +2571,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnTablename(ByteStream& bs, std::string& fWEWrapper.setBulkFlag(false); fWEWrapper.startTransaction(txnID); - rc = fWEWrapper.updateColumnRec(txnID, colExtentsStruct, colValuesList, colOldValuesList, + rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); if (rc != NO_ERROR) @@ -2625,6 +2667,7 @@ uint8_t WE_DDLCommandProc::updateSystableAuto(ByteStream& bs, std::string& err) WriteEngine::ColValueList colValuesList; WriteEngine::ColTupleList aColList; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colOldValuesList; std::map oids; //std::vector oidsToFlush; @@ -2658,12 +2701,14 @@ uint8_t WE_DDLCommandProc::updateSystableAuto(ByteStream& bs, std::string& err) } colStructs.push_back(colStruct); + cscColTypeList.push_back(column.colType); oids[colStruct.dataOid] = colStruct.dataOid; //oidsToFlush.push_back(colStruct.dataOid); dctnryStructList.push_back(dctnryStruct); aColList.push_back(colTuple); colValuesList.push_back(aColList); std::vector colExtentsStruct; + std::vector colExtentsColType; std::vector dctnryExtentsStruct; @@ -2701,6 +2746,7 @@ uint8_t WE_DDLCommandProc::updateSystableAuto(ByteStream& bs, std::string& err) } colExtentsStruct.push_back(colStructs); + colExtentsColType.push_back(cscColTypeList); dctnryExtentsStruct.push_back(dctnryStructList); } @@ -2710,7 +2756,7 @@ uint8_t WE_DDLCommandProc::updateSystableAuto(ByteStream& bs, std::string& err) fWEWrapper.setBulkFlag(false); fWEWrapper.startTransaction(txnID); - rc = fWEWrapper.updateColumnRec(txnID, colExtentsStruct, colValuesList, colOldValuesList, + rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); if (rc != NO_ERROR) @@ -2804,6 +2850,7 @@ uint8_t WE_DDLCommandProc::updateSystableTablename(ByteStream& bs, std::string& WriteEngine::ColValueList colValuesList; WriteEngine::ColTupleList aColList; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colOldValuesList; std::map oids; //std::vector oidsToFlush; @@ -2853,6 +2900,7 @@ uint8_t WE_DDLCommandProc::updateSystableTablename(ByteStream& bs, std::string& colStructs.push_back(colStruct); dctnryStructList.push_back(dctnryStruct); oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column.colType); //oidsToFlush.push_back(colStruct.dataOid); if (dctnryStruct.dctnryOid > 0) @@ -2865,6 +2913,7 @@ uint8_t WE_DDLCommandProc::updateSystableTablename(ByteStream& bs, std::string& colValuesList.push_back(aColList); std::vector colExtentsStruct; std::vector dctnryExtentsStruct; + std::vector colExtentsColType; dctColList = dictTuple; dctRowList.push_back(dctColList); @@ -2900,6 +2949,7 @@ uint8_t WE_DDLCommandProc::updateSystableTablename(ByteStream& bs, std::string& colExtentsStruct.push_back(colStructs); dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); } // call the write engine to update the row @@ -2908,7 +2958,7 @@ uint8_t WE_DDLCommandProc::updateSystableTablename(ByteStream& bs, std::string& fWEWrapper.setBulkFlag(false); fWEWrapper.startTransaction(txnID); - rc = fWEWrapper.updateColumnRec(txnID, colExtentsStruct, colValuesList, colOldValuesList, + rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); if (rc != NO_ERROR) @@ -3032,6 +3082,7 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& WriteEngine::ColValueList colValuesList; WriteEngine::ColTupleList aColList; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colOldValuesList; std::map oids; //std::vector oidsToFlush; @@ -3090,6 +3141,7 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& colStructs.push_back(colStruct); dctnryStructList.push_back(dctnryStruct); oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column.colType); //oidsToFlush.push_back(colStruct.dataOid); if (dctnryStruct.dctnryOid > 0) @@ -3102,6 +3154,7 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& colValuesList.push_back(aColList); std::vector colExtentsStruct; std::vector dctnryExtentsStruct; + std::vector colExtentsColType; dctColList = dictTuple; dctRowList.push_back(dctColList); @@ -3137,6 +3190,7 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& colExtentsStruct.push_back(colStructs); dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); } // call the write engine to update the row @@ -3145,7 +3199,7 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& fWEWrapper.setBulkFlag(false); fWEWrapper.startTransaction(txnID); - rc = fWEWrapper.updateColumnRec(txnID, colExtentsStruct, colValuesList, colOldValuesList, + rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); if (rc != NO_ERROR) @@ -3207,6 +3261,7 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& colValuesList.clear(); aColList.clear(); colStructs.clear(); + cscColTypeList.clear(); colOldValuesList.clear(); oids.clear(); tableName.schema = CALPONT_SCHEMA; @@ -3287,6 +3342,7 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& colStructs.push_back(colStruct); dctnryStructList.push_back(dctnryStruct); + cscColTypeList.push_back(column.colType); for (unsigned int i = 0; i < roList.size(); i++) { @@ -3307,6 +3363,7 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& dctnryValueList.push_back(dctRowList); extentsinfo.clear(); colExtentsStruct.clear(); + colExtentsColType.clear(); dctnryExtentsStruct.clear(); oid = 1021; @@ -3344,10 +3401,11 @@ uint8_t WE_DDLCommandProc::updateSystablesTablename(ByteStream& bs, std::string& colExtentsStruct.push_back(colStructs); dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); } // call the write engine to update the row - rc = fWEWrapper.updateColumnRec(txnID, colExtentsStruct, colValuesList, colOldValuesList, + rc = fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE); if (rc != NO_ERROR) @@ -3481,11 +3539,13 @@ uint8_t WE_DDLCommandProc::updateSyscolumnColumnposCol(messageqcpp::ByteStream& WriteEngine::ColStruct colStruct; WriteEngine::DctnryStructList dctnryStructList; WriteEngine::DctnryValueList dctnryValueList; + WriteEngine::CSCTypesList cscColTypeList; + CalpontSystemCatalog::ColType colType; //Build column structure for COLUMNPOS_COL - colStruct.dataOid = OID_SYSCOLUMN_COLUMNPOS; - colStruct.colWidth = 4; + colType.columnOID = colStruct.dataOid = OID_SYSCOLUMN_COLUMNPOS; + colType.colWidth = colStruct.colWidth = 4; colStruct.tokenFlag = false; - colStruct.colDataType = CalpontSystemCatalog::INT; + colType.colDataType = colStruct.colDataType = CalpontSystemCatalog::INT; colStruct.fColDbRoot = dbRoot; if (idbdatafile::IDBPolicy::useHdfs()) @@ -3494,9 +3554,10 @@ uint8_t WE_DDLCommandProc::updateSyscolumnColumnposCol(messageqcpp::ByteStream& } colStructs.push_back(colStruct); + cscColTypeList.push_back(colType); oids[colStruct.dataOid] = colStruct.dataOid; //oidsToFlush.push_back(colStruct.dataOid); - rc = fWEWrapper.updateColumnRecs( txnID, colStructs, colValuesList, ridList, SYSCOLUMN_BASE ); + rc = fWEWrapper.updateColumnRecs( txnID, cscColTypeList, colStructs, colValuesList, ridList, SYSCOLUMN_BASE ); } int rc1 = 0; @@ -3592,7 +3653,7 @@ uint8_t WE_DDLCommandProc::fillNewColumn(ByteStream& bs, std::string& err) std::map oids; oids[dataOid] = dataOid; oids[refColOID] = refColOID; - rc = fWEWrapper.fillColumn(txnID, dataOid, dataType, dataWidth, defaultVal, refColOID, refColDataType, + rc = fWEWrapper.fillColumn(txnID, dataOid, colType, defaultVal, refColOID, refColDataType, refColWidth, refCompressionType, isNULL, compressionType, defaultValStr, dictOid, autoincrement); if ( rc != 0 ) @@ -4163,6 +4224,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnSetDefault(messageqcpp::ByteStream& bs WriteEngine::ColValueList colValuesList; WriteEngine::ColTupleList aColList1; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colOldValuesList; WriteEngine::DctnryStructList dctnryStructList; WriteEngine::DctnryValueList dctnryValueList; @@ -4268,6 +4330,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnSetDefault(messageqcpp::ByteStream& bs colStructs.push_back(colStruct); oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column.colType); //oidsToFlush.push_back(colStruct.dataOid); if (dctnryStruct.dctnryOid > 0) @@ -4299,6 +4362,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnSetDefault(messageqcpp::ByteStream& bs std::vector colExtentsStruct; std::vector dctnryExtentsStruct; + std::vector colExtentsColType; std::vector ridLists; ridLists.push_back(ridList); @@ -4329,11 +4393,12 @@ uint8_t WE_DDLCommandProc::updateSyscolumnSetDefault(messageqcpp::ByteStream& bs colExtentsStruct.push_back(colStructs); dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); } // call the write engine to update the row - if (NO_ERROR != fWEWrapper.updateColumnRec(txnID, colExtentsStruct, colValuesList, colOldValuesList, + if (NO_ERROR != fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE)) { err = "WE: Update failed on: " + atableName.table; @@ -4439,6 +4504,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& WriteEngine::ColValueList colValuesList; WriteEngine::ColTupleList aColList1; WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; std::vector colOldValuesList; std::map oids; //std::vector oidsToFlush; @@ -4557,6 +4623,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& colStructs.push_back(colStruct); oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column1.colType); //oidsToFlush.push_back(colStruct.dataOid); if (dctnryStruct.dctnryOid > 0) @@ -4594,6 +4661,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& colStructs.push_back(colStruct); oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column2.colType); //oidsToFlush.push_back(colStruct.dataOid); if (dctnryStruct.dctnryOid > 0) @@ -4627,6 +4695,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& colStructs.push_back(colStruct); oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column3.colType); //oidsToFlush.push_back(colStruct.dataOid); if (dctnryStruct.dctnryOid > 0) @@ -4661,6 +4730,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& colStructs.push_back(colStruct); oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column4.colType); //oidsToFlush.push_back(colStruct.dataOid); if (dctnryStruct.dctnryOid > 0) @@ -4770,6 +4840,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& colStructs.push_back(colStruct); dctnryStructList.push_back(dctnryStruct); oids[colStruct.dataOid] = colStruct.dataOid; + cscColTypeList.push_back(column5.colType); //oidsToFlush.push_back(colStruct.dataOid); if (dctnryStruct.dctnryOid > 0) @@ -4797,6 +4868,7 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& dctRowList.push_back(dctColList); dctnryValueList.push_back(dctRowList); std::vector colExtentsStruct; + std::vector colExtentsColType; std::vector dctnryExtentsStruct; std::vector ridLists; ridLists.push_back(ridList); @@ -4828,10 +4900,11 @@ uint8_t WE_DDLCommandProc::updateSyscolumnRenameColumn(messageqcpp::ByteStream& colExtentsStruct.push_back(colStructs); dctnryExtentsStruct.push_back(dctnryStructList); + colExtentsColType.push_back(cscColTypeList); } // call the write engine to update the row - if (NO_ERROR != fWEWrapper.updateColumnRec(txnID, colExtentsStruct, colValuesList, colOldValuesList, + if (NO_ERROR != fWEWrapper.updateColumnRec(txnID, colExtentsColType, colExtentsStruct, colValuesList, colOldValuesList, ridLists, dctnryExtentsStruct, dctnryValueList, SYSCOLUMN_BASE)) { err = "WE: Update failed on: " + atableName.table; diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index b5b00f81b..bbef08857 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -116,7 +116,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: RowList rows = tablePtr->get_RowList(); WriteEngine::ColStructList colStructs; - WriteEngine::CSCTypesList cscColTypes; + WriteEngine::CSCTypesList cscColTypeList; WriteEngine::DctnryStructList dctnryStructList; WriteEngine::DctnryValueList dctnryValueList; WriteEngine::ColValueList colValuesList; @@ -141,7 +141,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: Row* rowPtr = rows.at(0); ColumnList columns = rowPtr->get_ColumnList(); unsigned int numcols = rowPtr->get_NumberOfColumns(); - cscColTypes.reserve(numcols); + cscColTypeList.reserve(numcols); // WIP // We presume that DictCols number is low colStructs.reserve(numcols); @@ -159,7 +159,6 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: CalpontSystemCatalog::ColType colType; colType = systemCatalogPtr->colType(oid); - cscColTypes.push_back(colType); WriteEngine::ColStruct colStruct; colStruct.fColDbRoot = dbroot; WriteEngine::DctnryStruct dctnryStruct; @@ -169,7 +168,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: colStruct.fCompressionType = colType.compressionType; // Token - if ( isDictCol(colType) ) + if (isDictCol(colType) ) { // WIP Hardcoded value colStruct.colWidth = 8; @@ -199,6 +198,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: colStructs.push_back(colStruct); dctnryStructList.push_back(dctnryStruct); + cscColTypeList.push_back(colType); ++column_iterator; } @@ -538,7 +538,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: if (colValuesList[0].size() > 0) { if (NO_ERROR != - (error = fWEWrapper.insertColumnRec_Single(txnid.id, cscColTypes, colStructs, colValuesList, dctnryStructList, dicStringList, tableRoPair.objnum))) + (error = fWEWrapper.insertColumnRec_Single(txnid.id, cscColTypeList, colStructs, colValuesList, dctnryStructList, dicStringList, tableRoPair.objnum))) { if (error == ERR_BRM_DEAD_LOCK) { @@ -842,6 +842,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std:: bool isInsertSelect = insertPkg.get_isInsertSelect(); WriteEngine::ColStructList colStructs; + WriteEngine::CSCTypesList cscColTypeList; WriteEngine::DctnryStructList dctnryStructList; WriteEngine::DctnryValueList dctnryValueList; WriteEngine::ColValueList colValuesList; @@ -1046,7 +1047,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std:: colStruct.fCompressionType = colType.compressionType; // Token - if ( isDictCol(colType) ) + if (isDictCol(colType) ) { colStruct.colWidth = 8; colStruct.tokenFlag = true; @@ -1075,6 +1076,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std:: colStructs.push_back(colStruct); dctnryStructList.push_back(dctnryStruct); + cscColTypeList.push_back(colType); ++column_iterator; } @@ -1324,7 +1326,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std:: */ if (NO_ERROR != - (error = fWEWrapper.insertColumnRecs(txnid.id, colStructs, colValuesList, dctnryStructList, dicStringList, + (error = fWEWrapper.insertColumnRecs(txnid.id, cscColTypeList, colStructs, colValuesList, dctnryStructList, dicStringList, dbRootExtTrackerVec, 0, bFirstExtentOnThisPM, isInsertSelect, isAutocommitOn, roPair.objnum, fIsFirstBatchPm))) { if (error == ERR_BRM_DEAD_LOCK) @@ -2691,6 +2693,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, WriteEngine::ColStruct colStruct; WriteEngine::ColValueList colValueList; WriteEngine::RIDList rowIDLists; + WriteEngine::CSCTypesList cscColTypeList; WriteEngine::DctnryStructList dctnryStructList; WriteEngine::DctnryStruct dctnryStruct; @@ -2858,7 +2861,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, colStruct.fCompressionType = colType.compressionType; tableColName.column = columnsUpdated[j]->get_Name(); - if ( !ridsFetched) + if (!ridsFetched) { // querystats uint64_t relativeRID = 0; @@ -3806,12 +3809,13 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, colStructList.push_back(colStruct); colValueList.push_back (colTupleList); + cscColTypeList.push_back(colType); } //end of bulding values and column structure. //timer.stop("fetch values"); if (rowIDLists.size() > 0) { - error = fWEWrapper.updateColumnRecs(txnId, colStructList, colValueList, rowIDLists, tableRO.objnum); + error = fWEWrapper.updateColumnRecs(txnId, cscColTypeList, colStructList, colValueList, rowIDLists, tableRO.objnum); } if (error != NO_ERROR) @@ -4116,13 +4120,14 @@ uint8_t WE_DMLCommandProc::processDelete(messageqcpp::ByteStream& bs, for (uint32_t j = 0; j < row.getColumnCount(); j++) { preBlkNums[j] = -1; - colWidth[j] = (row.getColumnWidth(j) >= 8 ? 8 : row.getColumnWidth(j)); + colWidth[j] = (row.getColumnWidth(j) >= 16 ? 16 : row.getColumnWidth(j)); } //Get the file information from rowgroup dbRoot = rowGroups[txnId]->getDBRoot(); rowGroups[txnId]->getLocation(&partition, &segment, &extentNum, &blockNum); WriteEngine::ColStructList colStructList; + WriteEngine::CSCTypesList cscColTypeList; WriteEngine::ColStruct colStruct; colStruct.fColPartition = partition; colStruct.fColSegment = segment; @@ -4158,7 +4163,9 @@ uint8_t WE_DMLCommandProc::processDelete(messageqcpp::ByteStream& bs, colStruct.tokenFlag = false; colStruct.fCompressionType = colType.compressionType; - if (colType.colWidth > 8) //token + if (colType.colWidth > 8 && + !(colType.colDataType == CalpontSystemCatalog::DECIMAL || + colType.colDataType == CalpontSystemCatalog::UDECIMAL)) //token { colStruct.colWidth = 8; colStruct.tokenFlag = true; @@ -4170,7 +4177,8 @@ uint8_t WE_DMLCommandProc::processDelete(messageqcpp::ByteStream& bs, colStruct.colDataType = colType.colDataType; - colStructList.push_back( colStruct ); + colStructList.push_back(colStruct); + cscColTypeList.push_back(colType); } } catch (exception& ex) @@ -4181,13 +4189,15 @@ uint8_t WE_DMLCommandProc::processDelete(messageqcpp::ByteStream& bs, } std::vector colExtentsStruct; + std::vector colExtentsColType; std::vector colOldValueList; std::vector ridLists; colExtentsStruct.push_back(colStructList); + colExtentsColType.push_back(cscColTypeList); ridLists.push_back(rowIDList); int error = 0; - error = fWEWrapper.deleteRow( txnId, colExtentsStruct, colOldValueList, ridLists, roPair.objnum ); + error = fWEWrapper.deleteRow(txnId, colExtentsColType, colExtentsStruct, colOldValueList, ridLists, roPair.objnum); if (error != NO_ERROR) { diff --git a/writeengine/server/we_dmlcommandproc.h b/writeengine/server/we_dmlcommandproc.h index 3786052b1..e80597d68 100644 --- a/writeengine/server/we_dmlcommandproc.h +++ b/writeengine/server/we_dmlcommandproc.h @@ -110,8 +110,8 @@ private: { if (((colType.colDataType == execplan::CalpontSystemCatalog::CHAR) && (colType.colWidth > 8)) || ((colType.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7)) - || ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 65)) - || ((colType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && (colType.precision > 65)) + || ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 38)) + || ((colType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && (colType.precision > 38)) || (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY) || (colType.colDataType == execplan::CalpontSystemCatalog::BLOB) || (colType.colDataType == execplan::CalpontSystemCatalog::TEXT)) diff --git a/writeengine/shared/we_blockop.cpp b/writeengine/shared/we_blockop.cpp index da104ad9b..01a8cd6f1 100644 --- a/writeengine/shared/we_blockop.cpp +++ b/writeengine/shared/we_blockop.cpp @@ -82,6 +82,7 @@ bool BlockOp::calculateRowId( * RETURN: * emptyVal - the value of empty row ***********************************************************/ +// TODO MCOL-641 Add support here uint64_t BlockOp::getEmptyRowValue( const CalpontSystemCatalog::ColDataType colDataType, const int width ) const { @@ -138,8 +139,10 @@ uint64_t BlockOp::getEmptyRowValue( emptyVal = joblist::SMALLINTEMPTYROW; else if ( width <= 4 ) emptyVal = joblist::INTEMPTYROW; - else + else if ( width <= 8 ) emptyVal = joblist::BIGINTEMPTYROW; + else + emptyVal = joblist::BINARYEMPTYROW; break; diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index af6723f7d..9c5c63454 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -328,6 +328,7 @@ void Convertor::mapErrnoToString(int errNum, std::string& errString) * none ******************************************************************************/ /* static */ +// TODO MCOL-641 void Convertor::convertColType(CalpontSystemCatalog::ColDataType dataType, ColType& internalType, bool isToken) { @@ -778,7 +779,6 @@ int Convertor::getCorrectRowWidth(CalpontSystemCatalog::ColDataType dataType, in newWidth = 8; else newWidth = 16; - break; case CalpontSystemCatalog::DATE: diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index 24ce85d0f..ce091b0ea 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -85,6 +85,7 @@ ColumnOp::~ColumnOp() * NO_ERROR if success * rowIdArray - allocation of the row id left here ***********************************************************/ +// TODO MCOL-641 add support here int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, Column& column, uint64_t totalRow, RID* rowIdArray, HWM& hwm, bool& newExtent, uint64_t& rowsLeft, HWM& newHwm, bool& newFile, ColStructList& newColStructList, DctnryStructList& newDctnryStructList, std::vector >& dbRootExtentTrackers, @@ -126,6 +127,9 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, unsigned char buf[BYTE_PER_BLOCK]; unsigned char* curVal; int64_t emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); // Seems is ok have it here and just once + // TODO MCOL-641 consolidate the emptyvalue logic + //__int128 bigEmptyVal; + //dataconvert::DataConvert::uint128Max(bigEmptyVal); if (useStartingExtent) { @@ -1455,6 +1459,7 @@ void ColumnOp::initColumn(Column& column) const // It is called at just 4 places on allocRowId() but all the time inside extend scanning loops // WIP Template this method +// TODO MCOL-641 Add support here. inline bool ColumnOp::isEmptyRow(uint64_t* curVal, uint64_t emptyVal, const int colWidth) { switch(colWidth){ @@ -1742,6 +1747,7 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, break; } + // TODO MCOL-641 do we need support here? if (bDelete) { emptyVal = getEmptyRowValue(curCol.colDataType, curCol.colWidth); @@ -1793,6 +1799,13 @@ int ColumnOp::writeRows(Column& curCol, uint64_t totalRow, const RIDList& ridLis uint64_t emptyVal; int rc = NO_ERROR; + int w = 0, incr = 8; + + if (curCol.colType == WriteEngine::WR_BINARY) + w = incr = curCol.colWidth; + else + w = curCol.colWidth > 8 ? 8 : curCol.colWidth; + while (!bExit) { curRowId = ridList[i]; @@ -1922,12 +1935,25 @@ int ColumnOp::writeRows(Column& curCol, uint64_t totalRow, const RIDList& ridLis if (bDelete) { - emptyVal = getEmptyRowValue(curCol.colDataType, curCol.colWidth); - pVal = &emptyVal; + if (curCol.colType != WriteEngine::WR_BINARY) + { + emptyVal = getEmptyRowValue(curCol.colDataType, curCol.colWidth); + pVal = &emptyVal; + } + else + { + // fix this + uint128_t bigEmptyVal; + emptyVal = getEmptyRowValue(curCol.colDataType, curCol.colWidth); + *(reinterpret_cast(&bigEmptyVal)) = emptyVal; + *(reinterpret_cast(&bigEmptyVal) + 1) = emptyVal; + //dataconvert::DataConvert::uint128Max(bigEmptyVal); + pVal = &bigEmptyVal; + } } // This is the write stuff - for(int b = 0, w = curCol.colWidth > 8 ? 8 : curCol.colWidth; b < curCol.colWidth; b += 8) //FIXME for no loop + for (int b = 0; b < curCol.colWidth; b += incr) //FIXME for no loop writeBufValue(dataBuf + dataBio + b, pVal, w); i++; diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 29edee9d4..7d295b397 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -223,83 +223,7 @@ void WriteEngineWrapper::findSmallestColumn(uint32_t& colId, ColStructList colSt using int128_t = __int128; using uint128_t = unsigned __int128; -struct uint128_pod -{ - uint64_t lo; - uint64_t hi; -}; - - - -inline void toString(uint128_t i, char *p) -{ - uint64_t div = 10000000000000000000ULL; - size_t div_log = 19; - uint128_t high = i; - uint128_t low; - low = high % div; - high /= div; - uint128_t mid; - mid = high % div; - high /= div; - - uint128_pod *high_pod = reinterpret_cast(&high); - uint128_pod *mid_pod = reinterpret_cast(&mid); - uint128_pod *low_pod = reinterpret_cast(&low); - int printed_chars = 0; - - // WIP replace snprintf with streams - if (high_pod->lo != 0) { - printed_chars = snprintf(p, div_log+1, "%ld", high_pod->lo); - p += printed_chars; - printed_chars = snprintf(p, div_log+1, "%019lu", mid_pod->lo); - p += printed_chars; - } else if (mid_pod->lo != 0) { - printed_chars = snprintf(p, div_log+1, "%lu", mid_pod->lo); - p += printed_chars; - } - snprintf(p, div_log+1, "%019ld", low_pod->lo); -} - - - -/*@convertValArray - Convert interface values to internal values - */ -/*********************************************************** - * DESCRIPTION: - * Convert interface values to internal values - * PARAMETERS: - * colStructList - column struct list - * colValueList - column value list - * RETURN: - * none - * valArray - output value array - * nullArray - output null flag array - ***********************************************************/ -void WriteEngineWrapper::convertValArray(const size_t totalRow, const ColType colType, ColTupleList& curTupleList, void* valArray, bool bFromList) -{ - ColTuple curTuple; - ColTupleList::size_type i; - - if (bFromList) - { - for (i = 0; i < curTupleList.size(); i++) - { - curTuple = curTupleList[i]; - convertValue(colType, valArray, i, curTuple.data); - } - } - else - { - for (i = 0; i < totalRow; i++) - { - convertValue(colType, valArray, i, curTuple.data, false); - curTupleList.push_back(curTuple); - } - } -} - -/*@convertValArray - Convert interface values to internal values +/*@convertValArray - Convert interface values to internal values */ /*********************************************************** * DESCRIPTION: @@ -313,7 +237,7 @@ void WriteEngineWrapper::convertValArray(const size_t totalRow, const ColType co * valArray - output value array * nullArray - output null flag array ***********************************************************/ -void WriteEngineWrapper::convertValArray(const size_t totalRow, const CalpontSystemCatalog::ColType &cscColType, const ColType colType, ColTupleList& curTupleList, void* valArray, bool bFromList) +void WriteEngineWrapper::convertValArray(const size_t totalRow, const CalpontSystemCatalog::ColType& cscColType, const ColType colType, ColTupleList& curTupleList, void* valArray, bool bFromList) { ColTuple curTuple; ColTupleList::size_type i; @@ -336,11 +260,10 @@ void WriteEngineWrapper::convertValArray(const size_t totalRow, const CalpontSys } } - /* * @brief Convert column value to its internal representation */ -void WriteEngineWrapper::convertValue(const ColType colType, void* value, boost::any& data) +void WriteEngineWrapper::convertValue(const execplan::CalpontSystemCatalog::ColType& cscColType, ColType colType, void* value, boost::any& data) { string curStr; int size; @@ -476,169 +399,11 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* value, boost: } break; - // Replace INT128 with WR_BINARY using CSC::ColType data - /*case WriteEngine::WR_INT128: - { - int128_t val = boost::any_cast(data); - size = 16; - // WIP Why do we use memcpy here? - memcpy(value, &val, size); - }*/ + // WIP MCOL-641 case WriteEngine::WR_BINARY: { - char val = boost::any_cast(data); - //TODO:FIXME how to determine size ? 16, 32,48 ? - // WIP - size = 16; - memcpy(value, &val, size); - } - break; - - } // end of switch (colType) -} /*@convertValue - The base for converting values */ - -// WIP this is ugly to have structs with the same name -void WriteEngineWrapper::convertValue(const execplan::CalpontSystemCatalog::ColType &fullColType, ColType colType, void* value, boost::any& data) -{ - string curStr; - int size; - - switch (colType) - { - case WriteEngine::WR_INT : - case WriteEngine::WR_MEDINT : - if (data.type() == typeid(int)) - { - int val = boost::any_cast(data); - size = sizeof(int); - memcpy(value, &val, size); - } - else - { - uint32_t val = boost::any_cast(data); - size = sizeof(uint32_t); - memcpy(value, &val, size); - } - - break; - - case WriteEngine::WR_UINT : - case WriteEngine::WR_UMEDINT : - { - uint32_t val = boost::any_cast(data); - size = sizeof(uint32_t); - memcpy(value, &val, size); - } - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR : - case WriteEngine::WR_BLOB : - case WriteEngine::WR_TEXT : - curStr = boost::any_cast(data); - - if ((int) curStr.length() > MAX_COLUMN_BOUNDARY) - curStr = curStr.substr(0, MAX_COLUMN_BOUNDARY); - - memcpy(value, curStr.c_str(), curStr.length()); - break; - - case WriteEngine::WR_FLOAT: - { - float val = boost::any_cast(data); - -//N.B.There is a bug in boost::any or in gcc where, if you store a nan, you will get back a nan, -// but not necessarily the same bits that you put in. This only seems to be for float (double seems -// to work). - if (isnan(val)) - { - uint32_t ti = joblist::FLOATNULL; - float* tfp = (float*)&ti; - val = *tfp; - } - - size = sizeof(float); - memcpy(value, &val, size); - } - break; - - case WriteEngine::WR_DOUBLE: - { - double val = boost::any_cast(data); - size = sizeof(double); - memcpy(value, &val, size); - } - break; - - case WriteEngine::WR_SHORT: - { - short val = boost::any_cast(data); - size = sizeof(short); - memcpy(value, &val, size); - } - break; - - case WriteEngine::WR_USHORT: - { - uint16_t val = boost::any_cast(data); - size = sizeof(uint16_t); - memcpy(value, &val, size); - } - break; - - case WriteEngine::WR_BYTE: - { - char val = boost::any_cast(data); - size = sizeof(char); - memcpy(value, &val, size); - } - break; - - case WriteEngine::WR_UBYTE: - { - uint8_t val = boost::any_cast(data); - size = sizeof(uint8_t); - memcpy(value, &val, size); - } - break; - - case WriteEngine::WR_LONGLONG: - if (data.type() == typeid(long long)) - { - long long val = boost::any_cast(data); - size = sizeof(long long); - memcpy(value, &val, size); - } - else - { - uint64_t val = boost::any_cast(data); - size = sizeof(uint64_t); - memcpy(value, &val, size); - } - - break; - - case WriteEngine::WR_ULONGLONG: - { - uint64_t val = boost::any_cast(data); - size = sizeof(uint64_t); - memcpy(value, &val, size); - } - break; - - case WriteEngine::WR_TOKEN: - { - Token val = boost::any_cast(data); - size = sizeof(Token); - memcpy(value, &val, size); - } - break; - - // WIP - case WriteEngine::WR_BINARY: - { - size = fullColType.colWidth; - if (fullColType.colDataType == CalpontSystemCatalog::DECIMAL) + size = cscColType.colWidth; + if (cscColType.colDataType == CalpontSystemCatalog::DECIMAL) { int128_t val = boost::any_cast(data); memcpy(value, &val, size); @@ -648,192 +413,12 @@ void WriteEngineWrapper::convertValue(const execplan::CalpontSystemCatalog::ColT char val = boost::any_cast(data); memcpy(value, &val, size); } - } break; } // end of switch (colType) } /*@convertValue - The base for converting values */ -// WIP -// Legacy version -void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, const size_t pos, boost::any& data, bool fromList) -{ - string curStr; - - if (fromList) - { - switch (colType) - { - case WriteEngine::WR_INT : - case WriteEngine::WR_MEDINT : - if (data.type() == typeid(long)) - ((int*)valArray)[pos] = static_cast(boost::any_cast(data)); - else if (data.type() == typeid(int)) - ((int*)valArray)[pos] = boost::any_cast(data); - else - ((int*)valArray)[pos] = boost::any_cast(data); - - break; - - case WriteEngine::WR_UINT : - case WriteEngine::WR_UMEDINT : - ((uint32_t*)valArray)[pos] = boost::any_cast(data); - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR : - case WriteEngine::WR_BLOB : - case WriteEngine::WR_TEXT : - curStr = boost::any_cast(data); - - if ((int) curStr.length() > MAX_COLUMN_BOUNDARY) - curStr = curStr.substr(0, MAX_COLUMN_BOUNDARY); - - memcpy((char*)valArray + pos * MAX_COLUMN_BOUNDARY, curStr.c_str(), curStr.length()); - break; - -// case WriteEngine::WR_LONG : ((long*)valArray)[pos] = boost::any_cast(curTuple.data); -// break; - case WriteEngine::WR_FLOAT: - ((float*)valArray)[pos] = boost::any_cast(data); - - if (isnan(((float*)valArray)[pos])) - { - uint32_t ti = joblist::FLOATNULL; - float* tfp = (float*)&ti; - ((float*)valArray)[pos] = *tfp; - } - - break; - - case WriteEngine::WR_DOUBLE: - ((double*)valArray)[pos] = boost::any_cast(data); - break; - - case WriteEngine::WR_SHORT: - ((short*)valArray)[pos] = boost::any_cast(data); - break; - - case WriteEngine::WR_USHORT: - ((uint16_t*)valArray)[pos] = boost::any_cast(data); - break; - -// case WriteEngine::WR_BIT: ((bool*)valArray)[pos] = boost::any_cast(data); -// break; - case WriteEngine::WR_BYTE: - ((char*)valArray)[pos] = boost::any_cast(data); - break; - - case WriteEngine::WR_UBYTE: - ((uint8_t*)valArray)[pos] = boost::any_cast(data); - break; - - case WriteEngine::WR_LONGLONG: - if (data.type() == typeid(long long)) - ((long long*)valArray)[pos] = boost::any_cast(data); - else if (data.type() == typeid(long)) - ((long long*)valArray)[pos] = (long long)boost::any_cast(data); - else - ((long long*)valArray)[pos] = boost::any_cast(data); - - break; - - case WriteEngine::WR_ULONGLONG: - ((uint64_t*)valArray)[pos] = boost::any_cast(data); - break; - - case WriteEngine::WR_TOKEN: - ((Token*)valArray)[pos] = boost::any_cast(data); - break; - - case WriteEngine::WR_BINARY: - { - curStr = boost::any_cast(data); - // String length or column width? - memcpy((char*)valArray + pos * curStr.length(), - curStr.c_str(), curStr.length()); - break; - } - } // end of switch (colType) - } - else - { - switch (colType) - { - case WriteEngine::WR_INT : - case WriteEngine::WR_MEDINT : - data = ((int*)valArray)[pos]; - break; - - case WriteEngine::WR_UINT : - case WriteEngine::WR_UMEDINT : - data = ((uint64_t*)valArray)[pos]; - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR : - case WriteEngine::WR_BLOB : - case WriteEngine::WR_TEXT : - char tmp[10]; - memcpy(tmp, (char*)valArray + pos * 8, 8); - curStr = tmp; - data = curStr; - break; - -// case WriteEngine::WR_LONG : ((long*)valArray)[pos] = boost::any_cast(curTuple.data); -// break; - case WriteEngine::WR_FLOAT: - data = ((float*)valArray)[pos]; - break; - - case WriteEngine::WR_DOUBLE: - data = ((double*)valArray)[pos]; - break; - - case WriteEngine::WR_SHORT: - data = ((short*)valArray)[pos]; - break; - - case WriteEngine::WR_USHORT: - data = ((uint16_t*)valArray)[pos]; - break; - -// case WriteEngine::WR_BIT: data = ((bool*)valArray)[pos]; -// break; - case WriteEngine::WR_BYTE: - data = ((char*)valArray)[pos]; - break; - - case WriteEngine::WR_UBYTE: - data = ((uint8_t*)valArray)[pos]; - break; - - case WriteEngine::WR_LONGLONG: - data = ((long long*)valArray)[pos]; - break; - - case WriteEngine::WR_ULONGLONG: - data = ((uint64_t*)valArray)[pos]; - break; - - case WriteEngine::WR_TOKEN: - data = ((Token*)valArray)[pos]; - break; - // WIP - case WriteEngine::WR_BINARY : - { - // WIP do we need tmp here? - char *tmp = (char*)alloca (sizeof(char) * 16); - memcpy(tmp, (char*)valArray + pos * 16, 16); - curStr = tmp; - data = curStr; - } - break; - } // end of switch (colType) - } // end of if -} - /*********************************************************** * DESCRIPTION: @@ -845,7 +430,7 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, con * RETURN: * none ***********************************************************/ -void WriteEngineWrapper::convertValue(const CalpontSystemCatalog::ColType &fullColType, const ColType colType, void* valArray, const size_t pos, boost::any& data, bool fromList) +void WriteEngineWrapper::convertValue(const CalpontSystemCatalog::ColType& cscColType, const ColType colType, void* valArray, const size_t pos, boost::any& data, bool fromList) { string curStr; @@ -936,7 +521,7 @@ void WriteEngineWrapper::convertValue(const CalpontSystemCatalog::ColType &fullC break; case WriteEngine::WR_BINARY: - if (fullColType.colDataType != CalpontSystemCatalog::DECIMAL) + if (cscColType.colDataType != CalpontSystemCatalog::DECIMAL) { curStr = boost::any_cast(data); // String length or column width? @@ -945,7 +530,7 @@ void WriteEngineWrapper::convertValue(const CalpontSystemCatalog::ColType &fullC else { int128_t val = boost::any_cast(data); - size_t size = fullColType.colWidth; + size_t size = cscColType.colWidth; // WIP Why do we use memcpy here? memcpy((uint8_t*)valArray+pos*size, &val, size); } @@ -1018,15 +603,15 @@ void WriteEngineWrapper::convertValue(const CalpontSystemCatalog::ColType &fullC break; // WIP case WriteEngine::WR_BINARY : - if (fullColType.colDataType == CalpontSystemCatalog::DECIMAL) + if (cscColType.colDataType == CalpontSystemCatalog::DECIMAL) { data = ((int128_t*)valArray)[pos]; } else { // WIP do we need tmp here? - char *tmp = (char*) alloca (sizeof(char) * fullColType.colWidth); - memcpy(tmp, (char*)valArray + pos * fullColType.colWidth, fullColType.colWidth); + char *tmp = (char*) alloca (sizeof(char) * cscColType.colWidth); + memcpy(tmp, (char*)valArray + pos * cscColType.colWidth, cscColType.colWidth); curStr = tmp; data = curStr; } @@ -1093,7 +678,7 @@ int WriteEngineWrapper::createColumn( * @brief Fill column with default values */ int WriteEngineWrapper::fillColumn(const TxnID& txnid, const OID& dataOid, - const CalpontSystemCatalog::ColDataType dataType, int dataWidth, + const CalpontSystemCatalog::ColType& colType, ColTuple defaultVal, const OID& refColOID, const CalpontSystemCatalog::ColDataType refColDataType, int refColWidth, int refCompressionType, @@ -1113,36 +698,36 @@ int WriteEngineWrapper::fillColumn(const TxnID& txnid, const OID& dataOid, colOpNewCol->initColumn(newCol); refColOp->initColumn(refCol); uint16_t dbRoot = 1; //not to be used - int newDataWidth = dataWidth; + int newDataWidth = colType.colWidth; //Convert HWM of the reference column for the new column //Bug 1703,1705 bool isToken = false; - if (((dataType == CalpontSystemCatalog::VARCHAR) && (dataWidth > 7)) || - ((dataType == CalpontSystemCatalog::CHAR) && (dataWidth > 8)) || - (dataType == CalpontSystemCatalog::VARBINARY) || - (dataType == CalpontSystemCatalog::BLOB) || - (dataType == CalpontSystemCatalog::TEXT)) + if (((colType.colDataType == CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7)) || + ((colType.colDataType == CalpontSystemCatalog::CHAR) && (colType.colWidth > 8)) || + (colType.colDataType == CalpontSystemCatalog::VARBINARY) || + (colType.colDataType == CalpontSystemCatalog::BLOB) || + (colType.colDataType == CalpontSystemCatalog::TEXT)) { isToken = true; } - Convertor::convertColType(dataType, newColType, isToken); + Convertor::convertColType(colType.colDataType, newColType, isToken); // WIP // replace with isDictCol if (((refColDataType == CalpontSystemCatalog::VARCHAR) && (refColWidth > 7)) || ((refColDataType == CalpontSystemCatalog::CHAR) && (refColWidth > 8)) || (refColDataType == CalpontSystemCatalog::VARBINARY) || - (dataType == CalpontSystemCatalog::BLOB) || - (dataType == CalpontSystemCatalog::TEXT)) + (colType.colDataType == CalpontSystemCatalog::BLOB) || + (colType.colDataType == CalpontSystemCatalog::TEXT)) { isToken = true; } - newDataWidth = colOpNewCol->getCorrectRowWidth(dataType, dataWidth); + newDataWidth = colOpNewCol->getCorrectRowWidth(colType.colDataType, colType.colWidth); // MCOL-1347 CS doubles the width for ALTER TABLE..ADD COLUMN - if ( dataWidth < 4 && dataType == CalpontSystemCatalog::VARCHAR ) + if ( colType.colWidth < 4 && colType.colDataType == CalpontSystemCatalog::VARCHAR ) { newDataWidth >>= 1; } @@ -1151,7 +736,7 @@ int WriteEngineWrapper::fillColumn(const TxnID& txnid, const OID& dataOid, refColOp->setColParam(refCol, 0, refColOp->getCorrectRowWidth(refColDataType, refColWidth), refColDataType, refColType, (FID)refColOID, refCompressionType, dbRoot); colOpNewCol->setColParam(newCol, 0, newDataWidth, - dataType, newColType, (FID)dataOid, compressionType, dbRoot); + colType.colDataType, newColType, (FID)dataOid, compressionType, dbRoot); int size = sizeof(Token); @@ -1167,28 +752,31 @@ int WriteEngineWrapper::fillColumn(const TxnID& txnid, const OID& dataOid, else { // WIP - convertValue(newColType, defVal.get(), defaultVal.data); + convertValue(colType, newColType, defVal.get(), defaultVal.data); } if (rc == NO_ERROR) - rc = colOpNewCol->fillColumn(txnid, newCol, refCol, defVal.get(), dctnry, refColOp, dictOid, dataWidth, defaultValStr, autoincrement); + rc = colOpNewCol->fillColumn(txnid, newCol, refCol, defVal.get(), dctnry, refColOp, dictOid, colType.colWidth, defaultValStr, autoincrement); // flushing files is in colOp->fillColumn() return rc; } -int WriteEngineWrapper::deleteRow(const TxnID& txnid, vector& colExtentsStruct, vector& colOldValueList, +int WriteEngineWrapper::deleteRow(const TxnID& txnid, const vector& colExtentsColType, + vector& colExtentsStruct, vector& colOldValueList, vector& ridLists, const int32_t tableOid) { ColTuple curTuple; ColStruct curColStruct; + CalpontSystemCatalog::ColType cscColType; DctnryStruct dctnryStruct; ColValueList colValueList; ColTupleList curTupleList; DctnryStructList dctnryStructList; DctnryValueList dctnryValueList; ColStructList colStructList; + CSCTypesList cscColTypeList; uint64_t emptyVal; int rc; string tmpStr(""); @@ -1204,17 +792,33 @@ int WriteEngineWrapper::deleteRow(const TxnID& txnid, vector& col for (unsigned extent = 0; extent < numExtents; extent++) { colStructList = colExtentsStruct[extent]; + cscColTypeList = colExtentsColType[extent]; for (ColStructList::size_type i = 0; i < colStructList.size(); i++) { curTupleList.clear(); curColStruct = colStructList[i]; - emptyVal = m_colOp[op(curColStruct.fCompressionType)]-> - getEmptyRowValue(curColStruct.colDataType, curColStruct.colWidth); + cscColType = cscColTypeList[i]; + Convertor::convertColType(&curColStruct); + + if (curColStruct.colType == WriteEngine::WR_BINARY) + { + uint128_t bigEmptyVal; + emptyVal = m_colOp[op(curColStruct.fCompressionType)]-> + getEmptyRowValue(curColStruct.colDataType, curColStruct.colWidth); + *(reinterpret_cast(&bigEmptyVal)) = emptyVal; + *(reinterpret_cast(&bigEmptyVal) + 1) = emptyVal; + //dataconvert::DataConvert::uint128Max(bigEmptyVal); + curTuple.data = bigEmptyVal; + } + else + { + emptyVal = m_colOp[op(curColStruct.fCompressionType)]-> + getEmptyRowValue(curColStruct.colDataType, curColStruct.colWidth); + + curTuple.data = emptyVal; + } - curTuple.data = emptyVal; - //for (RIDList::size_type j = 0; j < ridLists[extent].size(); j++) - // curTupleList.push_back(curTuple); curTupleList.push_back(curTuple); colValueList.push_back(curTupleList); @@ -1239,7 +843,7 @@ int WriteEngineWrapper::deleteRow(const TxnID& txnid, vector& col // unfortunately I don't have a better way to instruct without passing too many parameters m_opType = DELETE; - rc = updateColumnRec(txnid, colExtentsStruct, colValueList, colOldValueList, ridLists, dctnryExtentsStruct, dctnryValueList, tableOid); + rc = updateColumnRec(txnid, colExtentsColType, colExtentsStruct, colValueList, colOldValueList, ridLists, dctnryExtentsStruct, dctnryValueList, tableOid); m_opType = NOOP; return rc; @@ -1417,6 +1021,7 @@ void WriteEngineWrapper::flushVMCache() const ***********************************************************/ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, + const CSCTypesList& cscColTypeList, ColStructList& colStructList, ColValueList& colValueList, DctnryStructList& dctnryStructList, @@ -1746,7 +1351,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, //-------------------------------------------------------------------------- // allocate row id(s) //-------------------------------------------------------------------------- - curColStruct = colStructList[colId]; + curColStruct = colStructList[colId]; colOp = m_colOp[op(curColStruct.fCompressionType)]; colOp->initColumn(curCol); @@ -1757,12 +1362,12 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, vector fileInfo; dbRoot = curColStruct.fColDbRoot; //use the first column to calculate row id - ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[colId].dataOid); + ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[colId].dataOid); ColExtsInfo::iterator it = aColExtsInfo.begin(); while (it != aColExtsInfo.end()) { - if ((it->dbRoot == colStructList[colId].fColDbRoot) && + if ((it->dbRoot == colStructList[colId].fColDbRoot) && (it->partNum == colStructList[colId].fColPartition) && (it->segNum == colStructList[colId].fColSegment) && it->current ) { @@ -2179,7 +1784,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, //---------------------------------------------------------------------- // Write row(s) to database file(s) //---------------------------------------------------------------------- - rc = writeColumnRec(txnid, colStructList, colOldValueList, rowIdArray, newColStructList, colNewValueList, tableOid, useTmpSuffix); // @bug 5572 HDFS tmp file + rc = writeColumnRec(txnid, cscColTypeList, colStructList, colOldValueList, rowIdArray, newColStructList, colNewValueList, tableOid, useTmpSuffix); // @bug 5572 HDFS tmp file } return rc; @@ -2963,6 +2568,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, int WriteEngineWrapper::insertColumnRec_SYS(const TxnID& txnid, + const CSCTypesList& cscColTypeList, ColStructList& colStructList, ColValueList& colValueList, DctnryStructList& dctnryStructList, @@ -3544,11 +3150,11 @@ int WriteEngineWrapper::insertColumnRec_SYS(const TxnID& txnid, if (newExtent) { - rc = writeColumnRec(txnid, colStructList, colOldValueList, rowIdArray, newColStructList, colNewValueList, tableOid, false); // @bug 5572 HDFS tmp file + rc = writeColumnRec(txnid, cscColTypeList, colStructList, colOldValueList, rowIdArray, newColStructList, colNewValueList, tableOid, false); // @bug 5572 HDFS tmp file } else { - rc = writeColumnRec(txnid, colStructList, colValueList, rowIdArray, newColStructList, colNewValueList, tableOid, false); // @bug 5572 HDFS tmp file + rc = writeColumnRec(txnid, cscColTypeList, colStructList, colValueList, rowIdArray, newColStructList, colNewValueList, tableOid, false); // @bug 5572 HDFS tmp file } } @@ -3625,7 +3231,7 @@ int WriteEngineWrapper::insertColumnRec_SYS(const TxnID& txnid, } int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, - CSCTypesList& cscColTypesList, + const CSCTypesList& cscColTypeList, ColStructList& colStructList, ColValueList& colValueList, DctnryStructList& dctnryStructList, @@ -4219,13 +3825,13 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, { if (newExtent) { - rc = writeColumnRec(txnid, cscColTypesList, colStructList, colOldValueList, + rc = writeColumnRec(txnid, cscColTypeList, colStructList, colOldValueList, rowIdArray, newColStructList, colNewValueList, tableOid, false); // @bug 5572 HDFS tmp file } else { - rc = writeColumnRec(txnid, cscColTypesList, colStructList, colValueList, + rc = writeColumnRec(txnid, cscColTypeList, colStructList, colValueList, rowIdArray, newColStructList, colNewValueList, tableOid, true); // @bug 5572 HDFS tmp file } @@ -4635,6 +4241,7 @@ void WriteEngineWrapper::writeVBEnd(const TxnID& txnid, std::vector& colExtentsColType, vector& colExtentsStruct, ColValueList& colValueList, vector& colOldValueList, @@ -4644,40 +4251,20 @@ int WriteEngineWrapper::updateColumnRec(const TxnID& txnid, const int32_t tableOid) { int rc = 0; - //RID* rowIdArray = NULL; - //RIDList::size_type i; unsigned numExtents = colExtentsStruct.size(); - // ColValueList tmpColValueList; - RIDList::const_iterator ridsIter; ColStructList colStructList; DctnryStructList dctnryStructList; + WriteEngine::CSCTypesList cscColTypeList; ColumnOp* colOp = NULL; for (unsigned extent = 0; extent < numExtents; extent++) { - ridsIter = ridLists[extent].begin(); - - //rowIdArray = (RID*)calloc(sizeof(RID), ridLists[extent].size()); - colStructList = colExtentsStruct[extent]; dctnryStructList = dctnryExtentsStruct[extent]; + cscColTypeList = colExtentsColType[extent]; if (m_opType != DELETE) { - - /* ColTuple colTuple; - ColTupleList colTupleList; - for (i=0; i < colValueList.size(); i++) - { - colTupleList = colValueList[i]; - colTuple = colTupleList[0]; - for (unsigned i = 1; i < ridLists[extent].size(); i++) - { - colTupleList.push_back(colTuple); - } - tmpColValueList.push_back(colTupleList); - } - */ //Tokenize data if needed vector tokenList; @@ -4690,7 +4277,6 @@ int WriteEngineWrapper::updateColumnRec(const TxnID& txnid, { // only need to tokenize once dctCol_iter = dctnryValueList[i].begin(); - //col_iter = colValueList[i].begin(); Token token; if (!dctCol_iter->isNull) @@ -4703,14 +4289,7 @@ int WriteEngineWrapper::updateColumnRec(const TxnID& txnid, //timer.stop("tokenize"); #endif } - else - { - //if (dctnryStructList[i].dctnryOid == 2001) - // std::cout << " got null token for string " << dctCol_iter->sigValue <sigValue << " op:fbo = " << token.op <<":"< lbids; vector colDataTypes; bool successFlag = true; unsigned width = 0; - int curFbo = 0, curBio, lastFbo = -1; + int curFbo = 0, curBio, lastFbo = -1; rid_iter = ridLists[extent].begin(); RID aRid = *rid_iter; @@ -4777,24 +4347,18 @@ int WriteEngineWrapper::updateColumnRec(const TxnID& txnid, } } - //cout << "lbids size = " << lbids.size()<< endl; //#ifdef PROFILE //timer.start("markExtentsInvalid"); //#endif if (lbids.size() > 0) rc = BRMWrapper::getInstance()->markExtentsInvalid(lbids, colDataTypes); - //} - - if ( m_opType != DELETE) + if (m_opType != DELETE) m_opType = UPDATE; - rc = writeColumnRec(txnid, colStructList, colValueList, colOldValueList, + rc = writeColumnRec(txnid, cscColTypeList, colStructList, colValueList, colOldValueList, ridLists[extent], tableOid, true, ridLists[extent].size()); -// if (rowIdArray) -// free(rowIdArray); - m_opType = NOOP; if (rc != NO_ERROR) @@ -4805,6 +4369,7 @@ int WriteEngineWrapper::updateColumnRec(const TxnID& txnid, } int WriteEngineWrapper::updateColumnRecs(const TxnID& txnid, + const CSCTypesList& cscColTypeList, vector& colExtentsStruct, ColValueList& colValueList, const RIDList& ridLists, @@ -4857,12 +4422,13 @@ int WriteEngineWrapper::updateColumnRecs(const TxnID& txnid, if ( m_opType != DELETE) m_opType = UPDATE; - rc = writeColumnRecords (txnid, colExtentsStruct, colValueList, ridLists, tableOid); + rc = writeColumnRecords (txnid, cscColTypeList, colExtentsStruct, colValueList, ridLists, tableOid); m_opType = NOOP; return rc; } int WriteEngineWrapper::writeColumnRecords(const TxnID& txnid, + const CSCTypesList& cscColTypeList, vector& colStructList, ColValueList& colValueList, const RIDList& ridLists, const int32_t tableOid, bool versioning) @@ -5016,7 +4582,7 @@ int WriteEngineWrapper::writeColumnRecords(const TxnID& txnid, try { - convertValArray(totalRow, curColStruct.colType, curTupleList, valArray); + convertValArray(totalRow, cscColTypeList[i], curColStruct.colType, curTupleList, valArray); } catch (...) { @@ -5383,6 +4949,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, } // have to init the size here + // TODO MCOL-641 is commenting out the switch statement below correct? valArray = calloc(totalRow2, newColStructList[i].colWidth); /*switch (newColStructList[i].colType) { @@ -5454,7 +5021,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, try { - convertValArray(totalRow2, newColStructList[i].colType, newColValueList[i], valArray); + convertValArray(totalRow2, cscColTypeList[i], newColStructList[i].colType, newColValueList[i], valArray); } catch (...) { @@ -5564,6 +5131,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, // have to init the size here // shared pointers or memory in a stack + // TODO MCOL-641 is commenting out the switch statement below correct? valArray = calloc(totalRow1, colStructList[i].colWidth); // WIP /*switch (colStructList[i].colType) @@ -5688,609 +5256,6 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, return rc; } -int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, - const ColStructList& colStructList, - ColValueList& colValueList, - RID* rowIdArray, - const ColStructList& newColStructList, - ColValueList& newColValueList, - const int32_t tableOid, - bool useTmpSuffix, - bool versioning) -{ - bool bExcp; - int rc = 0; - void* valArray; - string segFile; - Column curCol; - ColTupleList oldTupleList; - ColStructList::size_type totalColumn; - ColStructList::size_type i; - ColTupleList::size_type totalRow1, totalRow2; - - setTransId(txnid); - - totalColumn = colStructList.size(); -#ifdef PROFILE - StopWatch timer; -#endif - - if (newColValueList.size() > 0) - { - totalRow1 = colValueList[0].size(); - totalRow2 = newColValueList[0].size(); - } - else - { - totalRow1 = colValueList[0].size(); - totalRow2 = 0; - } - - TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); - - for (i = 0; i < totalColumn; i++) - { - if (totalRow2 > 0) - { - RID* secondPart = rowIdArray + totalRow1; - - //@Bug 2205 Check if all rows go to the new extent - if (totalRow1 > 0) - { - //Write the first batch - valArray = NULL; - RID* firstPart = rowIdArray; - ColumnOp* colOp = m_colOp[op(colStructList[i].fCompressionType)]; - - // set params - colOp->initColumn(curCol); - // need to pass real dbRoot, partition, and segment to setColParam - colOp->setColParam(curCol, 0, colStructList[i].colWidth, - colStructList[i].colDataType, colStructList[i].colType, colStructList[i].dataOid, - colStructList[i].fCompressionType, colStructList[i].fColDbRoot, - colStructList[i].fColPartition, colStructList[i].fColSegment); - - ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(colStructList[i].dataOid); - ColExtsInfo::iterator it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == colStructList[i].fColDbRoot) && (it->partNum == colStructList[i].fColPartition) && (it->segNum == colStructList[i].fColSegment)) - break; - - it++; - } - - if (it == aColExtsInfo.end()) //add this one to the list - { - ColExtInfo aExt; - aExt.dbRoot = colStructList[i].fColDbRoot; - aExt.partNum = colStructList[i].fColPartition; - aExt.segNum = colStructList[i].fColSegment; - aExt.compType = colStructList[i].fCompressionType; - aColExtsInfo.push_back(aExt); - aTbaleMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); - } - - rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file - - if (rc != NO_ERROR) - break; - - // handling versioning - vector rangeList; - - if (versioning) - { - rc = processVersionBuffer(curCol.dataFile.pFile, txnid, colStructList[i], - colStructList[i].colWidth, totalRow1, firstPart, rangeList); - - if (rc != NO_ERROR) - { - if (colStructList[i].fCompressionType == 0) - { - curCol.dataFile.pFile->flush(); - } - - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - break; - } - } - - // WIP We can allocate based on column size and not colType - // have to init the size here - valArray = calloc(totalRow1, colStructList[i].colWidth); -#if 0 - switch (colStructList[i].colType) - { - // WIP we don't need type cast here only size - case WriteEngine::WR_INT: - case WriteEngine::WR_MEDINT: - valArray = (int*) calloc(sizeof(int), totalRow1); - break; - - case WriteEngine::WR_UINT: - case WriteEngine::WR_UMEDINT: - valArray = (uint32_t*) calloc(sizeof(uint32_t), totalRow1); - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR: - case WriteEngine::WR_BLOB: - case WriteEngine::WR_TEXT: - valArray = (char*) calloc(sizeof(char), totalRow1 * MAX_COLUMN_BOUNDARY); - break; - - case WriteEngine::WR_FLOAT: - valArray = (float*) calloc(sizeof(float), totalRow1); - break; - - case WriteEngine::WR_DOUBLE: - valArray = (double*) calloc(sizeof(double), totalRow1); - break; - - case WriteEngine::WR_BYTE: - valArray = (char*) calloc(sizeof(char), totalRow1); - break; - - case WriteEngine::WR_UBYTE: - valArray = (uint8_t*) calloc(sizeof(uint8_t), totalRow1); - break; - - case WriteEngine::WR_SHORT: - valArray = (short*) calloc(sizeof(short), totalRow1); - break; - - case WriteEngine::WR_USHORT: - valArray = (uint16_t*) calloc(sizeof(uint16_t), totalRow1); - break; - - case WriteEngine::WR_LONGLONG: - valArray = (long long*) calloc(sizeof(long long), totalRow1); - break; - - case WriteEngine::WR_ULONGLONG: - valArray = (uint64_t*) calloc(sizeof(uint64_t), totalRow1); - break; - - case WriteEngine::WR_TOKEN: - valArray = (Token*) calloc(sizeof(Token), totalRow1); - break; - - // WIP - case WriteEngine::WR_BINARY: - valArray = calloc(totalRow1, colStructList[i].colWidth); - break; - } -#endif - - // convert values to valArray - // WIP - // Is this m_opType ever set to DELETE? - if (m_opType != DELETE) - { - bExcp = false; - - try - { - // WIP We convert values twice!? - // dmlcommandproc converts strings to boost::any and this converts - // into actual type value masked by *void - // It is not clear why we need to convert to boost::any b/c we can convert from the original string here - convertValArray(totalRow1, colStructList[i].colType, colValueList[i], valArray); - } - catch (...) - { - bExcp = true; - } - - if (bExcp) - { - if (versioning) - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - - return ERR_PARSING; - } - -#ifdef PROFILE - iimer.start("writeRow "); -#endif - rc = colOp->writeRow(curCol, totalRow1, firstPart, valArray); -#ifdef PROFILE - timer.stop("writeRow "); -#endif - } - else - { -#ifdef PROFILE - timer.start("writeRow "); -#endif - rc = colOp->writeRow(curCol, totalRow1, rowIdArray, valArray, true); -#ifdef PROFILE - timer.stop("writeRow "); -#endif - } - - colOp->clearColumn(curCol); - - if (versioning) - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - - if (valArray != NULL) - free(valArray); - - // check error - if (rc != NO_ERROR) - break; - } - - //Process the second batch - valArray = NULL; - - ColumnOp* colOp = m_colOp[op(newColStructList[i].fCompressionType)]; - - // set params - colOp->initColumn(curCol); - colOp->setColParam(curCol, 0, newColStructList[i].colWidth, - newColStructList[i].colDataType, newColStructList[i].colType, newColStructList[i].dataOid, - newColStructList[i].fCompressionType, newColStructList[i].fColDbRoot, - newColStructList[i].fColPartition, newColStructList[i].fColSegment); - - ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(newColStructList[i].dataOid); - ColExtsInfo::iterator it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == newColStructList[i].fColDbRoot) && (it->partNum == newColStructList[i].fColPartition) && (it->segNum == newColStructList[i].fColSegment)) - break; - - it++; - } - - if (it == aColExtsInfo.end()) //add this one to the list - { - ColExtInfo aExt; - aExt.dbRoot = newColStructList[i].fColDbRoot; - aExt.partNum = newColStructList[i].fColPartition; - aExt.segNum = newColStructList[i].fColSegment; - aExt.compType = newColStructList[i].fCompressionType; - aColExtsInfo.push_back(aExt); - aTbaleMetaData->setColExtsInfo(newColStructList[i].dataOid, aColExtsInfo); - } - - // Pass "false" for hdfs tmp file flag. Since we only allow 1 - // extent per segment file (with HDFS), we can assume a second - // extent is going to a new file (and won't need tmp file). - rc = colOp->openColumnFile(curCol, segFile, false, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file - - if (rc != NO_ERROR) - break; - - // handling versioning - vector rangeList; - - if (versioning) - { - rc = processVersionBuffer(curCol.dataFile.pFile, txnid, newColStructList[i], - newColStructList[i].colWidth, totalRow2, secondPart, rangeList); - - if (rc != NO_ERROR) - { - if (newColStructList[i].fCompressionType == 0) - { - curCol.dataFile.pFile->flush(); - } - - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - break; - } - } - - // have to init the size here - switch (newColStructList[i].colType) - { - case WriteEngine::WR_INT: - case WriteEngine::WR_MEDINT: - valArray = (int*) calloc(sizeof(int), totalRow2); - break; - - case WriteEngine::WR_UINT: - case WriteEngine::WR_UMEDINT: - valArray = (uint32_t*) calloc(sizeof(uint32_t), totalRow2); - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR: - case WriteEngine::WR_BLOB: - case WriteEngine::WR_TEXT: - valArray = (char*) calloc(sizeof(char), totalRow2 * MAX_COLUMN_BOUNDARY); - break; - - case WriteEngine::WR_FLOAT: - valArray = (float*) calloc(sizeof(float), totalRow2); - break; - - case WriteEngine::WR_DOUBLE: - valArray = (double*) calloc(sizeof(double), totalRow2); - break; - - case WriteEngine::WR_BYTE: - valArray = (char*) calloc(sizeof(char), totalRow2); - break; - - case WriteEngine::WR_UBYTE: - valArray = (uint8_t*) calloc(sizeof(uint8_t), totalRow2); - break; - - case WriteEngine::WR_SHORT: - valArray = (short*) calloc(sizeof(short), totalRow2); - break; - - case WriteEngine::WR_USHORT: - valArray = (uint16_t*) calloc(sizeof(uint16_t), totalRow2); - break; - - case WriteEngine::WR_LONGLONG: - valArray = (long long*) calloc(sizeof(long long), totalRow2); - break; - - case WriteEngine::WR_ULONGLONG: - valArray = (uint64_t*) calloc(sizeof(uint64_t), totalRow2); - break; - - case WriteEngine::WR_TOKEN: - valArray = (Token*) calloc(sizeof(Token), totalRow2); - break; - - case WriteEngine::WR_BINARY: - //case WriteEngine::WR_INT128: - // WIP - valArray = calloc(totalRow2, 16); - break; - - } - - // convert values to valArray - if (m_opType != DELETE) - { - bExcp = false; - - try - { - convertValArray(totalRow2, newColStructList[i].colType, newColValueList[i], valArray); - } - catch (...) - { - bExcp = true; - } - - if (bExcp) - { - if (versioning) - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - - return ERR_PARSING; - } - -#ifdef PROFILE - timer.start("writeRow "); -#endif - rc = colOp->writeRow(curCol, totalRow2, secondPart, valArray); -#ifdef PROFILE - timer.stop("writeRow "); -#endif - } - else - { -#ifdef PROFILE - timer.start("writeRow "); -#endif - rc = colOp->writeRow(curCol, totalRow2, rowIdArray, valArray, true); -#ifdef PROFILE - timer.stop("writeRow "); -#endif - } - - - colOp->clearColumn(curCol); - - if (versioning) - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - - if (valArray != NULL) - free(valArray); - - // check error - if (rc != NO_ERROR) - break; - } - else - { - valArray = NULL; - - ColumnOp* colOp = m_colOp[op(colStructList[i].fCompressionType)]; - - // set params - colOp->initColumn(curCol); - colOp->setColParam(curCol, 0, colStructList[i].colWidth, - colStructList[i].colDataType, colStructList[i].colType, colStructList[i].dataOid, - colStructList[i].fCompressionType, colStructList[i].fColDbRoot, - colStructList[i].fColPartition, colStructList[i].fColSegment); - - rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file - - //cout << " Opened file oid " << curCol.dataFile.pFile << endl; - if (rc != NO_ERROR) - break; - - ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(colStructList[i].dataOid); - ColExtsInfo::iterator it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == colStructList[i].fColDbRoot) && (it->partNum == colStructList[i].fColPartition) && (it->segNum == colStructList[i].fColSegment)) - break; - - it++; - } - - if (it == aColExtsInfo.end()) //add this one to the list - { - ColExtInfo aExt; - aExt.dbRoot = colStructList[i].fColDbRoot; - aExt.partNum = colStructList[i].fColPartition; - aExt.segNum = colStructList[i].fColSegment; - aExt.compType = colStructList[i].fCompressionType; - aColExtsInfo.push_back(aExt); - aTbaleMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); - } - - // handling versioning - vector rangeList; - - if (versioning) - { - rc = processVersionBuffer(curCol.dataFile.pFile, txnid, colStructList[i], - colStructList[i].colWidth, totalRow1, rowIdArray, rangeList); - - if (rc != NO_ERROR) - { - if (colStructList[i].fCompressionType == 0) - { - curCol.dataFile.pFile->flush(); - } - - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - break; - } - } - - // have to init the size here -// nullArray = (bool*) malloc(sizeof(bool) * totalRow); - switch (colStructList[i].colType) - { - case WriteEngine::WR_INT: - case WriteEngine::WR_MEDINT: - valArray = (int*) calloc(sizeof(int), totalRow1); - break; - - case WriteEngine::WR_UINT: - case WriteEngine::WR_UMEDINT: - valArray = (uint32_t*) calloc(sizeof(uint32_t), totalRow1); - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR: - case WriteEngine::WR_BLOB: - case WriteEngine::WR_TEXT: - valArray = (char*) calloc(sizeof(char), totalRow1 * MAX_COLUMN_BOUNDARY); - break; - - case WriteEngine::WR_FLOAT: - valArray = (float*) calloc(sizeof(float), totalRow1); - break; - - case WriteEngine::WR_DOUBLE: - valArray = (double*) calloc(sizeof(double), totalRow1); - break; - - case WriteEngine::WR_BYTE: - valArray = (char*) calloc(sizeof(char), totalRow1); - break; - - case WriteEngine::WR_UBYTE: - valArray = (uint8_t*) calloc(sizeof(uint8_t), totalRow1); - break; - - case WriteEngine::WR_SHORT: - valArray = (short*) calloc(sizeof(short), totalRow1); - break; - - case WriteEngine::WR_USHORT: - valArray = (uint16_t*) calloc(sizeof(uint16_t), totalRow1); - break; - - case WriteEngine::WR_LONGLONG: - valArray = (long long*) calloc(sizeof(long long), totalRow1); - break; - - case WriteEngine::WR_ULONGLONG: - valArray = (uint64_t*) calloc(sizeof(uint64_t), totalRow1); - break; - - case WriteEngine::WR_TOKEN: - valArray = (Token*) calloc(sizeof(Token), totalRow1); - break; - - case WriteEngine::WR_BINARY: - //case WriteEngine::WR_INT128: - valArray = calloc(colStructList[i].colWidth, totalRow1); - break; - } - - // convert values to valArray - if (m_opType != DELETE) - { - bExcp = false; - - try - { - convertValArray(totalRow1, colStructList[i].colType, colValueList[i], valArray); - } - catch (...) - { - bExcp = true; - } - - if (bExcp) - { - if (versioning) - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - - return ERR_PARSING; - } - -#ifdef PROFILE - timer.start("writeRow "); -#endif - rc = colOp->writeRow(curCol, totalRow1, rowIdArray, valArray); -#ifdef PROFILE - timer.stop("writeRow "); -#endif - } - else - { -#ifdef PROFILE - timer.start("writeRow "); -#endif - rc = colOp->writeRow(curCol, totalRow1, rowIdArray, valArray, true); -#ifdef PROFILE - timer.stop("writeRow "); -#endif - } - - colOp->clearColumn(curCol); - - if (versioning) - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeList); - - if (valArray != NULL) - free(valArray); - - // check error - if (rc != NO_ERROR) - break; - } - } // end of for (i = 0 - -#ifdef PROFILE - timer.finish(); -#endif - return rc; -} - - - int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, const ColStructList& colStructList, std::vector& colValueList, @@ -6637,6 +5602,7 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, } int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, + const CSCTypesList& cscColTypeList, const ColStructList& colStructList, const ColValueList& colValueList, vector& colOldValueList, @@ -6759,11 +5725,6 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, } // handling versioning - //cout << " pass to processVersionBuffer rid " << rowIdArray[0] << endl; - //cout << "dataOid:fColPartition = " << curColStruct.dataOid << ":" << curColStruct.fColPartition << endl; -//timer.start("processVersionBuffers"); - //vector rangeList; - // rc = processVersionBuffers(curCol.dataFile.pFile, txnid, curColStruct, curColStruct.colWidth, totalRow, ridList, rangeList); std::vector curFreeList; uint32_t blockUsed = 0; @@ -6777,7 +5738,6 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, aRange.vbFBO = freeList[0].vbFBO + blocksProcessed; aRange.size = rangeLists[i].size(); curFreeList.push_back(aRange); - //cout << "range size = " << aRange.size <<" and blocksProcessed = " << blocksProcessed<< endl; } else { @@ -6800,14 +5760,10 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, rc = 1; break; } - - //cout << "curFreeList size = " << curFreeList.size() << endl; - } blocksProcessed += rangeLists[i].size(); - //timer.start("Delete:writeVB"); rc = BRMWrapper::getInstance()-> writeVB(curCol.dataFile.pFile, (BRM::VER_t)txnid, curColStruct.dataOid, fboLists[i], rangeLists[i], @@ -6815,9 +5771,6 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, } } - //timer.stop("Delete:writeVB"); -//timer.stop("processVersionBuffers"); - // cout << " rc for processVersionBuffer is " << rc << endl; if (rc != NO_ERROR) { if (curColStruct.fCompressionType == 0) @@ -6886,8 +5839,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, valArray = (Token*) calloc(sizeof(Token), 1); break; case WriteEngine::WR_BINARY: - //case WriteEngine::WR_INT128: - valArray = calloc(sizeof(char), curColStruct.colWidth); //FIXME maybe + valArray = calloc(1, curColStruct.colWidth); break; } @@ -6900,7 +5852,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, try { - convertValue(curColStruct.colType, valArray, curTuple.data); + convertValue(cscColTypeList[i], curColStruct.colType, valArray, curTuple.data); } catch (...) { @@ -6934,7 +5886,6 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, #endif } -// colOldValueList.push_back(oldValArray); //timer.start("Delete:closefile"); colOp->clearColumn(curCol); @@ -6952,340 +5903,6 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, if ((idbdatafile::IDBPolicy::useHdfs()) && (files.size() > 0)) cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); -//if (idbdatafile::IDBPolicy::useHdfs()) -// cacheutils::dropPrimProcFdCache(); -//timer.stop("Delete:purgePrimProcFdCache"); - if (rangeListTot.size() > 0) - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeListTot); - -//timer.stop("Delete:writecolrec"); -//#ifdef PROFILE -//timer.finish(); -//#endif - return rc; -} - - - -int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, - const CSCTypesList& cscColTypes, - const ColStructList& colStructList, - const ColValueList& colValueList, - vector& colOldValueList, - const RIDList& ridList, - const int32_t tableOid, - bool convertStructFlag, - ColTupleList::size_type nRows) -{ - bool bExcp; - int rc = 0; - void* valArray = NULL; - Column curCol; - ColStruct curColStruct; - ColTupleList curTupleList, oldTupleList; - ColStructList::size_type totalColumn; - ColStructList::size_type i; - ColTupleList::size_type totalRow; - - setTransId(txnid); - colOldValueList.clear(); - totalColumn = colStructList.size(); - totalRow = nRows; - -#ifdef PROFILE - StopWatch timer; -#endif - - vector rangeListTot; - std::vector freeList; - vector > fboLists; - vector > rangeLists; - rc = processBeginVBCopy(txnid, colStructList, ridList, freeList, fboLists, rangeLists, rangeListTot); - - if (rc != NO_ERROR) - { - if (rangeListTot.size() > 0) - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeListTot); - - switch (rc) - { - case BRM::ERR_DEADLOCK: - return ERR_BRM_DEAD_LOCK; - - case BRM::ERR_VBBM_OVERFLOW: - return ERR_BRM_VB_OVERFLOW; - - case BRM::ERR_NETWORK: - return ERR_BRM_NETWORK; - - case BRM::ERR_READONLY: - return ERR_BRM_READONLY; - - default: - return ERR_BRM_BEGIN_COPY; - } - } - - VBRange aRange; - uint32_t blocksProcessedThisOid = 0; - uint32_t blocksProcessed = 0; - std::vector files; - TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); - - for (i = 0; i < totalColumn; i++) - { - valArray = NULL; - curColStruct = colStructList[i]; - curTupleList = colValueList[i]; //same value for all rows - ColumnOp* colOp = m_colOp[op(curColStruct.fCompressionType)]; - - // convert column data type - if (convertStructFlag) - Convertor::convertColType(&curColStruct); - - // set params - colOp->initColumn(curCol); - colOp->setColParam(curCol, 0, curColStruct.colWidth, - curColStruct.colDataType, curColStruct.colType, curColStruct.dataOid, - curColStruct.fCompressionType, curColStruct.fColDbRoot, - curColStruct.fColPartition, curColStruct.fColSegment); - - - ColExtsInfo aColExtsInfo = aTbaleMetaData->getColExtsInfo(curColStruct.dataOid); - ColExtsInfo::iterator it = aColExtsInfo.begin(); - - while (it != aColExtsInfo.end()) - { - if ((it->dbRoot == curColStruct.fColDbRoot) && (it->partNum == curColStruct.fColPartition) && (it->segNum == curColStruct.fColSegment)) - break; - - it++; - } - - if (it == aColExtsInfo.end()) //add this one to the list - { - ColExtInfo aExt; - aExt.dbRoot = curColStruct.fColDbRoot; - aExt.partNum = curColStruct.fColPartition; - aExt.segNum = curColStruct.fColSegment; - aExt.compType = curColStruct.fCompressionType; - aColExtsInfo.push_back(aExt); - aTbaleMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); - } - - string segFile; - rc = colOp->openColumnFile(curCol, segFile, true, IO_BUFF_SIZE); // @bug 5572 HDFS tmp file - - if (rc != NO_ERROR) - break; - - if (curColStruct.fCompressionType == 0) - { - BRM::FileInfo aFile; - aFile.oid = curColStruct.dataOid; - aFile.partitionNum = curColStruct.fColPartition; - aFile.dbRoot = curColStruct.fColDbRoot;; - aFile.segmentNum = curColStruct.fColSegment; - aFile.compType = curColStruct.fCompressionType; - files.push_back(aFile); - } - - // handling versioning - //cout << " pass to processVersionBuffer rid " << rowIdArray[0] << endl; - //cout << "dataOid:fColPartition = " << curColStruct.dataOid << ":" << curColStruct.fColPartition << endl; -//timer.start("processVersionBuffers"); - //vector rangeList; - // rc = processVersionBuffers(curCol.dataFile.pFile, txnid, curColStruct, curColStruct.colWidth, totalRow, ridList, rangeList); - std::vector curFreeList; - uint32_t blockUsed = 0; - - if (!idbdatafile::IDBPolicy::useHdfs()) - { - if (rangeListTot.size() > 0) - { - if (freeList[0].size >= (blocksProcessed + rangeLists[i].size())) - { - aRange.vbOID = freeList[0].vbOID; - aRange.vbFBO = freeList[0].vbFBO + blocksProcessed; - aRange.size = rangeLists[i].size(); - curFreeList.push_back(aRange); - //cout << "range size = " << aRange.size <<" and blocksProcessed = " << blocksProcessed<< endl; - } - else - { - aRange.vbOID = freeList[0].vbOID; - aRange.vbFBO = freeList[0].vbFBO + blocksProcessed; - aRange.size = freeList[0].size - blocksProcessed; - blockUsed = aRange.size; - curFreeList.push_back(aRange); - - if (freeList.size() > 1) - { - aRange.vbOID = freeList[1].vbOID; - aRange.vbFBO = freeList[1].vbFBO + blocksProcessedThisOid; - aRange.size = rangeLists[i].size() - blockUsed; - curFreeList.push_back(aRange); - blocksProcessedThisOid += aRange.size; - } - else - { - rc = 1; - break; - } - - //cout << "curFreeList size = " << curFreeList.size() << endl; - - } - - blocksProcessed += rangeLists[i].size(); - - //timer.start("Delete:writeVB"); - rc = BRMWrapper::getInstance()-> - writeVB(curCol.dataFile.pFile, (BRM::VER_t)txnid, - curColStruct.dataOid, fboLists[i], rangeLists[i], - colOp, curFreeList, curColStruct.fColDbRoot, true); - } - } - - //timer.stop("Delete:writeVB"); -//timer.stop("processVersionBuffers"); - // cout << " rc for processVersionBuffer is " << rc << endl; - if (rc != NO_ERROR) - { - if (curColStruct.fCompressionType == 0) - { - curCol.dataFile.pFile->flush(); - } - - if (rangeListTot.size() > 0) - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeListTot); - - break; - } - - switch (curColStruct.colType) - { - case WriteEngine::WR_INT: - case WriteEngine::WR_MEDINT: - valArray = (int*) calloc(sizeof(int), 1); - break; - - case WriteEngine::WR_UINT: - case WriteEngine::WR_UMEDINT: - valArray = (uint32_t*) calloc(sizeof(uint32_t), 1); - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR: - case WriteEngine::WR_BLOB: - case WriteEngine::WR_TEXT: - valArray = (char*) calloc(sizeof(char), 1 * MAX_COLUMN_BOUNDARY); - break; - - case WriteEngine::WR_FLOAT: - valArray = (float*) calloc(sizeof(float), 1); - break; - - case WriteEngine::WR_DOUBLE: - valArray = (double*) calloc(sizeof(double), 1); - break; - - case WriteEngine::WR_BYTE: - valArray = (char*) calloc(sizeof(char), 1); - break; - - case WriteEngine::WR_UBYTE: - valArray = (uint8_t*) calloc(sizeof(uint8_t), 1); - break; - - case WriteEngine::WR_SHORT: - valArray = (short*) calloc(sizeof(short), 1); - break; - - case WriteEngine::WR_USHORT: - valArray = (uint16_t*) calloc(sizeof(uint16_t), 1); - break; - - case WriteEngine::WR_LONGLONG: - valArray = (long long*) calloc(sizeof(long long), 1); - break; - - case WriteEngine::WR_ULONGLONG: - valArray = (uint64_t*) calloc(sizeof(uint64_t), 1); - break; - - case WriteEngine::WR_TOKEN: - valArray = (Token*) calloc(sizeof(Token), 1); - break; - case WriteEngine::WR_BINARY: - //case WriteEngine::WR_INT128: - valArray = calloc(sizeof(char), curColStruct.colWidth); //FIXME maybe - break; - } - - // convert values to valArray - if (m_opType != DELETE) - { - bExcp = false; - ColTuple curTuple; - curTuple = curTupleList[0]; - - try - { - convertValue(curColStruct.colType, valArray, curTuple.data); - } - catch (...) - { - bExcp = true; - } - - if (bExcp) - { - if (rangeListTot.size() > 0) - BRMWrapper::getInstance()->writeVBEnd(txnid, rangeListTot); - - return ERR_PARSING; - } - -#ifdef PROFILE - timer.start("writeRow "); -#endif - rc = colOp->writeRows(curCol, totalRow, ridList, valArray); -#ifdef PROFILE - timer.stop("writeRow "); -#endif - } - else - { -#ifdef PROFILE - timer.start("writeRows "); -#endif - rc = colOp->writeRows(curCol, totalRow, ridList, valArray, 0, true); -#ifdef PROFILE - timer.stop("writeRows "); -#endif - } - -// colOldValueList.push_back(oldValArray); -//timer.start("Delete:closefile"); - colOp->clearColumn(curCol); - -//timer.stop("Delete:closefile"); - if (valArray != NULL) - free(valArray); - - // check error - if (rc != NO_ERROR) - break; - - } // end of for (i = 0) - -// timer.start("Delete:purgePrimProcFdCache"); - if ((idbdatafile::IDBPolicy::useHdfs()) && (files.size() > 0)) - cacheutils::purgePrimProcFdCache(files, Config::getLocalModuleID()); - -//if (idbdatafile::IDBPolicy::useHdfs()) -// cacheutils::dropPrimProcFdCache(); //timer.stop("Delete:purgePrimProcFdCache"); if (rangeListTot.size() > 0) BRMWrapper::getInstance()->writeVBEnd(txnid, rangeListTot); @@ -7750,17 +6367,20 @@ int WriteEngineWrapper::updateNextValue(const TxnID txnId, const OID& columnoid, ColValueList colValueList; WriteEngine::ColTupleList colTuples; ColStructList colStructList; + WriteEngine::CSCTypesList cscColTypeList; WriteEngine::ColStruct colStruct; - colStruct.dataOid = OID_SYSCOLUMN_NEXTVALUE; - colStruct.colWidth = 8; + CalpontSystemCatalog::ColType colType; + colType.columnOID = colStruct.dataOid = OID_SYSCOLUMN_NEXTVALUE; + colType.colWidth = colStruct.colWidth = 8; colStruct.tokenFlag = false; - colStruct.colDataType = CalpontSystemCatalog::UBIGINT; + colType.colDataType = colStruct.colDataType = CalpontSystemCatalog::UBIGINT; colStruct.fColDbRoot = dbRoot; if (idbdatafile::IDBPolicy::useHdfs()) colStruct.fCompressionType = 2; colStructList.push_back(colStruct); + cscColTypeList.push_back(colType); ColTuple colTuple; systemCatalogPtr = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); systemCatalogPtr->identity(CalpontSystemCatalog::EC); @@ -7782,7 +6402,7 @@ int WriteEngineWrapper::updateNextValue(const TxnID txnId, const OID& columnoid, colTuple.data = nextVal; colTuples.push_back(colTuple); colValueList.push_back(colTuples); - rc = writeColumnRecords(txnId, colStructList, colValueList, ridList, SYSCOLUMN_BASE, false); + rc = writeColumnRecords(txnId, cscColTypeList, colStructList, colValueList, ridList, SYSCOLUMN_BASE, false); if (rc != NO_ERROR) return rc; diff --git a/writeengine/wrapper/writeengine.h b/writeengine/wrapper/writeengine.h index f3ca7ec4e..95de97e8d 100644 --- a/writeengine/wrapper/writeengine.h +++ b/writeengine/wrapper/writeengine.h @@ -163,12 +163,6 @@ public: ColTupleList& curTupleList, void* valArray, bool bFromList = true) ; - // WIP legacy - EXPORT void convertValArray(const size_t totalRow, - const ColType colType, - ColTupleList& curTupleList, void* valArray, - bool bFromList = true) ; - /** * @brief Create a column, include object ids for column data and bitmap files * @param dataOid column datafile object id @@ -195,8 +189,8 @@ public: * @param refColDataType Data-type of the referecne column * @param refColWidth Width of the reference column */ - EXPORT int fillColumn(const TxnID& txnid, const OID& dataOid, execplan::CalpontSystemCatalog::ColDataType dataType, - int dataWidth, ColTuple defaultVal, + EXPORT int fillColumn(const TxnID& txnid, const OID& dataOid, const execplan::CalpontSystemCatalog::ColType& colType, + ColTuple defaultVal, const OID& refColOID, execplan::CalpontSystemCatalog::ColDataType refColDataType, int refColWidth, int refCompressionType, bool isNULL, int compressionType, const std::string& defaultValStr, const OID& dictOid = 0, bool autoincrement = false); @@ -230,7 +224,7 @@ public: * @param colOldValueList column old values list (return value) * @param rowIdList row id list */ - EXPORT int deleteRow(const TxnID& txnid, std::vector& colExtentsStruct, + EXPORT int deleteRow(const TxnID& txnid, const std::vector& colExtentsColType, std::vector& colExtentsStruct, std::vector& colOldValueList, std::vector& ridLists, const int32_t tableOid); /** @@ -326,6 +320,7 @@ public: * @param isFirstBatchPm to track if this batch is first batch for this PM. */ EXPORT int insertColumnRecs(const TxnID& txnid, + const CSCTypesList& cscColTypeList, ColStructList& colStructList, ColValueList& colValueList, DctnryStructList& dctnryStructList, @@ -359,6 +354,7 @@ public: * @param dicStringListt dictionary values list */ EXPORT int insertColumnRec_SYS(const TxnID& txnid, + const CSCTypesList& cscColTypeList, ColStructList& colStructList, ColValueList& colValueList, DctnryStructList& dctnryStructList, @@ -372,7 +368,7 @@ public: * @param dicStringListt dictionary values list */ EXPORT int insertColumnRec_Single(const TxnID& txnid, - CSCTypesList& cscColTypesList, + const CSCTypesList& cscColTypeList, ColStructList& colStructList, ColValueList& colValueList, DctnryStructList& dctnryStructList, @@ -550,6 +546,7 @@ public: * @param ridList row id list */ EXPORT int updateColumnRec(const TxnID& txnid, + const std::vector& colExtentsColType, std::vector& colExtentsStruct, ColValueList& colValueList, std::vector& colOldValueList, @@ -566,6 +563,7 @@ public: */ EXPORT int updateColumnRecs(const TxnID& txnid, + const CSCTypesList& cscColTypeList, std::vector& colStructList, ColValueList& colValueList, const RIDList& ridLists, @@ -657,10 +655,10 @@ private: void findSmallestColumn(uint32_t &colId, ColStructList colStructList); /** - * @brief Convert interface column type to a internal column type + * @brief Convert interface column type to an internal column type */ - void convertValue(const execplan::CalpontSystemCatalog::ColType &fullColType, ColType colType, void* valArray, size_t pos, boost::any& data, bool fromList = true); - void convertValue(const ColType colType, void* valArray, size_t pos, boost::any& data, bool fromList = true); + void convertValue(const execplan::CalpontSystemCatalog::ColType& cscColType, ColType colType, void* valArray, size_t pos, boost::any& data, bool fromList = true); + /** * @brief Convert column value to its internal representation * @@ -668,8 +666,7 @@ private: * @param value Memory pointer for storing output value. Should be pre-allocated * @param data Column data */ - void convertValue(const execplan::CalpontSystemCatalog::ColType &fullColType, const ColType colType, void* value, boost::any& data); - void convertValue(const ColType colType, void* value, boost::any& data); + void convertValue(const execplan::CalpontSystemCatalog::ColType& cscColType, const ColType colType, void* value, boost::any& data); /** * @brief Print input value from DDL/DML processors @@ -705,14 +702,6 @@ private: RID* rowIdArray, const ColStructList& newColStructList, ColValueList& newColValueList, const int32_t tableOid, bool useTmpSuffix, bool versioning = true); - // WIP - int writeColumnRec(const TxnID& txnid, - const ColStructList& colStructList, - ColValueList& colValueList, - RID* rowIdArray, const ColStructList& newColStructList, - ColValueList& newColValueList, const int32_t tableOid, - bool useTmpSuffix, bool versioning = true); - int writeColumnRecBinary(const TxnID& txnid, const ColStructList& colStructList, std::vector& colValueList, @@ -721,24 +710,17 @@ private: const int32_t tableOid, bool useTmpSuffix, bool versioning = true); - //@Bug 1886,2870 pass the address of ridList vector int writeColumnRec(const TxnID& txnid, - const CSCTypesList& cscColTypes, + const CSCTypesList& cscColTypeList, const ColStructList& colStructList, const ColValueList& colValueList, std::vector& colOldValueList, const RIDList& ridList, const int32_t tableOid, bool convertStructFlag = true, ColTupleList::size_type nRows = 0); - // WIP legacy - int writeColumnRec(const TxnID& txnid, - const ColStructList& colStructList, - const ColValueList& colValueList, std::vector& colOldValueList, - const RIDList& ridList, const int32_t tableOid, - bool convertStructFlag = true, ColTupleList::size_type nRows = 0); - //For update column from column to use - int writeColumnRecords(const TxnID& txnid, std::vector& colStructList, + int writeColumnRecords(const TxnID& txnid, const CSCTypesList& cscColTypeList, + std::vector& colStructList, ColValueList& colValueList, const RIDList& ridLists, const int32_t tableOid, bool versioning = true); From 2eb5af1d242cd027102c0c47a0503466f0853b52 Mon Sep 17 00:00:00 2001 From: drrtuy Date: Tue, 18 Feb 2020 11:55:51 +0300 Subject: [PATCH 14/78] MCOL-641 This commit adds support for SIGNED and ZEROFILL keywords in CREATE TABLE. ZEROFILL is dummy though. There is a new file with column width utilities. Array access was replaced by a variable that is calculated only once in TupleJoiner::updateCPData. --- dbcon/ddlpackage/ddl.l | 2 + dbcon/ddlpackage/ddl.y | 25 ++++++++---- dbcon/ddlpackage/ddlpkg.cpp | 42 +------------------- utils/common/columnwidth.h | 74 ++++++++++++++++++++++++++++++++++++ utils/joiner/tuplejoiner.cpp | 29 ++++++++------ utils/joiner/tuplejoiner.h | 1 + 6 files changed, 115 insertions(+), 58 deletions(-) create mode 100644 utils/common/columnwidth.h diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index a6ab67d59..61741cb6b 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -156,6 +156,7 @@ REFERENCES {return REFERENCES;} RENAME {return RENAME;} RESTRICT {return RESTRICT;} SESSION_USER {return SESSION_USER;} +SIGNED {return SIGNED;} SYSTEM_USER {return SYSTEM_USER;} SET {return SET;} SMALLINT {return SMALLINT;} @@ -189,6 +190,7 @@ BOOL {return BOOL;} BOOLEAN {return BOOLEAN;} MEDIUMINT {return MEDIUMINT;} BINARY {return BINARY;} +ZEROFILL {return ZEROFILL;} \n { lineno++;} diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index 510c7a3b0..34af1c593 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -63,6 +63,7 @@ char* copy_string(const char *str); %pure-parser %lex-param {void * scanner} %parse-param {struct ddlpackage::pass_to_bison * x} +%debug /* Bison uses this to generate a C union definition. This is used to store the application created values associated with syntactic @@ -104,15 +105,17 @@ char* copy_string(const char *str); %token ACTION ADD ALTER AUTO_INCREMENT BIGINT BIT BLOB IDB_BLOB CASCADE IDB_CHAR CHARACTER CHECK CLOB COLUMN +BOOL BOOLEAN BINARY COLUMNS COMMENT CONSTRAINT CONSTRAINTS CREATE CURRENT_USER DATETIME DEC DECIMAL DEFAULT DEFERRABLE DEFERRED IDB_DELETE DROP ENGINE FOREIGN FULL IMMEDIATE INDEX INITIALLY IDB_INT INTEGER KEY LONGBLOB LONGTEXT -MATCH MAX_ROWS MEDIUMBLOB MEDIUMTEXT +MATCH MAX_ROWS MEDIUMBLOB MEDIUMTEXT MEDIUMINT MIN_ROWS MODIFY NO NOT NULL_TOK NUMBER NUMERIC ON PARTIAL PRECISION PRIMARY REFERENCES RENAME RESTRICT SET SMALLINT TABLE TEXT TINYBLOB TINYTEXT -TINYINT TO UNIQUE UNSIGNED UPDATE USER SESSION_USER SYSTEM_USER VARCHAR VARBINARY +TINYINT TO UNIQUE UNSIGNED UPDATE USER SESSION_USER SIGNED SYSTEM_USER VARCHAR VARBINARY VARYING WITH ZONE DOUBLE IDB_FLOAT REAL CHARSET COLLATE IDB_IF EXISTS CHANGE TRUNCATE -BOOL BOOLEAN MEDIUMINT TIMESTAMP BINARY +TIMESTAMP +ZEROFILL %token DQ_IDENT IDENT FCONST SCONST CP_SEARCH_CONDITION_TEXT ICONST DATE TIME @@ -197,6 +200,8 @@ BOOL BOOLEAN MEDIUMINT TIMESTAMP BINARY %type opt_display_precision_scale_null %type opt_if_exists %type opt_if_not_exists +%type opt_signed +%type opt_zerofill %type trunc_table_statement %type rename_table_statement %type ident @@ -1002,16 +1007,14 @@ exact_numeric_type: $2->fLength = DDLDatatypeLength[DDL_UNSIGNED_NUMERIC]; $$ = $2; } - | DECIMAL opt_precision_scale + | DECIMAL opt_precision_scale opt_signed opt_zerofill { $2->fType = DDL_DECIMAL; -/* $2->fLength = DDLDatatypeLength[DDL_DECIMAL]; */ $$ = $2; } - | DECIMAL opt_precision_scale UNSIGNED + | DECIMAL opt_precision_scale UNSIGNED opt_zerofill { $2->fType = DDL_UNSIGNED_DECIMAL; -/* $3->fLength = DDLDatatypeLength[DDL_DECIMAL]; */ $$ = $2; } | NUMBER opt_precision_scale @@ -1106,6 +1109,14 @@ opt_precision_scale: | {$$ = new ColumnType(10,0);} ; +opt_signed: + SIGNED {$$ = NULL;} + | {$$ = NULL;} + +opt_zerofill: + ZEROFILL {$$ = NULL;} + | {$$ = NULL;} + opt_display_width: '(' ICONST ')' {$$ = NULL;} | {$$ = NULL;} diff --git a/dbcon/ddlpackage/ddlpkg.cpp b/dbcon/ddlpackage/ddlpkg.cpp index ecf6264dc..90bfa45a9 100644 --- a/dbcon/ddlpackage/ddlpkg.cpp +++ b/dbcon/ddlpackage/ddlpkg.cpp @@ -26,6 +26,7 @@ #define DDLPKG_DLLEXPORT #include "ddlpkg.h" #undef DDLPKG_DLLEXPORT +#include "../../utils/common/columnwidth.h" namespace ddlpackage { @@ -61,32 +62,6 @@ ostream& operator<<(ostream& os, const QualifiedName& qname) return os; } - -/** @brief Map a DECIMAL precision to data width in bytes */ -unsigned int precision_width(unsigned p) -{ - switch (p) - { - case 1: - case 2: - return 1; - - case 3: - case 4: - return 2; - - case 5: - case 6: - case 7: - case 8: - case 9: - return 4; - - default: - return 8; - } -} - ColumnType::ColumnType(int prec, int scale) : fType(DDL_INVALID_DATATYPE), fLength(0), @@ -94,7 +69,7 @@ ColumnType::ColumnType(int prec, int scale) : fScale(scale), fWithTimezone(false) { - fLength = precision_width(fPrecision); + fLength = utils::widthByPrecision(fPrecision); } ColumnType::ColumnType(int type) : @@ -141,19 +116,6 @@ ColumnType::ColumnType(int type) : break; } } -#if 0 -ColumnType::ColumnType(int type, int length, int precision, int scale, int compressiontype, const char* autoIncrement, int64_t nextValue, bool withTimezone) : - fType(type), - fLength(length), - fPrecision(precision), - fScale(scale), - fWithTimezone(withTimezone), - fCompressiontype(compressiontype), - fAutoincrement(autoIncrement), - fNextvalue(nextValue) -{ -} -#endif ColumnConstraintDef::ColumnConstraintDef(DDL_CONSTRAINTS type) : SchemaObject(), fDeferrable(false), diff --git a/utils/common/columnwidth.h b/utils/common/columnwidth.h new file mode 100644 index 000000000..dfcf767d4 --- /dev/null +++ b/utils/common/columnwidth.h @@ -0,0 +1,74 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#ifndef UTILS_COLWIDTH_H +#define UTILS_COLWIDTH_H + +#define MAXLEGACYWIDTH 8 + +namespace utils +{ + inline bool isWide(uint8_t width) + { + return width > MAXLEGACYWIDTH; + } + + inline bool isNarrow(uint8_t width) + { + return width <= MAXLEGACYWIDTH; + } + + // WIP MCOL-641 Replace with template + /** @brief Map a DECIMAL precision to data width in bytes */ + inline uint8_t widthByPrecision(unsigned p) + { + switch (p) + { + case 1: + case 2: + return 1; + + case 3: + case 4: + return 2; + + case 5: + case 6: + case 7: + case 8: + case 9: + return 4; + + case 10: + case 11: + case 12: + case 13: + case 14: + case 15: + case 16: + case 17: + case 18: + return 8; + + default: + return 16; + } + } + +} + +#endif // UTILS_COLWIDTH_H diff --git a/utils/joiner/tuplejoiner.cpp b/utils/joiner/tuplejoiner.cpp index 30ad8c3dc..9dd6626c2 100644 --- a/utils/joiner/tuplejoiner.cpp +++ b/utils/joiner/tuplejoiner.cpp @@ -1070,15 +1070,15 @@ void TupleJoiner::updateCPData(const Row& r) for (col = 0; col < smallKeyColumns.size(); col++) { -// if (r.getColumnWidth(smallKeyColumns[col]) > 8) - if (r.isLongString(smallKeyColumns[col])) + auto colIdx = smallKeyColumns[col]; + if (r.isLongString(colIdx)) continue; - int64_t& min = cpValues[col][0], &max = cpValues[col][1]; + auto& min = cpValues[col][0], &max = cpValues[col][1]; - if (r.isCharType(smallKeyColumns[col])) + if (r.isCharType(colIdx)) { - int64_t val = r.getIntField(smallKeyColumns[col]); + int64_t val = r.getIntField(colIdx); if (order_swap(val) < order_swap(min) || min == numeric_limits::max()) @@ -1092,10 +1092,10 @@ void TupleJoiner::updateCPData(const Row& r) max = val; } } - else if (r.isUnsigned(smallKeyColumns[col])) + else if (r.isUnsigned(colIdx)) { uint64_t uval; - if (r.getColType(smallKeyColumns[col]) == CalpontSystemCatalog::LONGDOUBLE) + if (r.getColType(colIdx) == CalpontSystemCatalog::LONGDOUBLE) { double dval = (double)roundl(r.getLongDoubleField(smallKeyColumns[col])); switch (largeRG.getColType(largeKeyColumns[col])) @@ -1116,7 +1116,7 @@ void TupleJoiner::updateCPData(const Row& r) } else { - uval = r.getUintField(smallKeyColumns[col]); + uval = r.getUintField(colIdx); } if (uval > static_cast(max)) @@ -1128,9 +1128,9 @@ void TupleJoiner::updateCPData(const Row& r) else { int64_t val; - if (r.getColType(smallKeyColumns[col]) == CalpontSystemCatalog::LONGDOUBLE) + if (r.getColType(colIdx) == CalpontSystemCatalog::LONGDOUBLE) { - double dval = (double)roundl(r.getLongDoubleField(smallKeyColumns[col])); + double dval = (double)roundl(r.getLongDoubleField(colIdx)); switch (largeRG.getColType(largeKeyColumns[col])) { case CalpontSystemCatalog::DOUBLE: @@ -1147,9 +1147,16 @@ void TupleJoiner::updateCPData(const Row& r) } } } + else if (utils::isWide(r.getColumnWidth(colIdx)) + && (r.getColType(colIdx) == CalpontSystemCatalog::DECIMAL + || r.getColType(colIdx) == CalpontSystemCatalog::UDECIMAL)) + { + // WIP MCOL-641 + } + else { - val = r.getIntField(smallKeyColumns[col]); + val = r.getIntField(colIdx); } if (val > max) diff --git a/utils/joiner/tuplejoiner.h b/utils/joiner/tuplejoiner.h index b0d38364d..d91297845 100644 --- a/utils/joiner/tuplejoiner.h +++ b/utils/joiner/tuplejoiner.h @@ -39,6 +39,7 @@ #include "stlpoolallocator.h" #include "hasher.h" #include "threadpool.h" +#include "columnwidth.h" namespace joiner { From 31d597d87e020473d92df993670222173621e2c7 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 18 Feb 2020 12:30:30 +0000 Subject: [PATCH 15/78] MCOL-641 This commit enables CS to return a warning on non-supported ZEROFILL keyword. The change potentially replaces tabs with spaces in the bison's ddl.y file. --- dbcon/ddlpackage/ddl.y | 229 ++++++++++++++++++------------------- dbcon/mysql/ha_mcs_ddl.cpp | 24 ++++ 2 files changed, 138 insertions(+), 115 deletions(-) diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index 34af1c593..974dc6ad8 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -63,7 +63,6 @@ char* copy_string(const char *str); %pure-parser %lex-param {void * scanner} %parse-param {struct ddlpackage::pass_to_bison * x} -%debug /* Bison uses this to generate a C union definition. This is used to store the application created values associated with syntactic @@ -1007,7 +1006,7 @@ exact_numeric_type: $2->fLength = DDLDatatypeLength[DDL_UNSIGNED_NUMERIC]; $$ = $2; } - | DECIMAL opt_precision_scale opt_signed opt_zerofill + | DECIMAL opt_precision_scale opt_signed { $2->fType = DDL_DECIMAL; $$ = $2; @@ -1113,128 +1112,128 @@ opt_signed: SIGNED {$$ = NULL;} | {$$ = NULL;} -opt_zerofill: - ZEROFILL {$$ = NULL;} - | {$$ = NULL;} + opt_zerofill: + ZEROFILL {$$ = NULL;} + | {$$ = NULL;} -opt_display_width: - '(' ICONST ')' {$$ = NULL;} - | {$$ = NULL;} - ; - -approximate_numeric_type: - DOUBLE opt_display_precision_scale_null - { - $$ = new ColumnType(DDL_DOUBLE); - $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; - } - | DOUBLE opt_display_precision_scale_null UNSIGNED - { - $$ = new ColumnType(DDL_UNSIGNED_DOUBLE); - $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; - } - | DOUBLE PRECISION opt_display_precision_scale_null - { - $$ = new ColumnType(DDL_DOUBLE); - $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; - } - | DOUBLE PRECISION opt_display_precision_scale_null UNSIGNED - { - $$ = new ColumnType(DDL_UNSIGNED_DOUBLE); - $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; - } - | REAL opt_display_precision_scale_null - { - $$ = new ColumnType(DDL_DOUBLE); - $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; - } - | REAL opt_display_precision_scale_null UNSIGNED - { - $$ = new ColumnType(DDL_UNSIGNED_DOUBLE); - $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; - } - | IDB_FLOAT opt_display_precision_scale_null - { - $$ = new ColumnType(DDL_FLOAT); - $$->fLength = DDLDatatypeLength[DDL_FLOAT]; - } - | IDB_FLOAT opt_display_precision_scale_null UNSIGNED - { - $$ = new ColumnType(DDL_UNSIGNED_FLOAT); - $$->fLength = DDLDatatypeLength[DDL_FLOAT]; - } - ; - -opt_display_precision_scale_null: - '(' ICONST ')' {$$ = NULL;} - | - '(' ICONST ',' ICONST ')' {$$ = NULL;} + opt_display_width: + '(' ICONST ')' {$$ = NULL;} | {$$ = NULL;} ; - -literal: - ICONST - | string_literal - | FCONST - ; -datetime_type: - DATETIME opt_time_precision - { - $$ = new ColumnType(DDL_DATETIME); - $$->fLength = DDLDatatypeLength[DDL_DATETIME]; - $$->fPrecision = $2; - } - | - DATE - { - $$ = new ColumnType(DDL_DATE); - $$->fLength = DDLDatatypeLength[DDL_DATE]; - } - | - TIME opt_time_precision - { - $$ = new ColumnType(DDL_TIME); - $$->fLength = DDLDatatypeLength[DDL_TIME]; - $$->fPrecision = $2; - } - | - TIMESTAMP opt_time_precision - { - $$ = new ColumnType(DDL_TIMESTAMP); - $$->fLength = DDLDatatypeLength[DDL_TIMESTAMP]; - $$->fPrecision = $2; - } + approximate_numeric_type: + DOUBLE opt_display_precision_scale_null + { + $$ = new ColumnType(DDL_DOUBLE); + $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; + } + | DOUBLE opt_display_precision_scale_null UNSIGNED + { + $$ = new ColumnType(DDL_UNSIGNED_DOUBLE); + $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; + } + | DOUBLE PRECISION opt_display_precision_scale_null + { + $$ = new ColumnType(DDL_DOUBLE); + $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; + } + | DOUBLE PRECISION opt_display_precision_scale_null UNSIGNED + { + $$ = new ColumnType(DDL_UNSIGNED_DOUBLE); + $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; + } + | REAL opt_display_precision_scale_null + { + $$ = new ColumnType(DDL_DOUBLE); + $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; + } + | REAL opt_display_precision_scale_null UNSIGNED + { + $$ = new ColumnType(DDL_UNSIGNED_DOUBLE); + $$->fLength = DDLDatatypeLength[DDL_DOUBLE]; + } + | IDB_FLOAT opt_display_precision_scale_null + { + $$ = new ColumnType(DDL_FLOAT); + $$->fLength = DDLDatatypeLength[DDL_FLOAT]; + } + | IDB_FLOAT opt_display_precision_scale_null UNSIGNED + { + $$ = new ColumnType(DDL_UNSIGNED_FLOAT); + $$->fLength = DDLDatatypeLength[DDL_FLOAT]; + } + ; -opt_time_precision: - '(' ICONST ')' {$$ = atoi($2);} - | {$$ = -1;} - ; + opt_display_precision_scale_null: + '(' ICONST ')' {$$ = NULL;} + | + '(' ICONST ',' ICONST ')' {$$ = NULL;} + | {$$ = NULL;} + ; + + literal: + ICONST + | string_literal + | FCONST + ; -drop_column_def: - DROP column_name drop_behavior {$$ = new AtaDropColumn($2, $3);} - | DROP COLUMN column_name drop_behavior {$$ = new AtaDropColumn($3, $4);} - | DROP COLUMN '(' column_name_list ')' {$$ = new AtaDropColumns($4);} - | DROP '(' column_name_list ')' {$$ = new AtaDropColumns($3);} - | DROP COLUMNS '(' column_name_list ')' {$$ = new AtaDropColumns($4);} - ; + datetime_type: + DATETIME opt_time_precision + { + $$ = new ColumnType(DDL_DATETIME); + $$->fLength = DDLDatatypeLength[DDL_DATETIME]; + $$->fPrecision = $2; + } + | + DATE + { + $$ = new ColumnType(DDL_DATE); + $$->fLength = DDLDatatypeLength[DDL_DATE]; + } + | + TIME opt_time_precision + { + $$ = new ColumnType(DDL_TIME); + $$->fLength = DDLDatatypeLength[DDL_TIME]; + $$->fPrecision = $2; + } + | + TIMESTAMP opt_time_precision + { + $$ = new ColumnType(DDL_TIMESTAMP); + $$->fLength = DDLDatatypeLength[DDL_TIMESTAMP]; + $$->fPrecision = $2; + } -drop_behavior: - CASCADE {$$ = DDL_CASCADE;} - | RESTRICT {$$ = DDL_RESTRICT;} - | {$$ = DDL_NO_ACTION;} - ; + opt_time_precision: + '(' ICONST ')' {$$ = atoi($2);} + | {$$ = -1;} + ; -alter_column_def: - ALTER opt_column column_name SET default_clause {$$ = new AtaSetColumnDefault($3, $5);} - | ALTER opt_column column_name DROP DEFAULT {$$ = new AtaDropColumnDefault($3);} - ; + drop_column_def: + DROP column_name drop_behavior {$$ = new AtaDropColumn($2, $3);} + | DROP COLUMN column_name drop_behavior {$$ = new AtaDropColumn($3, $4);} + | DROP COLUMN '(' column_name_list ')' {$$ = new AtaDropColumns($4);} + | DROP '(' column_name_list ')' {$$ = new AtaDropColumns($3);} + | DROP COLUMNS '(' column_name_list ')' {$$ = new AtaDropColumns($4);} + ; -opt_column: - COLUMN - | - ; + drop_behavior: + CASCADE {$$ = DDL_CASCADE;} + | RESTRICT {$$ = DDL_RESTRICT;} + | {$$ = DDL_NO_ACTION;} + ; -%% + alter_column_def: + ALTER opt_column column_name SET default_clause {$$ = new AtaSetColumnDefault($3, $5);} + | ALTER opt_column column_name DROP DEFAULT {$$ = new AtaDropColumnDefault($3);} + ; + + opt_column: + COLUMN + | + ; + + %% diff --git a/dbcon/mysql/ha_mcs_ddl.cpp b/dbcon/mysql/ha_mcs_ddl.cpp index 08cfe199d..32daf2c48 100644 --- a/dbcon/mysql/ha_mcs_ddl.cpp +++ b/dbcon/mysql/ha_mcs_ddl.cpp @@ -2299,6 +2299,22 @@ static bool get_field_default_value(THD *thd, Field *field, String *def_value, return has_default; } +/* + Utility function search for ZEROFILL +*/ + +bool hasZerofillDecimal(TABLE *table_arg) +{ + for (Field **field= table_arg->field; *field; field++) + { + if (((*field)->flags & ZEROFILL_FLAG) + && typeid (**field) == typeid(Field_new_decimal)) + return true; + } + + return false; +} + int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* create_info, cal_connection_info& ci) { #ifdef MCS_DEBUG @@ -2578,6 +2594,14 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea cout << "ha_mcs_impl_create_: ProcessDDL error, now in state NOT_ALTER" << endl; #endif } + else + { + if (hasZerofillDecimal(table_arg)) + { + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, + WARN_OPTION_IGNORED, "ZEROFILL is ignored in ColumnStore"); + } + } return rc; } From f73de304271290ec1dc1b641cdc76e80bb824799 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Wed, 19 Feb 2020 18:50:48 +0000 Subject: [PATCH 16/78] MCOL-641 This commit introduces GTest Suite into CS. Binary NULL magic now consists of a series of BINARYEMPTYROW-s + BINARYNULL in the end. ByteStream now has hexbyte alias. Added ColumnCommand::getEmptyRowValue to support 16 byte EMPTY values. --- CMakeLists.txt | 7 ++ cmake/FindGTest.cmake | 38 ++++++++ primitives/linux-port/column.cpp | 9 +- primitives/primproc/columncommand.cpp | 21 +++- primitives/primproc/columncommand.h | 8 +- utils/common/columnwidth.h | 1 + utils/messageqcpp/bytestream.cpp | 12 +-- utils/messageqcpp/bytestream.h | 9 +- utils/rowgroup/CMakeLists.txt | 5 + utils/rowgroup/rowaggregation.cpp | 10 +- utils/rowgroup/rowgroup-tests.cpp | 110 +++++++++++++++++++++ utils/rowgroup/rowgroup.cpp | 134 ++++++++++++++------------ utils/rowgroup/rowgroup.h | 6 +- 13 files changed, 283 insertions(+), 87 deletions(-) create mode 100644 cmake/FindGTest.cmake create mode 100644 utils/rowgroup/rowgroup-tests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index bc7033ccb..7a084400d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -148,6 +148,13 @@ if (NOT CURL_FOUND) return() endif() +IF (WITH_GTEST) + INCLUDE (FindGTest) + IF (NOT GTEST_FOUND) + MESSAGE(FATAL_ERROR "GSuite libs not found but are requested. Please install them or build.") + ENDIF() + SET (GTEST_LIBRARIES ${GTEST_LIBRARY} ${GTESTMAIN_LIBRARY} ${PTHREAD_LIBRARY}) +ENDIF() FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable") if(NOT AWK_EXECUTABLE) diff --git a/cmake/FindGTest.cmake b/cmake/FindGTest.cmake new file mode 100644 index 000000000..7b1fb3957 --- /dev/null +++ b/cmake/FindGTest.cmake @@ -0,0 +1,38 @@ +find_path(GTEST_ROOT_DIR + NAMES include/gtest/gtest.h +) + +find_library(GTEST_LIBRARY + NAMES gtest + HINTS ${GTEST_ROOT_DIR}/lib +) + +find_library(GTESTMAIN_LIBRARY + NAMES gtest_main + HINTS ${GTEST_ROOT_DIR}/lib +) + +find_library(PTHREAD_LIBRARY + NAMES pthread + HINTS ${GTEST_ROOT_DIR}/lib +) + + +find_path(GTEST_INCLUDE_DIR + NAMES gtest.h + HINTS ${GTEST_ROOT_DIR}/include +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(GTest DEFAULT_MSG + GTEST_LIBRARY + GTESTMAIN_LIBRARY + PTHREAD_LIBRARY + GTEST_INCLUDE_DIR +) + +mark_as_advanced( + GTEST_ROOT_DIR + GTEST_LIBRARIES + GTEST_INCLUDE_DIR +) diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index c8fd6673d..e246e8e95 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -285,7 +285,6 @@ template<> inline bool isEmptyVal<16>(uint8_t type, const uint8_t* ival) // For BINARY { const uint64_t* val = reinterpret_cast(ival); - // WIP ugly speed hack return ((val[0] == joblist::BINARYEMPTYROW) && (val[1] == joblist::BINARYEMPTYROW)); } @@ -415,15 +414,15 @@ template<> inline bool isNullVal<16>(uint8_t type, const uint8_t* ival) // For BINARY { const uint64_t* val = reinterpret_cast(ival); - return ((val[0] == joblist::BINARYNULL) && (val[1] == joblist::BINARYNULL)); + return ((val[0] == joblist::BINARYEMPTYROW) && (val[1] == joblist::BINARYNULL)); } template<> inline bool isNullVal<32>(uint8_t type, const uint8_t* ival) // For BINARY { const uint64_t* val = reinterpret_cast(ival); - return ((val[0] == joblist::BINARYNULL) && (val[1] == joblist::BINARYNULL) - && (val[2] == joblist::BINARYNULL) && (val[3] == joblist::BINARYNULL)); + return ((val[0] == joblist::BINARYEMPTYROW) && (val[1] == joblist::BINARYEMPTYROW) + && (val[2] == joblist::BINARYEMPTYROW) && (val[3] == joblist::BINARYNULL)); } template<> @@ -614,7 +613,7 @@ inline bool isMinMaxValid(const NewColRequestHeader* in) case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - return (in->DataSize <= 16); + return (in->DataSize <= 16 ); default: return false; diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index ac33e47a3..4d4d7bf79 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -165,6 +165,7 @@ void ColumnCommand::loadData() { // fill remaining blocks with empty values when col scan int blockLen = BLOCK_SIZE / colType.colWidth; + ByteStream::hexbyte* hPtr = NULL; ByteStream::octbyte* oPtr = NULL; ByteStream::quadbyte* qPtr = NULL; ByteStream::byte* bPtr = NULL; @@ -183,6 +184,10 @@ void ColumnCommand::loadData() if (colType.colWidth == 8) oPtr = reinterpret_cast(&bpp->blockData[i * BLOCK_SIZE]); + if (colType.colWidth == 16) + hPtr = reinterpret_cast(&bpp->blockData[i * BLOCK_SIZE]); + + for (int idx = 0; idx < blockLen; idx++) { if (bPtr && colType.colWidth == 1) @@ -208,9 +213,7 @@ void ColumnCommand::loadData() } else if (colType.colWidth == 16) { - uint64_t *ptr = reinterpret_cast(&bpp->blockData[i * BLOCK_SIZE] + (idx*16) ); - *ptr = joblist::BINARYEMPTYROW; - *(ptr + 1) = joblist::BINARYEMPTYROW; + getEmptyRowValue(colType.colDataType, colType.colWidth, &hPtr[idx]); } } @@ -965,7 +968,7 @@ void ColumnCommand::enableFilters() * RETURN: * emptyVal - the value of empty row ***********************************************************/ -uint64_t ColumnCommand::getEmptyRowValue( const execplan::CalpontSystemCatalog::ColDataType dataType, const int width ) const +const uint64_t ColumnCommand::getEmptyRowValue( const CSCDataType dataType, const int width ) const { uint64_t emptyVal = 0; int offset; @@ -1056,6 +1059,16 @@ uint64_t ColumnCommand::getEmptyRowValue( const execplan::CalpontSystemCatalog:: return emptyVal; } +void ColumnCommand::getEmptyRowValue(const CSCDataType dataType, + const int width, messageqcpp::ByteStream::hexbyte* space) const +{ + uint64_t *ptr = reinterpret_cast(space); + ptr[0] = joblist::BINARYEMPTYROW; + ptr[1] = joblist::BINARYEMPTYROW; +} + + + void ColumnCommand::getLBIDList(uint32_t loopCount, vector* lbids) { int64_t firstLBID = lbid, lastLBID = firstLBID + (loopCount * colType.colWidth) - 1, i; diff --git a/primitives/primproc/columncommand.h b/primitives/primproc/columncommand.h index e1d67aa0d..52544021f 100644 --- a/primitives/primproc/columncommand.h +++ b/primitives/primproc/columncommand.h @@ -34,6 +34,8 @@ #include "command.h" #include "calpontsystemcatalog.h" +using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; + namespace primitiveprocessor { @@ -82,8 +84,10 @@ public: makeAbsRids = m; } bool willPrefetch(); - uint64_t getEmptyRowValue( const execplan::CalpontSystemCatalog::ColDataType dataType, const int width ) const; - int64_t getLastLbid(); + const uint64_t getEmptyRowValue( const CSCDataType dataType, const int width ) const; + void getEmptyRowValue(const CSCDataType dataType, + const int width, messageqcpp::ByteStream::hexbyte* space) const; + const int64_t getLastLbid(); void getLBIDList(uint32_t loopCount, std::vector* lbids); virtual SCommand duplicate(); diff --git a/utils/common/columnwidth.h b/utils/common/columnwidth.h index dfcf767d4..8e36f5356 100644 --- a/utils/common/columnwidth.h +++ b/utils/common/columnwidth.h @@ -19,6 +19,7 @@ #define UTILS_COLWIDTH_H #define MAXLEGACYWIDTH 8 +#define MAXCOLUMNWIDTH 16 namespace utils { diff --git a/utils/messageqcpp/bytestream.cpp b/utils/messageqcpp/bytestream.cpp index 4cc4a103a..73ca1487c 100644 --- a/utils/messageqcpp/bytestream.cpp +++ b/utils/messageqcpp/bytestream.cpp @@ -236,12 +236,12 @@ ByteStream& ByteStream::operator<<(const uint64_t o) } // WIP MCOL-641 -ByteStream& ByteStream::operator<<(const unsigned __int128 o) +ByteStream& ByteStream::operator<<(const uint128_t o) { if (fBuf == 0 || (fCurInPtr - fBuf + 16U > fMaxLen + ISSOverhead)) growBuf(fMaxLen + BlockSize); - *((unsigned __int128*) fCurInPtr) = o; + *((uint128_t*) fCurInPtr) = o; fCurInPtr += 16; return *this; @@ -332,7 +332,7 @@ ByteStream& ByteStream::operator>>(uint64_t& o) } // WIP MCOL-641 -ByteStream& ByteStream::operator>>(unsigned __int128& o) +ByteStream& ByteStream::operator>>(uint128_t& o) { peek(o); fCurOutPtr += 16; @@ -420,13 +420,13 @@ void ByteStream::peek(uint64_t& o) const } // WIP MCOL-641 -void ByteStream::peek(unsigned __int128& o) const +void ByteStream::peek(uint128_t& o) const { if (length() < 16) - throw underflow_error("ByteStream>unsigned __int128: not enough data in stream to fill datatype"); + throw underflow_error("ByteStream>uint128_t: not enough data in stream to fill datatype"); - o = *((unsigned __int128*) fCurOutPtr); + o = *((uint128_t*) fCurOutPtr); } void ByteStream::peek(string& s) const diff --git a/utils/messageqcpp/bytestream.h b/utils/messageqcpp/bytestream.h index 16d547fba..9f5339243 100644 --- a/utils/messageqcpp/bytestream.h +++ b/utils/messageqcpp/bytestream.h @@ -45,6 +45,8 @@ class ByteStreamTestSuite; #define EXPORT #endif +using uint128_t = unsigned __int128; + namespace messageqcpp { @@ -74,6 +76,7 @@ public: typedef uint16_t doublebyte; typedef uint32_t quadbyte; typedef uint64_t octbyte; + typedef uint128_t hexbyte; typedef boost::uuids::uuid uuid; /** @@ -147,7 +150,7 @@ public: /** * push an unsigned __int128 onto the end of the stream. The byte order is whatever the native byte order is. */ - EXPORT ByteStream& operator<<(const unsigned __int128 o); + EXPORT ByteStream& operator<<(const uint128_t o); /** * push a float onto the end of the stream. The byte order is * whatever the native byte order is. @@ -216,7 +219,7 @@ public: /** * extract an unsigned __int128 from the front of the stream. The byte order is whatever the native byte order is. */ - EXPORT ByteStream& operator>>(unsigned __int128& o); + EXPORT ByteStream& operator>>(uint128_t& o); /** * extract a float from the front of the stream. The byte * order is whatever the native byte order is. @@ -291,7 +294,7 @@ public: /** * Peek at an unsigned __int128 from the front of the stream. The byte order is whatever the native byte order is. */ - EXPORT void peek(unsigned __int128& o) const; + EXPORT void peek(uint128_t& o) const; /** * Peek at a float from the front of the stream. The byte order * is whatever the native byte order is. diff --git a/utils/rowgroup/CMakeLists.txt b/utils/rowgroup/CMakeLists.txt index edda8e943..86c5fa9db 100644 --- a/utils/rowgroup/CMakeLists.txt +++ b/utils/rowgroup/CMakeLists.txt @@ -16,3 +16,8 @@ target_link_libraries(rowgroup ${NETSNMP_LIBRARIES} funcexp) install(TARGETS rowgroup DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) +if (WITH_ROWGROUP_UT) + add_executable(rowgroup_tests rowgroup-tests.cpp) + target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) + install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) +endif() diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index 0176e87b5..e6baec5fc 100755 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -225,7 +225,12 @@ inline string getStringNullValue() inline uint64_t getBinaryNullValue() { return joblist::BINARYNULL; -} +} + +inline uint64_t getBinaryEmptyValue() +{ + return joblist::BINARYEMPTYROW; +} } @@ -1173,10 +1178,11 @@ void RowAggregation::makeAggFieldsNull(Row& row) { // WIP This is only 1st part of the value uint64_t nullValue = getBinaryNullValue(); + uint64_t emptyValue = getBinaryEmptyValue(); uint32_t offset = row.getOffset(colOut); row.setBinaryField_offset(&nullValue, sizeof(nullValue), offset); - row.setBinaryField_offset(&nullValue, sizeof(nullValue), + row.setBinaryField_offset(&emptyValue, sizeof(nullValue), offset+sizeof(nullValue)); } break; diff --git a/utils/rowgroup/rowgroup-tests.cpp b/utils/rowgroup/rowgroup-tests.cpp new file mode 100644 index 000000000..7ef43a122 --- /dev/null +++ b/utils/rowgroup/rowgroup-tests.cpp @@ -0,0 +1,110 @@ +#include // googletest header file +#include "rowgroup.h" +#include "columnwidth.h" +#include "joblisttypes.h" + +#define WIDE_DEC_PRECISION 38 +#define INITIAL_ROW_OFFSET 2 + +using int128_t = __int128; +using uint128_t = unsigned __int128; + +class RowTest : public ::testing::Test { + protected: + void SetUp() override { + uint8_t width = utils::widthByPrecision(WIDE_DEC_PRECISION); + uint32_t oid =3001; + + std::vector offsets, roids, tkeys, cscale, cprecision; + std::vector types; + offsets.push_back(INITIAL_ROW_OFFSET); + offsets.push_back(width+INITIAL_ROW_OFFSET); + offsets.push_back(width*2+INITIAL_ROW_OFFSET); + roids.push_back(oid); roids.push_back(oid+1); + tkeys.push_back(1); tkeys.push_back(1); + types.push_back(execplan::CalpontSystemCatalog::DECIMAL); + types.push_back(execplan::CalpontSystemCatalog::UDECIMAL); + cscale.push_back(0); cscale.push_back(0); + cprecision.push_back(WIDE_DEC_PRECISION); + cprecision.push_back(WIDE_DEC_PRECISION); + rowgroup::RowGroup inRG(roids.size(), //column count + offsets, //oldOffset + roids, // column oids + tkeys, //keys + types, // types + cscale, //scale + cprecision, // precision + 20, // sTableThreshold + false //useStringTable + ); + rg = inRG; + rgD.reinit(rg); + rg.setData(&rgD); + + rg.initRow(&r); + rowSize = r.getSize(); + rg.getRow(0, &r); + + std::vector sValueVector; + std::vector uValueVector; + int128_t nullValue = 0; + uint64_t* uint128_pod = reinterpret_cast(&nullValue); + uint128_pod[0] = joblist::BINARYEMPTYROW; + uint128_pod[1] = joblist::BINARYNULL; + + sValueVector.push_back(nullValue); + sValueVector.push_back(-42); + sValueVector.push_back(-42*0xFFFFFFFFFFFFFFFFLL); + sValueVector.push_back(0); + sValueVector.push_back(nullValue-1); + + uValueVector.push_back(nullValue); + uValueVector.push_back(42); + uValueVector.push_back(42*0xFFFFFFFFFFFFFFFFLL); + uValueVector.push_back(0); + uValueVector.push_back(nullValue); + uValueVector.push_back(nullValue-1); + + for(size_t i = 0; i < sValueVector.size(); i++) + { + r.setBinaryField_offset(&sValueVector[i], + sizeof(sValueVector[0]), INITIAL_ROW_OFFSET); + r.setBinaryField_offset(&uValueVector[i], + sizeof(uValueVector[0]), INITIAL_ROW_OFFSET+width); + r.nextRow(rowSize); + } + rowCount = sValueVector.size(); + } + // void TearDown() override {} + + rowgroup::Row r; + rowgroup::RowGroup rg; + rowgroup::RGData rgD; + uint32_t rowSize; + size_t rowCount; +}; + +TEST_F(RowTest, NonNULLValuesCheck) { + rg.getRow(1, &r); + for (size_t i = 0; i <= rg.getRowCount(); i++) + { + EXPECT_FALSE(r.isNullValue(0)); + EXPECT_FALSE(r.isNullValue(1)); + r.nextRow(rowSize); + } +} + +TEST_F(RowTest, NULLValuesCheck) { + rg.getRow(0, &r); + EXPECT_TRUE(r.isNullValue(0)); + EXPECT_TRUE(r.isNullValue(1)); +} + +//Row::isNullValue_offset +//toString +//initToNull +//toCSV +//applyMapping +//setBinaryField remove Field1 and combine setBinaryField +//getBinaryField +//Remove from set/getIntFields Varbinary diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index e3be508ab..b3fb50084 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -848,7 +848,7 @@ void Row::initToNull() case 16 : { uint64_t *dec = reinterpret_cast(&data[offsets[i]]); - dec[0] = joblist::BINARYNULL; + dec[0] = joblist::BINARYEMPTYROW; dec[1] = joblist::BINARYNULL; break; } @@ -888,7 +888,7 @@ void Row::initToNull() } } -template +template inline bool Row::isNullValue_offset(uint32_t offset) const { ostringstream os; @@ -898,69 +898,76 @@ inline bool Row::isNullValue_offset(uint32_t offset) const os << width << endl; throw logic_error(os.str()); } -/* -// WIP how to make if that enables explicit template for two cscTypes? -// Method template resolution could impose some perf degradation -template -inline bool Row::isNullValue_offset(uint32_t offset) const + +// WIP Method template resolution could impose some perf degradation +// Compare perf with switch-case +template<> +inline bool +Row::isNullValue_offset( + uint32_t offset) const { - return (*reinterpret_cast(&data[offset]) == static_cast(joblist::BIGINTNULL)); + const int64_t *intPtr = reinterpret_cast(&data[offset]); + return ((intPtr[0] == static_cast(joblist::BINARYEMPTYROW)) && + (intPtr[1] == static_cast(joblist::BINARYEMPTYROW)) && + (intPtr[2] == static_cast(joblist::BINARYEMPTYROW)) && + (intPtr[3] == static_cast(joblist::BINARYNULL))); } -template -inline bool Row::isNullValue_offset(uint32_t offset) const +template<> +inline bool +Row::isNullValue_offset( + uint32_t offset) const { - return (*reinterpret_cast(&data[offset]) == static_cast(joblist::BIGINTNULL)); + const int64_t *intPtr = reinterpret_cast(&data[offset]); + return ((intPtr[0] == static_cast(joblist::BINARYEMPTYROW)) + && (intPtr[1] == static_cast(joblist::BINARYNULL))); } -template -inline bool Row::isNullValue_offset(uint32_t offset) const +template<> +inline bool +Row::isNullValue_offset( + uint32_t offset) const { - return (*reinterpret_cast(&data[offset]) == static_cast(joblist::BIGINTNULL)); + const int64_t *intPtr = reinterpret_cast(&data[offset]); + return ((intPtr[0] == static_cast(joblist::BINARYEMPTYROW)) + && (intPtr[1] == static_cast(joblist::BINARYNULL))); } -template -inline bool Row::isNullValue_offset(uint32_t offset) const +template<> +inline bool +Row::isNullValue_offset( + uint32_t offset) const { - return (*reinterpret_cast(&data[offset]) == static_cast(joblist::BIGINTNULL)); + return (*reinterpret_cast(&data[offset]) + == static_cast(joblist::BIGINTNULL)); } -template -inline bool Row::isNullValue_offset(uint32_t offset) const +template<> +inline bool +Row::isNullValue_offset( + uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) + == static_cast(joblist::INTNULL)); +} + +template<> +inline bool +Row::isNullValue_offset( + uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) + == static_cast(joblist::SMALLINTNULL)); +} + +template<> +inline bool +Row::isNullValue_offset( + uint32_t offset) const { return (data[offset] == joblist::TINYINTNULL); } -template -inline bool Row::isNullValue_offset(uint32_t offset) const -{ - return (data[offset] == joblist::TINYINTNULL); -} - -template -inline bool Row::isNullValue_offset(uint32_t offset) const -{ - return (*reinterpret_cast(&data[offset]) == static_cast(joblist::SMALLINTNULL)); -} - -template -inline bool Row::isNullValue_offset(uint32_t offset) const -{ - return (*reinterpret_cast(&data[offset]) == static_cast(joblist::SMALLINTNULL)); -} - -template -inline bool Row::isNullValue_offset(uint32_t offset) const -{ - return (*reinterpret_cast(&data[offset]) == static_cast(joblist::INTNULL)); -} - -template -inline bool Row::isNullValue_offset(uint32_t offset) const -{ - return (*reinterpret_cast(&data[offset]) == static_cast(joblist::INTNULL)); -} -*/ bool Row::isNullValue(uint32_t colIndex) const { switch (types[colIndex]) @@ -1044,16 +1051,22 @@ bool Row::isNullValue(uint32_t colIndex) const case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - uint32_t len = getColumnWidth(colIndex); - const int64_t *dec; + // WIP MCOL-641 Allmighty hack. + const uint32_t len = 16; + uint32_t* lenPtr = const_cast(&len); + *lenPtr = getColumnWidth(colIndex); + return isNullValue_offset + (offsets[colIndex]); +// WIP +/* + const int64_t *dec; switch (len) { // MCOL-641 case 16: - dec = reinterpret_cast(&data[offsets[colIndex]]); - return ((dec[0] == static_cast(joblist::BINARYNULL)) - && (dec[1] == static_cast(joblist::BINARYNULL))); + return isNullValue_offset + (offsets[colIndex]); case 1 : return (data[offsets[colIndex]] == joblist::TINYINTNULL); @@ -1067,7 +1080,7 @@ bool Row::isNullValue(uint32_t colIndex) const default: return (*((int64_t*) &data[offsets[colIndex]]) == static_cast(joblist::BIGINTNULL)); } - +*/ break; } @@ -1112,12 +1125,11 @@ bool Row::isNullValue(uint32_t colIndex) const case CalpontSystemCatalog::BINARY: { - // When is null? I dont know. Wait for bitmap null empty implemtenttion ? - // Also still pendig rework discussed use pointers for empty null values - - std::cout << __FILE__<< ":" << __LINE__ << " isNullValue value " << (*((uint64_t*) &data[offsets[colIndex]])) << std::endl; - //return false; - return (*((uint64_t*) &data[offsets[colIndex]]) == joblist::BINARYEMPTYROW); + const uint32_t len = 16; + uint32_t* lenPtr = const_cast(&len); + *lenPtr = getColumnWidth(colIndex); + return isNullValue_offset + (offsets[colIndex]); } default: diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 37f1ad3e4..67668501c 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -28,7 +28,6 @@ #ifndef ROWGROUP_H_ #define ROWGROUP_H_ -#include #include #include #include @@ -72,7 +71,6 @@ using uint128_t = unsigned __int128; namespace rowgroup { -//using cscType = execplan::CalpontSystemCatalog::ColDataType; const int16_t rgCommonSize = 8192; /* @@ -455,7 +453,7 @@ public: uint64_t getNullValue(uint32_t colIndex) const; bool isNullValue(uint32_t colIndex) const; - template + template inline bool isNullValue_offset(uint32_t offset) const; // when NULLs are pulled out via getIntField(), they come out with these values. @@ -820,7 +818,7 @@ inline void Row::setBinaryField1(T* value, uint32_t width, uint32_t colIndex) template inline void Row::setBinaryField_offset(T* value, uint32_t width, uint32_t offset) { - // WIP + // WIP Compare performance. //memcpy(&data[offset], value, width); *reinterpret_cast(&data[offset]) = *value; } From b07db9a8f4ffd0b865526f0c6443d0e479d155f4 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Thu, 20 Feb 2020 17:23:25 -0500 Subject: [PATCH 17/78] MCOL-641 Basic support for updates. --- dbcon/mysql/ha_mcs_impl.cpp | 19 ++- utils/dataconvert/dataconvert.cpp | 1 + writeengine/server/we_dmlcommandproc.cpp | 89 +++++-------- writeengine/wrapper/we_colop.cpp | 162 ++++++++++------------- writeengine/wrapper/writeengine.cpp | 16 +-- 5 files changed, 124 insertions(+), 163 deletions(-) diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 84d49ebe4..e3662ddad 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -1502,6 +1502,17 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c isFromCol = true; columnAssignmentPtr->fFromCol = true; Item_field* setIt = reinterpret_cast (value); + + // Minor optimization: + // do not perform updates of the form "update t1 set a = a;" + if (!strcmp(item->name.str, setIt->name.str) + && item->table_name && setIt->table_name && !strcmp(item->table_name, setIt->table_name) + && item->db_name && setIt->db_name && !strcmp(item->db_name, setIt->db_name)) + { + delete columnAssignmentPtr; + continue; + } + string sectableName = string(setIt->table_name.str); if ( setIt->db_name.str ) //derived table @@ -1609,6 +1620,13 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c ci->stats.fQueryType = updateCP->queryType(); } + // Exit early if there is nothing to update + if (colAssignmentListPtr->empty()) + { + ci->affectedRows = 0; + return 0; + } + //save table oid for commit/rollback to use uint32_t sessionID = tid2sid(thd->thread_id); boost::shared_ptr csc = CalpontSystemCatalog::makeCalpontSystemCatalog(sessionID); @@ -1645,7 +1663,6 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c TableName* qualifiedTablName = new TableName(); - UpdateSqlStatement updateStmt; //@Bug 2753. To make sure the momory is freed. updateStmt.fColAssignmentListPtr = colAssignmentListPtr; diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 5016f478c..7db6e023d 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1382,6 +1382,7 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, // Simplest form of a template will use colType and width as a parameter // There will be lots specializations case CalpontSystemCatalog::DECIMAL: + // TODO MCOL-641 implement decimal38 version of number_int_value if (colType.colWidth == 16) { int128_t bigint; diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index bbef08857..2eaa1895d 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -2631,6 +2631,7 @@ uint8_t WE_DMLCommandProc::rollbackBatchAutoOff(messageqcpp::ByteStream& bs, std //Rollbacked all versioned blocks return rc; } + uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId, @@ -2804,54 +2805,6 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, for (unsigned int j = 0; j < columnsUpdated.size(); j++) { - /* WriteEngine::ColTupleList colTupleList; - //timer.start("lookupsyscat"); - tableColName.column = columnsUpdated[j]->get_Name(); - try - { - oid = systemCatalogPtr->lookupOID(tableColName); - } - catch (std::exception& ex) - { - rc = 1; - ostringstream oss; - oss << "lookupOID got exception " << ex.what() << " with column " << tableColName.schema << "." << tableColName.table << "." << tableColName.column; - err = oss.str(); - } - catch ( ... ) - { - rc = 1; - ostringstream oss; - oss << "lookupOID got unknown exception with column " << tableColName.schema << "." << tableColName.table << "." << tableColName.column; - err = oss.str(); - } - - if (rc != 0) - return rc; - - CalpontSystemCatalog::ColType colType; - try - { - colType = systemCatalogPtr->colType(oid); - } - catch (std::exception& ex) - { - rc = 1; - ostringstream oss; - oss << "colType got exception " << ex.what() << " with column oid " << oid; - err = oss.str(); - } - catch ( ... ) - { - rc = 1; - ostringstream oss; - oss << "colType got unknown exception with column oid " << oid; - err = oss.str(); - } - - if (rc !=0) - return rc; - */ WriteEngine::ColTupleList colTupleList; CalpontSystemCatalog::ColType colType = colTypes[j]; oid = oids[j]; @@ -2874,13 +2827,15 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, rid = relativeRID; convertToRelativeRid (rid, extentNum, blockNum); rowIDLists.push_back(rid); - uint32_t colWidth = (colTypes[j].colWidth > 8 ? 8 : colTypes[j].colWidth); + uint32_t colWidth = ((colTypes[j].colWidth > 8 && + !(colTypes[j].colDataType == CalpontSystemCatalog::DECIMAL || + colTypes[j].colDataType == CalpontSystemCatalog::UDECIMAL)) ? 8 : colTypes[j].colWidth); int rrid = (int) relativeRID / (BYTE_PER_BLOCK / colWidth); // populate stats.blocksChanged if (rrid > preBlkNums[j]) { - preBlkNums[j] = rrid ; - blocksChanged++; + preBlkNums[j] = rrid ; + blocksChanged++; } } @@ -3044,12 +2999,22 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { + // WIP MCOL-641 // decimal width > 8 cannot be stored in an integer if (fetchColColwidths[fetchColPos] > 8) { - value = row.getStringField(fetchColPos); - unsigned i = strlen(value.c_str()); - value = value.substr(0, i); + int128_t* dec; + char buf[41]; + dec = row.getBinaryField(fetchColPos); + dataconvert::DataConvert::decimalToString(dec, + (unsigned)fetchColScales[fetchColPos], buf, + sizeof(buf), fetchColTypes[fetchColPos]); + + value = buf; + + //value = row.getStringField(fetchColPos); + //unsigned i = strlen(value.c_str()); + //value = value.substr(0, i); break; } @@ -3403,12 +3368,22 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { + // WIP MCOL-641 // decimal width > 8 cannot be stored in an integer if (fetchColColwidths[fetchColPos] > 8) { - value = row.getStringField(fetchColPos); - unsigned i = strlen(value.c_str()); - value = value.substr(0, i); + int128_t* dec; + char buf[41]; + dec = row.getBinaryField(fetchColPos); + dataconvert::DataConvert::decimalToString(dec, + (unsigned)fetchColScales[fetchColPos], buf, + sizeof(buf), fetchColTypes[fetchColPos]); + + value = buf; + + //value = row.getStringField(fetchColPos); + //unsigned i = strlen(value.c_str()); + //value = value.substr(0, i); break; } diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index ce091b0ea..de6119752 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -1733,9 +1733,6 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, if (!bDelete) pVal = &((uint64_t*) valArray)[i]; break; - // WIP - //case WriteEngine::WR_INT128: - case WriteEngine::WR_BINARY: // WIP CSCCol type pVal = &((uint128_t*) valArray)[i]; @@ -1838,102 +1835,73 @@ int ColumnOp::writeRows(Column& curCol, uint64_t totalRow, const RIDList& ridLis // This is a awkward way to convert void* and get ith element, I just don't have a good solution for that // How about pVal = valArray? You're always getting the 0'th element here anyways. - switch (curCol.colType) + // TODO MCOL-641 add support here + // This branch does not seem to be called from anywhere + if (!bDelete) { -// case WriteEngine::WR_LONG : pVal = &((long *) valArray)[i]; break; - case WriteEngine::WR_FLOAT : - if (!bDelete) pVal = &((float*) valArray)[0]; - - //pOldVal = &((float *) oldValArray)[i]; - break; - - case WriteEngine::WR_DOUBLE : - if (!bDelete) pVal = &((double*) valArray)[0]; - - //pOldVal = &((double *) oldValArray)[i]; - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_BLOB : - case WriteEngine::WR_TEXT : - case WriteEngine::WR_CHAR : - if (!bDelete) - { + switch (curCol.colType) + { + case WriteEngine::WR_FLOAT : + pVal = &((float*) valArray)[0]; + break; + + case WriteEngine::WR_DOUBLE : + pVal = &((double*) valArray)[0]; + break; + + case WriteEngine::WR_VARBINARY : // treat same as char for now + case WriteEngine::WR_BLOB : + case WriteEngine::WR_TEXT : + case WriteEngine::WR_CHAR : memcpy(charTmpBuf, (char*)valArray, 8); pVal = charTmpBuf; - } - - //pOldVal = (char*)oldValArray + i*8; - break; - -// case WriteEngine::WR_BIT : pVal = &((bool *) valArray)[i]; break; - case WriteEngine::WR_SHORT : - if (!bDelete) pVal = &((short*) valArray)[0]; - - //pOldVal = &((short *) oldValArray)[i]; - break; - - case WriteEngine::WR_BYTE : - if (!bDelete) pVal = &((char*) valArray)[0]; - - //pOldVal = &((char *) oldValArray)[i]; - break; - - case WriteEngine::WR_LONGLONG: - if (!bDelete) pVal = &((long long*) valArray)[0]; - - //pOldVal = &((long long *) oldValArray)[i]; - break; - - case WriteEngine::WR_TOKEN: - if (!bDelete) pVal = &((Token*) valArray)[0]; - - //pOldVal = &((Token *) oldValArray)[i]; - break; - - case WriteEngine::WR_INT : - case WriteEngine::WR_MEDINT : - if (!bDelete) pVal = &((int*) valArray)[0]; - - //pOldVal = &((int *) oldValArray)[i]; - break; - - case WriteEngine::WR_USHORT : - if (!bDelete) pVal = &((uint16_t*) valArray)[0]; - - //pOldVal = &((uint16_t *) oldValArray)[i]; - break; - - case WriteEngine::WR_UBYTE : - if (!bDelete) pVal = &((uint8_t*) valArray)[0]; - - //pOldVal = &((uint8_t *) oldValArray)[i]; - break; - - case WriteEngine::WR_ULONGLONG: - if (!bDelete) pVal = &((uint64_t*) valArray)[0]; - - //pOldVal = &((uint64_t *) oldValArray)[i]; - break; - - case WriteEngine::WR_UINT : - case WriteEngine::WR_UMEDINT : - if (!bDelete) pVal = &((uint32_t*) valArray)[0]; - - //pOldVal = &((uint32_t *) oldValArray)[i]; - break; - - default : - if (!bDelete) pVal = &((int*) valArray)[0]; - - //pOldVal = &((int *) oldValArray)[i]; - break; + break; + + // case WriteEngine::WR_BIT : pVal = &((bool *) valArray)[i]; break; + case WriteEngine::WR_SHORT : + pVal = &((short*) valArray)[0]; + break; + + case WriteEngine::WR_BYTE : + pVal = &((char*) valArray)[0]; + break; + + case WriteEngine::WR_LONGLONG: + pVal = &((long long*) valArray)[0]; + break; + + case WriteEngine::WR_TOKEN: + pVal = &((Token*) valArray)[0]; + break; + + case WriteEngine::WR_INT : + case WriteEngine::WR_MEDINT : + pVal = &((int*) valArray)[0]; + break; + + case WriteEngine::WR_USHORT : + pVal = &((uint16_t*) valArray)[0]; + break; + + case WriteEngine::WR_UBYTE : + pVal = &((uint8_t*) valArray)[0]; + break; + + case WriteEngine::WR_ULONGLONG: + pVal = &((uint64_t*) valArray)[0]; + break; + + case WriteEngine::WR_UINT : + case WriteEngine::WR_UMEDINT : + pVal = &((uint32_t*) valArray)[0]; + break; + + default : + pVal = &((int*) valArray)[0]; + break; + } } - - // This is the stuff to retrieve old value - //memcpy(pOldVal, dataBuf + dataBio, curCol.colWidth); - - if (bDelete) + else { if (curCol.colType != WriteEngine::WR_BINARY) { @@ -2084,6 +2052,10 @@ int ColumnOp::writeRowsValues(Column& curCol, uint64_t totalRow, const RIDList& pVal = &((uint32_t*) valArray)[i]; break; + case WriteEngine::WR_BINARY: + pVal = &((int128_t*) valArray)[i]; + break; + default : pVal = &((int*) valArray)[i]; break; diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 7d295b397..f08943a51 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -4376,9 +4376,6 @@ int WriteEngineWrapper::updateColumnRecs(const TxnID& txnid, const int32_t tableOid) { //Mark extents invalid - //int rc = 0; - //if (colExtentsStruct[0].dataOid < 3000) - //{ vector lbids; vector colDataTypes; ColumnOp* colOp = NULL; @@ -4414,15 +4411,13 @@ int WriteEngineWrapper::updateColumnRecs(const TxnID& txnid, if (lbids.size() > 0) { -// cout << "BRMWrapper::getInstance()->markExtentsInvalid(lbids); " << lbids.size() << " lbids" << endl; rc = BRMWrapper::getInstance()->markExtentsInvalid(lbids, colDataTypes); } - //} - if ( m_opType != DELETE) + if (m_opType != DELETE) m_opType = UPDATE; - rc = writeColumnRecords (txnid, cscColTypeList, colExtentsStruct, colValueList, ridLists, tableOid); + rc = writeColumnRecords(txnid, cscColTypeList, colExtentsStruct, colValueList, ridLists, tableOid); m_opType = NOOP; return rc; } @@ -4438,6 +4433,7 @@ int WriteEngineWrapper::writeColumnRecords(const TxnID& txnid, void* valArray = NULL; Column curCol; ColStruct curColStruct; + CalpontSystemCatalog::ColType curColType; ColTupleList curTupleList; ColStructList::size_type totalColumn; ColStructList::size_type i; @@ -4452,6 +4448,7 @@ int WriteEngineWrapper::writeColumnRecords(const TxnID& txnid, { valArray = NULL; curColStruct = colStructList[i]; + curColType = cscColTypeList[i]; curTupleList = colValueList[i]; ColumnOp* colOp = m_colOp[op(curColStruct.fCompressionType)]; @@ -4568,11 +4565,10 @@ int WriteEngineWrapper::writeColumnRecords(const TxnID& txnid, valArray = (Token*) calloc(sizeof(Token), totalRow); break; - // WIP + // WIP MCOL-641 case WriteEngine::WR_BINARY: - //case WriteEngine::WR_INT128: // Use column width and remove all C-casts from above - valArray = calloc(totalRow, 16); + valArray = calloc(totalRow, curColType.colWidth); break; } From de85e21c38e80ce9c6902ed21799a09f90b6b1ff Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 20 Feb 2020 22:28:38 +0000 Subject: [PATCH 18/78] MCOL-641 This commit cleans up Row methods and adds couple UT for Row. --- dbcon/execplan/simplecolumn.cpp | 4 +- dbcon/mysql/ha_mcs_impl.cpp | 8 +- primitives/primproc/columncommand.cpp | 2 +- primitives/primproc/passthrucommand.cpp | 2 +- utils/funcexp/funcexp.cpp | 4 +- utils/rowgroup/rowaggregation.cpp | 14 +- utils/rowgroup/rowgroup-tests.cpp | 162 +++++++++++++++++++----- utils/rowgroup/rowgroup.cpp | 39 +++--- utils/rowgroup/rowgroup.h | 58 ++++----- 9 files changed, 196 insertions(+), 97 deletions(-) diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index 96466403a..eeb287e53 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -716,7 +716,9 @@ void SimpleColumn::evaluate(Row& row, bool& isNull) case CalpontSystemCatalog::BINARY: { - fResult.strVal = row.getBinaryField(fInputIndex); + // WIP MCOL-641 Binary representation could contain \0. + std::string value(row.getBinaryField(fInputIndex)); + fResult.strVal.swap(value); break; } diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index e3662ddad..ea23b3f44 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -824,14 +824,14 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h if (colType.colDataType == CalpontSystemCatalog::DECIMAL) { dec = row.getBinaryField(s); - dataconvert::DataConvert::decimalToString(dec, + dataconvert::DataConvert::decimalToString(dec, (unsigned)colType.scale, buf, sizeof(buf), colType.colDataType); } else { udec = row.getBinaryField(s); - dataconvert::DataConvert::decimalToString(udec, + dataconvert::DataConvert::decimalToString(udec, (unsigned)colType.scale, buf, sizeof(buf), colType.colDataType); } @@ -861,7 +861,9 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h case CalpontSystemCatalog::BINARY: { Field_varstring* f2 = (Field_varstring*)*f; - f2->store(row.getBinaryField(s).c_str(), 16, f2->charset()); + // WIP MCOL-641 Binary representation could contain \0. + char* binaryString = row.getBinaryField(s); + f2->store(binaryString, colType.colWidth, f2->charset()); if ((*f)->null_ptr) *(*f)->null_ptr &= ~(*f)->null_bit; diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index 4d4d7bf79..0270c8f9a 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -814,7 +814,7 @@ void ColumnCommand::projectResultRG(RowGroup& rg, uint32_t pos) cout << __FILE__<< ":" <<__LINE__ << " ColumnCommand::projectResultRG " << endl; for (i = 0; i < outMsg->NVALS; ++i, msg8 += gapSize) { - r.setBinaryField(msg8, colType.colWidth, offset); + r.setBinaryField_offset(msg8, colType.colWidth, offset); r.nextRow(rowSize); } break; diff --git a/primitives/primproc/passthrucommand.cpp b/primitives/primproc/passthrucommand.cpp index d33d7a6f0..7e362b1a6 100644 --- a/primitives/primproc/passthrucommand.cpp +++ b/primitives/primproc/passthrucommand.cpp @@ -167,7 +167,7 @@ void PassThruCommand::projectIntoRowGroup(RowGroup& rg, uint32_t col) << *(((int64_t*) bpp->values[i]) +1) << endl; // values[i] is 8 bytes so it contains the pointer to bpp->outputMsg set by ColumnCommand::process_OT_BOTH() - r.setBinaryField((uint8_t*)bpp->values[i], 16, offset); + r.setBinaryField_offset((uint8_t*)bpp->values[i], 16, offset); r.nextRow(rowSize); } diff --git a/utils/funcexp/funcexp.cpp b/utils/funcexp/funcexp.cpp index cce824b47..854fbd8ed 100644 --- a/utils/funcexp/funcexp.cpp +++ b/utils/funcexp/funcexp.cpp @@ -477,11 +477,11 @@ void FuncExp::evaluate(rowgroup::Row& row, std::vector& expressi { // WIP make this a separate function w and w/o overflow check if (expression[i]->resultType().colDataType == execplan::CalpontSystemCatalog::DECIMAL) - row.setBinaryField(reinterpret_cast(&val.__v.__s128), + row.setBinaryField_offset(reinterpret_cast(&val.__v.__s128), expression[i]->resultType().colWidth, row.getOffset(expression[i]->outputIndex())); else - row.setBinaryField(reinterpret_cast(&val.__v.__u128), + row.setBinaryField_offset(reinterpret_cast(&val.__v.__u128), expression[i]->resultType().colWidth, row.getOffset(expression[i]->outputIndex())); } diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index e6baec5fc..da216c177 100755 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -1483,19 +1483,19 @@ void RowAggregation::doSum(const Row& rowIn, int64_t colIn, int64_t colOut, int } else { + uint32_t offset = fRow.getOffset(colOut); if (colDataType == execplan::CalpontSystemCatalog::DECIMAL) { int128_t *dec = reinterpret_cast(wideValInPtr); - // WIP MCOL-641 Replace Row::setBinaryField1 if (isNull(fRowGroupOut, fRow, colOut)) { - fRow.setBinaryField1(dec, width, colOut); + fRow.setBinaryField_offset(dec, sizeof(*dec), offset); } else { - int128_t *valOutPtr = fRow.getBinaryField(colOut); + int128_t *valOutPtr = fRow.getBinaryField(valOutPtr, colOut); int128_t sum = *valOutPtr + *dec; - fRow.setBinaryField1(&sum, width, colOut); + fRow.setBinaryField_offset(&sum, sizeof(sum), offset); } } else @@ -1503,13 +1503,13 @@ void RowAggregation::doSum(const Row& rowIn, int64_t colIn, int64_t colOut, int uint128_t *dec = reinterpret_cast(wideValInPtr); if (isNull(fRowGroupOut, fRow, colOut)) { - fRow.setBinaryField1(dec, width, colOut); + fRow.setBinaryField_offset(dec, sizeof(*dec), offset); } else { - uint128_t *valOutPtr = fRow.getBinaryField(colOut); + uint128_t *valOutPtr = fRow.getBinaryField(valOutPtr, colOut); uint128_t sum = *valOutPtr + *dec; - fRow.setBinaryField1(&sum, width, colOut); + fRow.setBinaryField_offset(&sum, sizeof(sum), offset); } } } // end-of isWideDataType block diff --git a/utils/rowgroup/rowgroup-tests.cpp b/utils/rowgroup/rowgroup-tests.cpp index 7ef43a122..f68e5365e 100644 --- a/utils/rowgroup/rowgroup-tests.cpp +++ b/utils/rowgroup/rowgroup-tests.cpp @@ -2,41 +2,65 @@ #include "rowgroup.h" #include "columnwidth.h" #include "joblisttypes.h" +#include "iostream" -#define WIDE_DEC_PRECISION 38 +#define WIDE_DEC_PRECISION 38U #define INITIAL_ROW_OFFSET 2 using int128_t = __int128; using uint128_t = unsigned __int128; +using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; -class RowTest : public ::testing::Test { +class RowDecimalTest : public ::testing::Test { protected: void SetUp() override { - uint8_t width = utils::widthByPrecision(WIDE_DEC_PRECISION); + uint32_t precision = WIDE_DEC_PRECISION; uint32_t oid =3001; - std::vector offsets, roids, tkeys, cscale, cprecision; - std::vector types; - offsets.push_back(INITIAL_ROW_OFFSET); - offsets.push_back(width+INITIAL_ROW_OFFSET); - offsets.push_back(width*2+INITIAL_ROW_OFFSET); - roids.push_back(oid); roids.push_back(oid+1); - tkeys.push_back(1); tkeys.push_back(1); + std::vectortypes; + std::vectorprecisionVec; + std::vector roids, tkeys, cscale; types.push_back(execplan::CalpontSystemCatalog::DECIMAL); types.push_back(execplan::CalpontSystemCatalog::UDECIMAL); - cscale.push_back(0); cscale.push_back(0); - cprecision.push_back(WIDE_DEC_PRECISION); - cprecision.push_back(WIDE_DEC_PRECISION); + for (size_t i=0; i <= 3; i++) { + types.push_back(execplan::CalpontSystemCatalog::DECIMAL); + } + precisionVec.push_back(precision); + precisionVec.push_back(precision); + precisionVec.push_back(18); + precisionVec.push_back(9); + precisionVec.push_back(4); + precisionVec.push_back(2); + std::vectorwidthVec; + uint32_t offset = INITIAL_ROW_OFFSET; + offsets.push_back(offset); + for (size_t i=0; i < types.size(); i++) { + uint8_t width = utils::widthByPrecision(precisionVec[i]); + widthVec.push_back(width); + offset += width; + offsets.push_back(offset); + roids.push_back(oid+i); + tkeys.push_back(i+1); cscale.push_back(0); + } + /*offsets.push_back(INITIAL_ROW_OFFSET); + offsets.push_back(16+INITIAL_ROW_OFFSET); + offsets.push_back(16*2+INITIAL_ROW_OFFSET); + roids.push_back(oid); roids.push_back(oid+1); + tkeys.push_back(1); tkeys.push_back(1); + cscale.push_back(0); cscale.push_back(0);*/ + rowgroup::RowGroup inRG(roids.size(), //column count offsets, //oldOffset roids, // column oids tkeys, //keys types, // types cscale, //scale - cprecision, // precision + precisionVec, // precision 20, // sTableThreshold false //useStringTable ); + + //std::cout << inRG.toString() << std::endl; rg = inRG; rgD.reinit(rg); rg.setData(&rgD); @@ -45,32 +69,61 @@ class RowTest : public ::testing::Test { rowSize = r.getSize(); rg.getRow(0, &r); - std::vector sValueVector; - std::vector uValueVector; int128_t nullValue = 0; + int128_t bigValue = 0; uint64_t* uint128_pod = reinterpret_cast(&nullValue); uint128_pod[0] = joblist::BINARYEMPTYROW; uint128_pod[1] = joblist::BINARYNULL; + bigValue = 42*0xFFFFFFFFFFFFFFFFLL; sValueVector.push_back(nullValue); sValueVector.push_back(-42); - sValueVector.push_back(-42*0xFFFFFFFFFFFFFFFFLL); + sValueVector.push_back(bigValue); sValueVector.push_back(0); sValueVector.push_back(nullValue-1); uValueVector.push_back(nullValue); uValueVector.push_back(42); - uValueVector.push_back(42*0xFFFFFFFFFFFFFFFFLL); + uValueVector.push_back(bigValue); uValueVector.push_back(0); - uValueVector.push_back(nullValue); uValueVector.push_back(nullValue-1); - for(size_t i = 0; i < sValueVector.size(); i++) - { - r.setBinaryField_offset(&sValueVector[i], - sizeof(sValueVector[0]), INITIAL_ROW_OFFSET); - r.setBinaryField_offset(&uValueVector[i], - sizeof(uValueVector[0]), INITIAL_ROW_OFFSET+width); + s8ValueVector.push_back(joblist::TINYINTNULL); + s8ValueVector.push_back(-0x79); + s8ValueVector.push_back(0); + s8ValueVector.push_back(0x81); + s8ValueVector.push_back(joblist::TINYINTNULL-1); + + s16ValueVector.push_back(joblist::SMALLINTNULL); + s16ValueVector.push_back(-0x79); + s16ValueVector.push_back(0); + s16ValueVector.push_back(0x81); + s16ValueVector.push_back(joblist::SMALLINTNULL-1); + + s32ValueVector.push_back(joblist::INTNULL); + s32ValueVector.push_back(-0x79); + s32ValueVector.push_back(0); + s32ValueVector.push_back(0x81); + s32ValueVector.push_back(joblist::INTNULL-1); + + s64ValueVector.push_back(joblist::INTNULL); + s64ValueVector.push_back(-0x79); + s64ValueVector.push_back(0); + s64ValueVector.push_back(0x81); + s64ValueVector.push_back(joblist::INTNULL-1); + + r.initToNull(); + r.nextRow(rowSize); + for(size_t i = 1; i < sValueVector.size(); i++) { + r.setBinaryField_offset(&sValueVector[i], + sizeof(sValueVector[0]), offsets[0]); + r.setBinaryField_offset(&uValueVector[i], + sizeof(uValueVector[0]), offsets[1]); + r.setIntField(s64ValueVector[i], 2); + r.setIntField(s32ValueVector[i], 3); + r.setIntField(s16ValueVector[i], 4); + r.setIntField(s8ValueVector[i], 5); + //std::cout << r.toString() << std::endl; r.nextRow(rowSize); } rowCount = sValueVector.size(); @@ -82,29 +135,70 @@ class RowTest : public ::testing::Test { rowgroup::RGData rgD; uint32_t rowSize; size_t rowCount; + std::vector sValueVector; + std::vector uValueVector; + std::vector s8ValueVector; + std::vector s16ValueVector; + std::vector s32ValueVector; + std::vector s64ValueVector; + std::vector offsets; }; -TEST_F(RowTest, NonNULLValuesCheck) { +TEST_F(RowDecimalTest, NonNULLValuesCheck) { rg.getRow(1, &r); - for (size_t i = 0; i <= rg.getRowCount(); i++) - { + for (size_t i = 1; i <= sValueVector.size(); i++) { EXPECT_FALSE(r.isNullValue(0)); EXPECT_FALSE(r.isNullValue(1)); + EXPECT_FALSE(r.isNullValue(2)); + EXPECT_FALSE(r.isNullValue(3)); + EXPECT_FALSE(r.isNullValue(4)); + EXPECT_FALSE(r.isNullValue(5)); r.nextRow(rowSize); } } -TEST_F(RowTest, NULLValuesCheck) { +TEST_F(RowDecimalTest, initToNullANDisNullValueValueCheck) { rg.getRow(0, &r); EXPECT_TRUE(r.isNullValue(0)); EXPECT_TRUE(r.isNullValue(1)); + EXPECT_TRUE(r.isNullValue(2)); + EXPECT_TRUE(r.isNullValue(3)); + EXPECT_TRUE(r.isNullValue(4)); + EXPECT_TRUE(r.isNullValue(5)); +} + +TEST_F(RowDecimalTest, getBinaryFieldCheck) { + rg.getRow(0, &r); + uint128_t* u128Value; + int128_t* s128Value; +// std::remove_reference::type uType; +// std::remove_reference::type sType; + + for (size_t i = 0; i < sValueVector.size(); i++) { + s128Value = r.getBinaryField(0); + EXPECT_EQ(*s128Value, sValueVector[i]); + u128Value = r.getBinaryField(1); + EXPECT_EQ(*u128Value, uValueVector[i]); + //EXPECT_EQ(r.getIntField(2),s64ValueVector[i]); + //EXPECT_EQ(r.getIntField(3),s32ValueVector[i]); + //EXPECT_EQ(r.getIntField(4),s16ValueVector[i]); + //EXPECT_EQ(r.getIntField(5),s8ValueVector[i]); + r.nextRow(rowSize); + } +} +TEST_F(RowDecimalTest, toStringCheck) { + std::string exemplar1("0: NULL NULL NULL NULL NULL NULL "); + rg.getRow(0, &r); + EXPECT_EQ(r.toString(), exemplar1); + r.nextRow(rowSize); + std::cout << r.toString() << std::endl; + r.nextRow(rowSize); + std::cout << r.toString() << std::endl; + r.nextRow(rowSize); + std::cout << r.toString() << std::endl; } -//Row::isNullValue_offset //toString -//initToNull //toCSV //applyMapping -//setBinaryField remove Field1 and combine setBinaryField -//getBinaryField -//Remove from set/getIntFields Varbinary +//equals diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index b3fb50084..46971ae81 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -43,6 +43,7 @@ using namespace execplan; #include "nullvaluemanip.h" #include "rowgroup.h" #include "dataconvert.h" +#include "columnwidth.h" #include "collation.h" @@ -636,16 +637,17 @@ string Row::toString() const break; // WIP case CalpontSystemCatalog::DECIMAL: - if (precision[i] > 18) + case CalpontSystemCatalog::UDECIMAL: + if (colWidths[i] > MAXLEGACYWIDTH) { char *buf = (char*)alloca(precision[i] + 3); // empty the buffer - dataconvert::DataConvert::toString(getBinaryField(i), + dataconvert::DataConvert::toString(getBinaryField(i), buf, precision[i]+3); os << buf << " "; break; } - // fallback if the precision < 18 + // fallback if the the legacy DECIMAL default: os << getIntField(i) << " "; break; @@ -708,7 +710,8 @@ string Row::toCSV() const break; } case CalpontSystemCatalog::BINARY: - std::cout << __FILE__<< __LINE__ << ":" << "toCSV"<< std::endl; + std::cout << __FILE__<< __LINE__ << ":" << "toCSV"<< std::endl; + default: os << getIntField(i); break; @@ -877,7 +880,13 @@ void Row::initToNull() *((uint64_t*) &data[offsets[i]]) = joblist::UBIGINTNULL; break; case CalpontSystemCatalog::BINARY: - std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; + { + uint64_t *dec = reinterpret_cast(&data[offsets[i]]); + dec[0] = joblist::BINARYEMPTYROW; + dec[1] = joblist::BINARYNULL; + } + break; + default: ostringstream os; os << "Row::initToNull(): got bad column type (" << types[i] << @@ -888,7 +897,7 @@ void Row::initToNull() } } -template +template inline bool Row::isNullValue_offset(uint32_t offset) const { ostringstream os; @@ -1052,21 +1061,13 @@ bool Row::isNullValue(uint32_t colIndex) const case CalpontSystemCatalog::UDECIMAL: { // WIP MCOL-641 Allmighty hack. - const uint32_t len = 16; - uint32_t* lenPtr = const_cast(&len); - *lenPtr = getColumnWidth(colIndex); - return isNullValue_offset - (offsets[colIndex]); - -// WIP -/* - const int64_t *dec; - switch (len) + int32_t width = getColumnWidth(colIndex); + switch (width) { // MCOL-641 case 16: return isNullValue_offset - (offsets[colIndex]); + (offsets[colIndex]); case 1 : return (data[offsets[colIndex]] == joblist::TINYINTNULL); @@ -1080,7 +1081,7 @@ bool Row::isNullValue(uint32_t colIndex) const default: return (*((int64_t*) &data[offsets[colIndex]]) == static_cast(joblist::BIGINTNULL)); } -*/ + break; } @@ -1647,7 +1648,7 @@ void applyMapping(const int* mapping, const Row& in, Row* out) // code precision 2 width convertor else if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::DECIMAL && in.getColumnWidth(i) == 16)) - out->setBinaryField(in.getBinaryField(i), 16, + out->setBinaryField_offset(in.getBinaryField(i), 16, out->getOffset(mapping[i])); else if (in.isUnsigned(i)) out->setUintField(in.getUintField(i), mapping[i]); diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 67668501c..6a2bc7961 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -424,9 +424,8 @@ public: inline uint32_t getStringLength(uint32_t colIndex) const; void setStringField(const std::string& val, uint32_t colIndex); inline void setStringField(const uint8_t*, uint32_t len, uint32_t colIndex); - inline void setBinaryField(const uint8_t* strdata, uint32_t length, uint32_t offset); template - inline void setBinaryField1(T* strdata, uint32_t width, uint32_t colIndex); + inline void setBinaryField(T* strdata, uint32_t width, uint32_t colIndex); template inline void setBinaryField_offset(T* strdata, uint32_t width, uint32_t colIndex); // support VARBINARY @@ -440,9 +439,12 @@ public: inline const uint8_t* getVarBinaryField(uint32_t& len, uint32_t colIndex) const; inline void setVarBinaryField(const uint8_t* val, uint32_t len, uint32_t colIndex); - inline std::string getBinaryField(uint32_t colIndex) const; + //inline std::string getBinaryField(uint32_t colIndex) const; template inline T* getBinaryField(uint32_t colIndex) const; + // To simplify parameter type deduction. + template + inline T* getBinaryField(T* argtype, uint32_t colIndex) const; template inline T* getBinaryField_offset(uint32_t offset) const; @@ -453,7 +455,7 @@ public: uint64_t getNullValue(uint32_t colIndex) const; bool isNullValue(uint32_t colIndex) const; - template + template inline bool isNullValue_offset(uint32_t offset) const; // when NULLs are pulled out via getIntField(), they come out with these values. @@ -803,18 +805,21 @@ inline uint32_t Row::getStringLength(uint32_t colIndex) const return strnlen((char*) &data[offsets[colIndex]], getColumnWidth(colIndex)); } +// WIP Remove this // Check whether memcpy affects perf here -inline void Row::setBinaryField(const uint8_t* strdata, uint32_t length, uint32_t offset) +/*inline void Row::setBinaryField(const uint8_t* strdata, uint32_t length, uint32_t offset) { memcpy(&data[offset], strdata, length); -} +}*/ +// WIP MCOL-641. This method can be applied to uint8_t* buffers. template -inline void Row::setBinaryField1(T* value, uint32_t width, uint32_t colIndex) +inline void Row::setBinaryField(T* value, uint32_t width, uint32_t colIndex) { - memcpy(&data[offsets[colIndex]], value, width); + memcpy(&data[offsets[colIndex]], value, width); } +// WIP MCOL-641. This method !cannot! be applied to uint8_t* buffers. template inline void Row::setBinaryField_offset(T* value, uint32_t width, uint32_t offset) { @@ -856,16 +861,24 @@ inline std::string Row::getStringField(uint32_t colIndex) const strnlen((char*) &data[offsets[colIndex]], getColumnWidth(colIndex))); } -inline std::string Row::getBinaryField(uint32_t colIndex) const +/*inline std::string Row::getBinaryField(uint32_t colIndex) const { return std::string((char*) &data[offsets[colIndex]], getColumnWidth(colIndex)); -} +}*/ // WIP MCOL-641 template inline T* Row::getBinaryField(uint32_t colIndex) const { - return reinterpret_cast(&data[offsets[colIndex]]); + //return reinterpret_cast(&data[offsets[colIndex]]); + return getBinaryField_offset(offsets[colIndex]); +} + +template +inline T* Row::getBinaryField(T* argtype, uint32_t colIndex) const +{ + //return reinterpret_cast(&data[offsets[colIndex]]); + return getBinaryField_offset(offsets[colIndex]); } template @@ -874,7 +887,6 @@ inline T* Row::getBinaryField_offset(uint32_t offset) const return reinterpret_cast(&data[offset]); } - inline std::string Row::getVarBinaryStringField(uint32_t colIndex) const { if (inStringTable(colIndex)) @@ -994,12 +1006,7 @@ inline void Row::setUintField_offset(uint64_t val, uint32_t offset) case 8: *((uint64_t*) &data[offset]) = val; break; - /* This doesn't look like appropriate place - case 16: - std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; - *((uint64_t*) &data[offset]) = val; - break; - */ + default: idbassert(0); throw std::logic_error("Row::setUintField called on a non-uint32_t field"); @@ -1037,10 +1044,7 @@ inline void Row::setUintField(uint64_t val, uint32_t colIndex) case 8: *((uint64_t*) &data[offsets[colIndex]]) = val; break; - case 16: - std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; - *((uint64_t*) &data[offsets[colIndex]]) = val; - break; + default: idbassert(0); throw std::logic_error("Row::setUintField called on a non-uint32_t field"); @@ -1066,9 +1070,7 @@ inline void Row::setUintField(uint64_t val, uint32_t colIndex) case 8: *((uint64_t*) &data[offsets[colIndex]]) = val; break; - case 16: - std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; - *((uint64_t*) &data[offsets[colIndex]]) = val; + default: idbassert(0); throw std::logic_error("Row::setUintField: bad length"); @@ -1095,8 +1097,7 @@ inline void Row::setIntField(int64_t val, uint32_t colIndex) case 8: *((int64_t*) &data[offsets[colIndex]]) = val; break; - case 16: - std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; + default: idbassert(0); throw std::logic_error("Row::setIntField: bad length"); @@ -1122,8 +1123,7 @@ inline void Row::setIntField(int64_t val, uint32_t colIndex) case 8: *((int64_t*) &data[offsets[colIndex]]) = val; break; - case 16: - std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; + default: idbassert(0); throw std::logic_error("Row::setIntField: bad length"); From c23ead2703d1fbd959b8540a710fef0575c07997 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Fri, 21 Feb 2020 17:47:04 +0000 Subject: [PATCH 19/78] MCOL-641 This commit changes NULL and EMPTY values. It also contains the refactored DataConvert::decimalToString(). Row::toString UT is finished. --- primitives/linux-port/column.cpp | 6 +- primitives/primproc/columncommand.cpp | 2 - utils/common/columnwidth.h | 12 +++ utils/dataconvert/dataconvert.cpp | 116 +++++++++++++++++++++++++- utils/dataconvert/dataconvert.h | 13 ++- utils/rowgroup/rowgroup-tests.cpp | 68 +++++++++------ utils/rowgroup/rowgroup.cpp | 24 +++--- 7 files changed, 194 insertions(+), 47 deletions(-) diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index e246e8e95..463764cd0 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -414,15 +414,15 @@ template<> inline bool isNullVal<16>(uint8_t type, const uint8_t* ival) // For BINARY { const uint64_t* val = reinterpret_cast(ival); - return ((val[0] == joblist::BINARYEMPTYROW) && (val[1] == joblist::BINARYNULL)); + return ((val[0] == joblist::BINARYNULL) && (val[1] == joblist::BINARYEMPTYROW)); } template<> inline bool isNullVal<32>(uint8_t type, const uint8_t* ival) // For BINARY { const uint64_t* val = reinterpret_cast(ival); - return ((val[0] == joblist::BINARYEMPTYROW) && (val[1] == joblist::BINARYEMPTYROW) - && (val[2] == joblist::BINARYEMPTYROW) && (val[3] == joblist::BINARYNULL)); + return ((val[0] == joblist::BINARYNULL) && (val[1] == joblist::BINARYEMPTYROW) + && (val[2] == joblist::BINARYEMPTYROW) && (val[3] == joblist::BINARYEMPTYROW)); } template<> diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index 0270c8f9a..d2145807c 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -1067,8 +1067,6 @@ void ColumnCommand::getEmptyRowValue(const CSCDataType dataType, ptr[1] = joblist::BINARYEMPTYROW; } - - void ColumnCommand::getLBIDList(uint32_t loopCount, vector* lbids) { int64_t firstLBID = lbid, lastLBID = firstLBID + (loopCount * colType.colWidth) - 1, i; diff --git a/utils/common/columnwidth.h b/utils/common/columnwidth.h index 8e36f5356..b16086494 100644 --- a/utils/common/columnwidth.h +++ b/utils/common/columnwidth.h @@ -69,6 +69,18 @@ namespace utils return 16; } } + inline uint8_t precisionByWidth(unsigned w) + { + switch(w) + { + case 16: + return 38; + // In case we will support decimals that spans 32 bytes. + default: + return 65; + } + } + } diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 7db6e023d..619ac6c77 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -28,6 +28,7 @@ #include #include #include +#include using namespace std; #include #include @@ -1164,7 +1165,110 @@ bool stringToTimestampStruct(const string& data, TimeStamp& timeStamp, const str } // WIP MCOL-641 -// Check for overflows with buflen +// Second DT is for POD representation. +template +size_t DataConvert::writeIntPart(T* dec, char* p, + const uint16_t buflen, + const uint8_t scale) +{ + T intPart = *dec; + if (scale) + { + // optimize this + for (size_t i = 0; i < scale; i++) + intPart /= 10; + } + + // optimize for less then uint64_t values + uint64_t div = 10000000000000000000ULL; + T high = intPart; + T low; + low = high % div; + high /= div; + T mid; + mid = high % div; + high /= div; + + // pod[0] is high 8 byte, pod[1] is low + uint64_t* high_pod = reinterpret_cast(&high); + uint64_t* mid_pod = reinterpret_cast(&mid); + uint64_t* low_pod = reinterpret_cast(&low); + char* original_p = p; + int written = 0; + + // WIP replace snprintf with streams + if (high_pod[0] != 0) + { + written = sprintf(p, "%lu", high_pod[0]); + p += written; + written = sprintf(p, "%019lu", mid_pod[0]); + p += written; + sprintf(p, "%019lu", low_pod[0]); + } + else if (mid_pod[0] != 0) + { + written = sprintf(p, "%lu", mid_pod[0]); + p += written; + written = sprintf(p, "%019lu", low_pod[0]); + p += written; + } + else + { + written = sprintf(p, "%lu", low_pod[0]); + p += written; + } + + if (buflen <= p-original_p) + { + throw QueryDataExcept("toString() char buffer overflow.", formatErr); + } + return p-original_p; +} + +template +size_t DataConvert::writeFractionalPart(T* dec, char* p, + const uint16_t buflen, + const uint8_t scale) +{ + T scaleDivisor = 10; + for (size_t i = 1; i < scale; i++) + scaleDivisor *= 10; + + T fractionalPart = *dec % scaleDivisor; + return writeIntPart(&fractionalPart, p, buflen, 0); +} + +// WIP MCOL-641 +// Limit this to Decimal only +// Replace decimalToString with this one +template +void DataConvert::toString1(T* dec, char *p, const uint16_t buflen, + const uint8_t scale) +{ + if (*dec < static_cast(0)) + { + *p++ = '-'; + *dec *= -1; + } + + char* original_p = p; + size_t written = 0; + written = writeIntPart(dec, p, buflen, scale); + p += written; + + // WIP To be finished for 0.042 + if (scale) + { + *p++ = '.'; + p += writeFractionalPart(dec, p, p-original_p, scale); + } + + if (buflen <= p-original_p) + { + throw QueryDataExcept("toString() char buffer overflow.", formatErr); + } +} + template void DataConvert::toString(T* dec, char *p, size_t buflen) { @@ -1206,6 +1310,7 @@ void DataConvert::toString(T* dec, char *p, size_t buflen) if (buflen <= p-original_p) std::cout << "DataConvert::toString char buffer overflow" << std::endl; } + // WIP MCOL-641 // Template this // result must be calloc-ed @@ -1239,13 +1344,20 @@ void atoi128(const std::string& arg, uint128_t& res) } // WIP MCOL-641 +// Doesn't work for -0.042 template void DataConvert::decimalToString(T* valuePtr, uint8_t scale, char* buf, unsigned int buflen, - cscDataType colDataType) + cscDataType colDataType) // We don't need the last one { + if (*valuePtr < static_cast(0)) + { + *buf++ = '-'; + *valuePtr *= -1; + } + toString(valuePtr, buf, buflen); // Biggest ColumnStore supports is DECIMAL(38,x), or 38 total digits+dp+sign for column diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 4bd30ca33..2301d75ec 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -1034,8 +1034,19 @@ public: template EXPORT static void decimalToString(T* value, uint8_t scale, char* buf, unsigned int buflen, cscDataType colDataType); + template + static void toString(T* dec, char *p, size_t buflen); template - EXPORT static void toString(T* dec, char *p, size_t buflen); + EXPORT static void toString1(T* dec, char* p, const uint16_t buflen, + const uint8_t scale = 0); + template + static size_t writeIntPart(T* dec, char* p, const uint16_t buflen, + const uint8_t scale); + template + static size_t writeFractionalPart(T* dec, char* p, + const uint16_t buflen, const uint8_t scale); + + static inline void int128Max(int128_t& i) { diff --git a/utils/rowgroup/rowgroup-tests.cpp b/utils/rowgroup/rowgroup-tests.cpp index f68e5365e..e63f0b2db 100644 --- a/utils/rowgroup/rowgroup-tests.cpp +++ b/utils/rowgroup/rowgroup-tests.cpp @@ -1,8 +1,10 @@ #include // googletest header file +#include + #include "rowgroup.h" #include "columnwidth.h" #include "joblisttypes.h" -#include "iostream" +#include "dataconvert.h" #define WIDE_DEC_PRECISION 38U #define INITIAL_ROW_OFFSET 2 @@ -60,7 +62,6 @@ class RowDecimalTest : public ::testing::Test { false //useStringTable ); - //std::cout << inRG.toString() << std::endl; rg = inRG; rgD.reinit(rg); rg.setData(&rgD); @@ -74,9 +75,14 @@ class RowDecimalTest : public ::testing::Test { uint64_t* uint128_pod = reinterpret_cast(&nullValue); uint128_pod[0] = joblist::BINARYEMPTYROW; uint128_pod[1] = joblist::BINARYNULL; - bigValue = 42*0xFFFFFFFFFFFFFFFFLL; - - sValueVector.push_back(nullValue); + bigValue = -static_cast(0xFFFFFFFF)*0xFFFFFFFFFFFFFFFF; + //char buf[utils::precisionByWidth(16)+3]; + //memset(&buf[0], 0, sizeof(buf)); + //int scale1 = 3; + //dataconvert::DataConvert::decimalToString(&bigValue, scale1, buf, + // utils::precisionByWidth(sizeof(bigValue))+3,types[0]); + //std::cout << buf << std::endl; + sValueVector.push_back(nullValue-2); sValueVector.push_back(-42); sValueVector.push_back(bigValue); sValueVector.push_back(0); @@ -106,15 +112,15 @@ class RowDecimalTest : public ::testing::Test { s32ValueVector.push_back(0x81); s32ValueVector.push_back(joblist::INTNULL-1); - s64ValueVector.push_back(joblist::INTNULL); + s64ValueVector.push_back(joblist::BIGINTNULL); s64ValueVector.push_back(-0x79); s64ValueVector.push_back(0); s64ValueVector.push_back(0x81); - s64ValueVector.push_back(joblist::INTNULL-1); + s64ValueVector.push_back(joblist::BIGINTNULL-1); - r.initToNull(); - r.nextRow(rowSize); - for(size_t i = 1; i < sValueVector.size(); i++) { + //r.initToNull(); + //r.nextRow(rowSize); + for(size_t i = 0; i < sValueVector.size(); i++) { r.setBinaryField_offset(&sValueVector[i], sizeof(sValueVector[0]), offsets[0]); r.setBinaryField_offset(&uValueVector[i], @@ -123,7 +129,6 @@ class RowDecimalTest : public ::testing::Test { r.setIntField(s32ValueVector[i], 3); r.setIntField(s16ValueVector[i], 4); r.setIntField(s8ValueVector[i], 5); - //std::cout << r.toString() << std::endl; r.nextRow(rowSize); } rowCount = sValueVector.size(); @@ -159,6 +164,7 @@ TEST_F(RowDecimalTest, NonNULLValuesCheck) { TEST_F(RowDecimalTest, initToNullANDisNullValueValueCheck) { rg.getRow(0, &r); + r.initToNull(); EXPECT_TRUE(r.isNullValue(0)); EXPECT_TRUE(r.isNullValue(1)); EXPECT_TRUE(r.isNullValue(2)); @@ -176,29 +182,37 @@ TEST_F(RowDecimalTest, getBinaryFieldCheck) { for (size_t i = 0; i < sValueVector.size(); i++) { s128Value = r.getBinaryField(0); - EXPECT_EQ(*s128Value, sValueVector[i]); + EXPECT_EQ(sValueVector[i], *s128Value); u128Value = r.getBinaryField(1); - EXPECT_EQ(*u128Value, uValueVector[i]); - //EXPECT_EQ(r.getIntField(2),s64ValueVector[i]); - //EXPECT_EQ(r.getIntField(3),s32ValueVector[i]); + EXPECT_EQ(uValueVector[i], *u128Value); + //EXPECT_EQ(s64ValueVector[i], r.getIntField(2)); + //EXPECT_EQ(s32ValueVector[i],r.getIntField(3)); //EXPECT_EQ(r.getIntField(4),s16ValueVector[i]); //EXPECT_EQ(r.getIntField(5),s8ValueVector[i]); r.nextRow(rowSize); } } + +// TBD Need to add asserts when toString will be finished TEST_F(RowDecimalTest, toStringCheck) { - std::string exemplar1("0: NULL NULL NULL NULL NULL NULL "); + std::vector exemplarVector; + exemplarVector.push_back(std::string("0: NULL NULL NULL NULL NULL NULL ")); + exemplarVector.push_back(std::string("0: -42 42 -121 -121 -121 -121 ")); + exemplarVector.push_back(std::string("0: -79228162495817593515539431425 -79228162495817593515539431425 0 0 0 0 ")); + exemplarVector.push_back(std::string("0: 0 0 129 129 129 -127 ")); + exemplarVector.push_back(std::string("0: -18446744073709551618 -18446744073709551618 9223372036854775807 2147483647 32767 127 ")); + rg.getRow(0, &r); - EXPECT_EQ(r.toString(), exemplar1); - r.nextRow(rowSize); - std::cout << r.toString() << std::endl; - r.nextRow(rowSize); - std::cout << r.toString() << std::endl; - r.nextRow(rowSize); - std::cout << r.toString() << std::endl; + r.initToNull(); + for (auto &el: exemplarVector) { + EXPECT_EQ(el, r.toString()); + r.nextRow(rowSize); + } } -//toString -//toCSV -//applyMapping -//equals +TEST_F(RowDecimalTest, toCSVCheck) { +} +TEST_F(RowDecimalTest, applyMappingCheck) { +} +TEST_F(RowDecimalTest, equalsCheck) { +} diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 46971ae81..bd84117a2 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -642,8 +642,8 @@ string Row::toString() const { char *buf = (char*)alloca(precision[i] + 3); // empty the buffer - dataconvert::DataConvert::toString(getBinaryField(i), - buf, precision[i]+3); + dataconvert::DataConvert::toString(getBinaryField(i), + buf, precision[i]+3); //WIP scale[i] os << buf << " "; break; } @@ -851,8 +851,8 @@ void Row::initToNull() case 16 : { uint64_t *dec = reinterpret_cast(&data[offsets[i]]); - dec[0] = joblist::BINARYEMPTYROW; - dec[1] = joblist::BINARYNULL; + dec[0] = joblist::BINARYNULL; + dec[1] = joblist::BINARYEMPTYROW; break; } default: @@ -882,8 +882,8 @@ void Row::initToNull() case CalpontSystemCatalog::BINARY: { uint64_t *dec = reinterpret_cast(&data[offsets[i]]); - dec[0] = joblist::BINARYEMPTYROW; - dec[1] = joblist::BINARYNULL; + dec[0] = joblist::BINARYNULL; + dec[1] = joblist::BINARYEMPTYROW; } break; @@ -916,10 +916,10 @@ Row::isNullValue_offset( uint32_t offset) const { const int64_t *intPtr = reinterpret_cast(&data[offset]); - return ((intPtr[0] == static_cast(joblist::BINARYEMPTYROW)) && + return ((intPtr[0] == static_cast(joblist::BINARYNULL)) && (intPtr[1] == static_cast(joblist::BINARYEMPTYROW)) && (intPtr[2] == static_cast(joblist::BINARYEMPTYROW)) && - (intPtr[3] == static_cast(joblist::BINARYNULL))); + (intPtr[3] == static_cast(joblist::BINARYEMPTYROW))); } template<> @@ -928,8 +928,8 @@ Row::isNullValue_offset( uint32_t offset) const { const int64_t *intPtr = reinterpret_cast(&data[offset]); - return ((intPtr[0] == static_cast(joblist::BINARYEMPTYROW)) - && (intPtr[1] == static_cast(joblist::BINARYNULL))); + return ((intPtr[0] == static_cast(joblist::BINARYNULL)) + && (intPtr[1] == static_cast(joblist::BINARYEMPTYROW))); } template<> @@ -938,8 +938,8 @@ Row::isNullValue_offset( uint32_t offset) const { const int64_t *intPtr = reinterpret_cast(&data[offset]); - return ((intPtr[0] == static_cast(joblist::BINARYEMPTYROW)) - && (intPtr[1] == static_cast(joblist::BINARYNULL))); + return ((intPtr[0] == static_cast(joblist::BINARYNULL)) + && (intPtr[1] == static_cast(joblist::BINARYEMPTYROW))); } template<> From b29d0c9daa43a3cc4c62e5963f08179f3b5ef33b Mon Sep 17 00:00:00 2001 From: drrtuy Date: Mon, 24 Feb 2020 17:22:52 +0300 Subject: [PATCH 20/78] MCOL-641 Changed the hint to search for GTest headers. This commit introduces DataConvert UTs. DataConvert::decimalToString now can negative values. Next version for Row::toString(), applyMapping UT checks. Row:equals() is now wide-DECIMAL aware. --- cmake/FindGTest.cmake | 3 +- dbcon/execplan/calpontsystemcatalog.h | 6 ++ utils/dataconvert/CMakeLists.txt | 6 ++ utils/dataconvert/dataconvert.cpp | 30 ++++----- utils/rowgroup/rowgroup-tests.cpp | 27 ++++++-- utils/rowgroup/rowgroup.cpp | 16 ++--- utils/rowgroup/rowgroup.h | 93 +++++++++++++++++++++++++-- 7 files changed, 142 insertions(+), 39 deletions(-) diff --git a/cmake/FindGTest.cmake b/cmake/FindGTest.cmake index 7b1fb3957..5860ca415 100644 --- a/cmake/FindGTest.cmake +++ b/cmake/FindGTest.cmake @@ -17,10 +17,9 @@ find_library(PTHREAD_LIBRARY HINTS ${GTEST_ROOT_DIR}/lib ) - find_path(GTEST_INCLUDE_DIR NAMES gtest.h - HINTS ${GTEST_ROOT_DIR}/include + HINTS ${GTEST_ROOT_DIR}/include/gtest ) include(FindPackageHandleStandardArgs) diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index 0f7b66aa2..4450ce429 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -1004,6 +1004,12 @@ inline bool isNumeric(const execplan::CalpontSystemCatalog::ColDataType type) } } +inline bool isDecimal(const execplan::CalpontSystemCatalog::ColDataType type) +{ + return (type == execplan::CalpontSystemCatalog::DECIMAL || + type == execplan::CalpontSystemCatalog::UDECIMAL); +} + /** convenience function to determine if column type is an * unsigned type */ diff --git a/utils/dataconvert/CMakeLists.txt b/utils/dataconvert/CMakeLists.txt index 3e37b3c0e..168efdf80 100644 --- a/utils/dataconvert/CMakeLists.txt +++ b/utils/dataconvert/CMakeLists.txt @@ -11,3 +11,9 @@ add_library(dataconvert SHARED ${dataconvert_LIB_SRCS}) add_dependencies(dataconvert loggingcpp) install(TARGETS dataconvert DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) + +if (WITH_DATACONVERT_UT) + add_executable(dataconvert_tests dataconvert-tests.cpp) + target_link_libraries(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) + install(TARGETS dataconvert_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) +endif() diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 619ac6c77..6838cebbe 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1220,7 +1220,7 @@ size_t DataConvert::writeIntPart(T* dec, char* p, if (buflen <= p-original_p) { - throw QueryDataExcept("toString() char buffer overflow.", formatErr); + throw QueryDataExcept("writeIntPart() char buffer overflow.", formatErr); } return p-original_p; } @@ -1352,31 +1352,27 @@ void DataConvert::decimalToString(T* valuePtr, unsigned int buflen, cscDataType colDataType) // We don't need the last one { - if (*valuePtr < static_cast(0)) + T value = *valuePtr; + char* ptr = &buf[0]; + size_t l1 = buflen; + if (value < static_cast(0)) { - *buf++ = '-'; - *valuePtr *= -1; + *ptr++ = '-'; + value *= -1; + idbassert(l1 >= 2); + l1--; } - toString(valuePtr, buf, buflen); + //we want to move the last scale chars right by one spot to insert the dp + //we want to move the trailing null as well, so it's really scale+1 chars + + toString(&value, ptr, buflen); // Biggest ColumnStore supports is DECIMAL(38,x), or 38 total digits+dp+sign for column if (scale == 0) return; - //we want to move the last scale chars right by one spot to insert the dp - //we want to move the trailing null as well, so it's really scale+1 chars - size_t l1 = strlen(buf); - char* ptr = &buf[0]; - - if (*valuePtr < 0) - { - ptr++; - idbassert(l1 >= 2); - l1--; - } - //need to make sure we have enough leading zeros for this to work... //at this point scale is always > 0 size_t l2 = 1; diff --git a/utils/rowgroup/rowgroup-tests.cpp b/utils/rowgroup/rowgroup-tests.cpp index e63f0b2db..7b9708526 100644 --- a/utils/rowgroup/rowgroup-tests.cpp +++ b/utils/rowgroup/rowgroup-tests.cpp @@ -67,14 +67,15 @@ class RowDecimalTest : public ::testing::Test { rg.setData(&rgD); rg.initRow(&r); + rg.initRow(&rOutMappingCheck); rowSize = r.getSize(); rg.getRow(0, &r); int128_t nullValue = 0; int128_t bigValue = 0; uint64_t* uint128_pod = reinterpret_cast(&nullValue); - uint128_pod[0] = joblist::BINARYEMPTYROW; - uint128_pod[1] = joblist::BINARYNULL; + uint128_pod[0] = joblist::BINARYNULL; + uint128_pod[1] = joblist::BINARYEMPTYROW; bigValue = -static_cast(0xFFFFFFFF)*0xFFFFFFFFFFFFFFFF; //char buf[utils::precisionByWidth(16)+3]; //memset(&buf[0], 0, sizeof(buf)); @@ -82,7 +83,7 @@ class RowDecimalTest : public ::testing::Test { //dataconvert::DataConvert::decimalToString(&bigValue, scale1, buf, // utils::precisionByWidth(sizeof(bigValue))+3,types[0]); //std::cout << buf << std::endl; - sValueVector.push_back(nullValue-2); + sValueVector.push_back(nullValue); sValueVector.push_back(-42); sValueVector.push_back(bigValue); sValueVector.push_back(0); @@ -136,6 +137,7 @@ class RowDecimalTest : public ::testing::Test { // void TearDown() override {} rowgroup::Row r; + rowgroup::Row rOutMappingCheck; rowgroup::RowGroup rg; rowgroup::RGData rgD; uint32_t rowSize; @@ -200,7 +202,7 @@ TEST_F(RowDecimalTest, toStringCheck) { exemplarVector.push_back(std::string("0: -42 42 -121 -121 -121 -121 ")); exemplarVector.push_back(std::string("0: -79228162495817593515539431425 -79228162495817593515539431425 0 0 0 0 ")); exemplarVector.push_back(std::string("0: 0 0 129 129 129 -127 ")); - exemplarVector.push_back(std::string("0: -18446744073709551618 -18446744073709551618 9223372036854775807 2147483647 32767 127 ")); + exemplarVector.push_back(std::string("0: -3 -3 9223372036854775807 2147483647 32767 127 ")); rg.getRow(0, &r); r.initToNull(); @@ -212,7 +214,22 @@ TEST_F(RowDecimalTest, toStringCheck) { TEST_F(RowDecimalTest, toCSVCheck) { } + TEST_F(RowDecimalTest, applyMappingCheck) { + int mapping[] = {0, 1, -1, -1, -1, -1}; + rg.getRow(1, &r); + rg.getRow(2, &rOutMappingCheck); + int128_t* s128Value = rOutMappingCheck.getBinaryField(0); + uint128_t* u128Value = rOutMappingCheck.getBinaryField(1); + EXPECT_NE(sValueVector[1], *s128Value); + EXPECT_NE(uValueVector[1], *u128Value); + applyMapping(mapping, r, &rOutMappingCheck); + s128Value = rOutMappingCheck.getBinaryField(0); + EXPECT_EQ(sValueVector[1], *s128Value); + u128Value = rOutMappingCheck.getBinaryField(1); + EXPECT_EQ(uValueVector[1], *u128Value); } -TEST_F(RowDecimalTest, equalsCheck) { + +// WIP +TEST_F(RowDecimalTest, RowEqualsCheck) { } diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index bd84117a2..448c8ad5c 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -638,12 +638,12 @@ string Row::toString() const // WIP case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - if (colWidths[i] > MAXLEGACYWIDTH) + if (colWidths[i] == sizeof(int128_t)) { char *buf = (char*)alloca(precision[i] + 3); // empty the buffer - dataconvert::DataConvert::toString(getBinaryField(i), - buf, precision[i]+3); //WIP scale[i] + dataconvert::DataConvert::decimalToString(getBinaryField(i), + scale[i], buf, precision[i]+3, types[i]); os << buf << " "; break; } @@ -1048,8 +1048,6 @@ bool Row::isNullValue(uint32_t colIndex) const case 8: return (*((uint64_t*) &data[offsets[colIndex]]) == joblist::CHAR8NULL); - case 16: - std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; default: return (*((uint64_t*) &data[offsets[colIndex]]) == *((uint64_t*) joblist::CPNULLSTRMARK.c_str())); } @@ -1646,10 +1644,10 @@ void applyMapping(const int* mapping, const Row& in, Row* out) // WIP this doesn't look right b/c we can pushdown colType // Migrate to offset based methods here // code precision 2 width convertor - else if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::DECIMAL && - in.getColumnWidth(i) == 16)) - out->setBinaryField_offset(in.getBinaryField(i), 16, - out->getOffset(mapping[i])); + else if (UNLIKELY(execplan::isDecimal(in.getColTypes()[i]) + && in.getColumnWidth(i) == 16)) + out->setBinaryField_offset(in.getBinaryField(i), 16, + out->getOffset(mapping[i])); else if (in.isUnsigned(i)) out->setUintField(in.getUintField(i), mapping[i]); else diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 6a2bc7961..831fa6d0b 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -375,6 +375,8 @@ public: inline uint64_t getUintField(uint32_t colIndex) const; template inline int64_t getIntField(uint32_t colIndex) const; inline int64_t getIntField(uint32_t colIndex) const; + template + inline bool equals(T* value, uint32_t colIndex) const; template inline bool equals(uint64_t val, uint32_t colIndex) const; inline bool equals(long double val, uint32_t colIndex) const; bool equals(const std::string& val, uint32_t colIndex) const; @@ -664,6 +666,12 @@ inline bool Row::inStringTable(uint32_t col) const return strings && getColumnWidth(col) >= sTableThreshold && !forceInline[col]; } +template +inline bool Row::equals(T* value, uint32_t colIndex) const +{ + return reinterpret_cast(&data[offsets[colIndex]]) == value; +} + template inline bool Row::equals(uint64_t val, uint32_t colIndex) const { @@ -681,8 +689,6 @@ inline bool Row::equals(uint64_t val, uint32_t colIndex) const case 8: return *((uint64_t*) &data[offsets[colIndex]]) == val; - case 16: - std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; default: idbassert(0); throw std::logic_error("Row::equals(): bad length."); @@ -710,8 +716,6 @@ inline uint64_t Row::getUintField(uint32_t colIndex) const case 8: return *((uint64_t*) &data[offsets[colIndex]]); - case 16: - std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; default: idbassert(0); throw std::logic_error("Row::getUintField(): bad length."); @@ -730,8 +734,6 @@ inline uint64_t Row::getUintField(uint32_t colIndex) const case 4: return *((uint32_t*) &data[offsets[colIndex]]); - case 16: - std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; case 8: return *((uint64_t*) &data[offsets[colIndex]]); @@ -1281,6 +1283,85 @@ inline uint64_t Row::hash(uint32_t lastCol) const return ret; } +inline bool Row::equals(const Row& r2, const std::vector& keyCols) const +{ + for (uint32_t i = 0; i < keyCols.size(); i++) + { + const uint32_t& col = keyCols[i]; + + cscDataType columnType = getColType(i); + + if (!isLongString(col)) + { + if (UNLIKELY(columnType == execplan::CalpontSystemCatalog::LONGDOUBLE)) + { + if (getLongDoubleField(i) != r2.getLongDoubleField(i)) + return false; + } + else if (UNLIKELY(execplan::isDecimal(columnType))) + { + if (getBinaryField(i) != r2.getBinaryField(i)) + return false; + } + else if (getUintField(col) != r2.getUintField(col)) + return false; + } + else + { + if (getStringLength(col) != r2.getStringLength(col)) + return false; + + if (memcmp(getStringPointer(col), r2.getStringPointer(col), getStringLength(col))) + return false; + } + } + + return true; +} + +inline bool Row::equals(const Row& r2, uint32_t lastCol) const +{ + // This check fires with empty r2 only. + if (lastCol >= columnCount) + return true; + + if (!useStringTable && !r2.useStringTable) + return !(memcmp(&data[offsets[0]], &r2.data[offsets[0]], offsets[lastCol + 1] - offsets[0])); + + for (uint32_t i = 0; i <= lastCol; i++) + { + cscDataType columnType = getColType(i); + if (!isLongString(i)) + { + if (UNLIKELY(getColType(i) == execplan::CalpontSystemCatalog::LONGDOUBLE)) + { + if (getLongDoubleField(i) != r2.getLongDoubleField(i)) + return false; + } + else if (UNLIKELY(execplan::isDecimal(columnType))) + { + if (getBinaryField(i) != r2.getBinaryField(i)) + return false; + } + + else if (getUintField(i) != r2.getUintField(i)) + return false; + } + else + { + uint32_t len = getStringLength(i); + + if (len != r2.getStringLength(i)) + return false; + + if (memcmp(getStringPointer(i), r2.getStringPointer(i), len)) + return false; + } + } + + return true; +} + inline bool Row::equals(const Row& r2) const { return equals(r2, columnCount - 1); From 8f80c1dee6dc57dbc4bc25f53419d00653931ce2 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Tue, 25 Feb 2020 17:11:32 -0500 Subject: [PATCH 21/78] MCOL-641 1. Implement int128 version of strtoll. 2. Templatize number_int_value. 3. Add test cases for strtoll128 and number_int_value for Decimal38. --- utils/common/widedecimalutils.h | 67 +++ utils/dataconvert/dataconvert-tests.cpp | 542 ++++++++++++++++++++++++ utils/dataconvert/dataconvert.cpp | 194 ++++++--- utils/dataconvert/dataconvert.h | 119 +++++- writeengine/xml/we_xmljob.cpp | 1 + 5 files changed, 869 insertions(+), 54 deletions(-) create mode 100644 utils/common/widedecimalutils.h create mode 100644 utils/dataconvert/dataconvert-tests.cpp diff --git a/utils/common/widedecimalutils.h b/utils/common/widedecimalutils.h new file mode 100644 index 000000000..be0c742d9 --- /dev/null +++ b/utils/common/widedecimalutils.h @@ -0,0 +1,67 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#ifndef WIDE_DECIMAL_UTILS_H +#define WIDE_DECIMAL_UTILS_H + +namespace utils +{ + +using int128_t = __int128; +using uint128_t = unsigned __int128; + +const uint64_t BINARYNULLVALUELOW = 0ULL; +const uint64_t BINARYNULLVALUEHIGH = 0x8000000000000000ULL; +const uint64_t BINARYEMPTYVALUELOW = 1ULL; +const uint64_t BINARYEMPTYVALUEHIGH = 0x8000000000000000ULL; + + inline bool isWideDecimalNullValue(const int128_t val) + { + const uint64_t* ptr = reinterpret_cast(&val); + return (ptr[0] == BINARYNULLVALUELOW && ptr[1] == BINARYNULLVALUEHIGH); + } + + inline bool isWideDecimalEmptyValue(const int128_t val) + { + const uint64_t* ptr = reinterpret_cast(&val); + return (ptr[0] == BINARYEMPTYVALUELOW && ptr[1] == BINARYEMPTYVALUEHIGH); + } + + inline void int128Max(int128_t& val) + { + uint64_t* ptr = reinterpret_cast(&val); + ptr[0] = 0xFFFFFFFFFFFFFFFF; + ptr[1] = 0x7FFFFFFFFFFFFFFF; + } + + inline void int128Min(int128_t& val) + { + uint64_t* ptr = reinterpret_cast(&val); + ptr[0] = 0; + ptr[1] = 0x8000000000000000; + } + + inline void uint128Max(uint128_t& val) + { + uint64_t* ptr = reinterpret_cast(&val); + ptr[0] = 0xFFFFFFFFFFFFFFFF; + ptr[1] = 0xFFFFFFFFFFFFFFFF; + } + +} + +#endif // WIDE_DECIMAL_UTILS_H diff --git a/utils/dataconvert/dataconvert-tests.cpp b/utils/dataconvert/dataconvert-tests.cpp new file mode 100644 index 000000000..3a18571e8 --- /dev/null +++ b/utils/dataconvert/dataconvert-tests.cpp @@ -0,0 +1,542 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include +using namespace std; + +#include "gtest/gtest.h" + +#include "calpontsystemcatalog.h" +using namespace execplan; +#include "dataconvert.h" +using namespace dataconvert; + +TEST(DataConvertTest, Strtoll128) +{ + char *ep = NULL; + bool saturate = false; + string str; + int128_t val, valMax; + bitset<64> b1, b2, b3, b4; + + // test empty + str = ""; + val = strtoll128(str.c_str(), saturate, &ep); + EXPECT_EQ(val, 0); + EXPECT_EQ(*ep, '\0'); + EXPECT_FALSE(saturate); + + // test simple values + str = "123"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + EXPECT_EQ(val, 123); + EXPECT_EQ(*ep, '\0'); + EXPECT_FALSE(saturate); + str = " 123"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + EXPECT_EQ(val, 123); + EXPECT_EQ(*ep, '\0'); + EXPECT_FALSE(saturate); + str = " 123.45"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + EXPECT_EQ(val, 123); + EXPECT_NE(*ep, '\0'); + EXPECT_FALSE(saturate); + str = "-123"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + EXPECT_EQ(val, -123); + EXPECT_EQ(*ep, '\0'); + EXPECT_FALSE(saturate); + str = " -123"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + EXPECT_EQ(val, -123); + EXPECT_EQ(*ep, '\0'); + EXPECT_FALSE(saturate); + str = " -123.45"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + EXPECT_EQ(val, -123); + EXPECT_NE(*ep, '\0'); + EXPECT_FALSE(saturate); + + // test max/min values + // test max + str = "170141183460469231731687303715884105727"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + b1 = *(reinterpret_cast(&val)); + b2 = *(reinterpret_cast(&val) + 1); + valMax = ((((((((int128_t)170141183 * 1000000000) + 460469231) * 1000000000) + 731687303) * 1000000000 ) + 715884105) * 1000) + 727; + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_EQ(*ep, '\0'); + EXPECT_FALSE(saturate); + // test min + str = "-170141183460469231731687303715884105728"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + b1 = *(reinterpret_cast(&val)); + b2 = *(reinterpret_cast(&val) + 1); + valMax = ((((((((int128_t)170141183 * 1000000000) + 460469231) * 1000000000) + 731687303) * 1000000000 ) + 715884105) * 1000) + 727; + valMax = -valMax - 1; + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_EQ(*ep, '\0'); + EXPECT_FALSE(saturate); + + // test saturation + // test saturation to max + str = "170141183460469231731687303715884105728"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + b1 = *(reinterpret_cast(&val)); + b2 = *(reinterpret_cast(&val) + 1); + valMax = ((((((((int128_t)170141183 * 1000000000) + 460469231) * 1000000000) + 731687303) * 1000000000 ) + 715884105) * 1000) + 727; + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_EQ(*ep, '\0'); + EXPECT_TRUE(saturate); + // test saturation to min + str = "-170141183460469231731687303715884105729"; + saturate = false; + val = strtoll128(str.c_str(), saturate, &ep); + b1 = *(reinterpret_cast(&val)); + b2 = *(reinterpret_cast(&val) + 1); + valMax = ((((((((int128_t)170141183 * 1000000000) + 460469231) * 1000000000) + 731687303) * 1000000000 ) + 715884105) * 1000) + 727; + valMax = -valMax - 1; + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_EQ(*ep, '\0'); + EXPECT_TRUE(saturate); +} + +TEST(DataConvertTest, NumberIntValue) +{ + CalpontSystemCatalog::ColType ct; + int128_t res, valMax; + string data; + bool noRoundup = false; + bool pushWarning; + bitset<64> b1, b2, b3, b4; + + // tests for signed decimal + // behaviour of number_int_value for unsigned decimal + // is similar to the signed case. + ct.colDataType = CalpontSystemCatalog::DECIMAL; + // test with decimal(38,0) + ct.precision = 38; + ct.scale = 0; + // test simple values + //data = ""; + data = "0"; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 0); + EXPECT_FALSE(pushWarning); + data = "1234"; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 1234); + EXPECT_FALSE(pushWarning); + data = "12.0"; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 12); + EXPECT_FALSE(pushWarning); + data = "12.34"; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 12); + EXPECT_TRUE(pushWarning); + data = "-1234"; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, -1234); + EXPECT_FALSE(pushWarning); + data = "-12.34"; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, -12); + EXPECT_TRUE(pushWarning); + // test max + data = "99999999999999999999999999999999999999"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + // test min + data = "-99999999999999999999999999999999999999"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + valMax = -valMax; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + // test rounding + data = "12.56"; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 13); + EXPECT_TRUE(pushWarning); + data = "-12.56"; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, -13); + EXPECT_TRUE(pushWarning); + // test saturation + data = "999999999999999999999999999999999999999"; // data has 39 9's + // valMax has 38 9's + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + data = "-999999999999999999999999999999999999999"; // data has 39 9's + // valMax has 38 9's + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + valMax = -valMax; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + // test scientific notation + data = "1.23e37"; + valMax = ((((((((int128_t)123000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 100) + 0; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + data = "1.23e38"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + + // test with decimal(38,10) + ct.scale = 10; + data = "0"; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 0); + EXPECT_FALSE(pushWarning); + data = "1234"; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 12340000000000); + EXPECT_FALSE(pushWarning); + data = "12.0"; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 120000000000); + EXPECT_FALSE(pushWarning); + data = "12.34"; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 123400000000); + EXPECT_FALSE(pushWarning); + data = "-1234"; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, -12340000000000); + EXPECT_FALSE(pushWarning); + data = "-12.34"; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, -123400000000); + EXPECT_FALSE(pushWarning); + // test max + data = "9999999999999999999999999999.9999999999"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + // test min + data = "-9999999999999999999999999999.9999999999"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + valMax = -valMax; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + // test rounding + data = "12.11111111119"; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 121111111112); + EXPECT_TRUE(pushWarning); + data = "-12.11111111119"; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, -121111111112); + EXPECT_TRUE(pushWarning); + // test saturation + data = "99999999999999999999999999999"; // data has 29 9's + // valMax has 38 9's + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + data = "-99999999999999999999999999999"; // data has 29 9's + // valMax has 38 9's + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + valMax = -valMax; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + // test scientific notation + data = "1.23e9"; + valMax = ((((int128_t)123000000 * 1000000000) + 0) * 100); + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + data = "1.23e28"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + // test with decimal(38,38) + ct.scale = 38; + data = "0"; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + EXPECT_EQ(res, 0); + EXPECT_FALSE(pushWarning); + data = "1.234"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + data = "0.123"; + valMax = ((((((((int128_t)123000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 100) + 0; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + data = "-1.234"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + valMax = -valMax; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + data = "-0.123"; + valMax = ((((((((int128_t)123000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 100) + 0; + valMax = -valMax; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + // test max + data = "0.99999999999999999999999999999999999999"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + // test min + data = "-0.99999999999999999999999999999999999999"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + valMax = -valMax; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + // test rounding + data = "0.199999999999999999999999999999999999999"; + valMax = ((((((((int128_t)200000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 100) + 0; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + data = "-0.199999999999999999999999999999999999999"; + valMax = ((((((((int128_t)200000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 100) + 0; + valMax = -valMax; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + // test saturation + data = "99999999999999999999999999999"; // data has 29 9's + // valMax has 38 9's + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + data = "-99999999999999999999999999999"; // data has 29 9's + // valMax has 38 9's + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + valMax = -valMax; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); + // test scientific notation + data = "123e-4"; + valMax = ((((((((int128_t)123000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 10) + 0; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_FALSE(pushWarning); + data = "123e-2"; + valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + pushWarning = false; + number_int_value(data, ct, pushWarning, noRoundup, res); + b1 = *(reinterpret_cast(&res)); + b2 = *(reinterpret_cast(&res) + 1); + b3 = *(reinterpret_cast(&valMax)); + b4 = *(reinterpret_cast(&valMax) + 1); + EXPECT_EQ(b1, b3); + EXPECT_EQ(b2, b4); + EXPECT_TRUE(pushWarning); +} diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 6838cebbe..1273e134f 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -37,7 +37,6 @@ using namespace boost::algorithm; #include "calpontsystemcatalog.h" #include "calpontselectexecutionplan.h" #include "columnresult.h" -#include "common/branchpred.h" using namespace execplan; #include "joblisttypes.h" @@ -79,6 +78,30 @@ const int64_t columnstore_precision[19] = 999999999999999999LL }; +const string columnstore_big_precision[20] = +{ + "9999999999999999999", + "99999999999999999999", + "999999999999999999999", + "9999999999999999999999", + "99999999999999999999999", + "999999999999999999999999", + "9999999999999999999999999", + "99999999999999999999999999", + "999999999999999999999999999", + "9999999999999999999999999999", + "99999999999999999999999999999", + "999999999999999999999999999999", + "9999999999999999999999999999999", + "99999999999999999999999999999999", + "999999999999999999999999999999999", + "9999999999999999999999999999999999", + "99999999999999999999999999999999999", + "999999999999999999999999999999999999", + "9999999999999999999999999999999999999", + "99999999999999999999999999999999999999" +}; + template bool from_string(T& t, const std::string& s, std::ios_base & (*f)(std::ios_base&)) { @@ -103,10 +126,17 @@ bool number_value ( const string& data ) return true; } -int64_t number_int_value(const string& data, - const CalpontSystemCatalog::ColType& ct, - bool& pushwarning, - bool noRoundup) +} // namespace anon + +namespace dataconvert +{ + +template +void number_int_value(const string& data, + const CalpontSystemCatalog::ColType& ct, + bool& pushwarning, + bool noRoundup, + T& intVal) { // copy of the original input string valStr(data); @@ -132,11 +162,13 @@ int64_t number_int_value(const string& data, if (boost::iequals(valStr, "true")) { - return 1; + intVal = 1; + return; } if (boost::iequals(valStr, "false")) { - return 0; + intVal = 0; + return; } // convert to fixed-point notation if input is in scientific notation @@ -152,9 +184,9 @@ int64_t number_int_value(const string& data, // get the exponent string exp = valStr.substr(epos + 1); bool overflow = false; - int64_t exponent = dataconvert::string_to_ll(exp, overflow); + T exponent = dataconvert::string_to_ll(exp, overflow); - // if the exponent can not be held in 64-bit, not supported or saturated. + // if the exponent can not be held in 64 or 128 bits, not supported or saturated. if (overflow) throw QueryDataExcept("value is invalid.", formatErr); @@ -265,7 +297,7 @@ int64_t number_int_value(const string& data, if (dp != string::npos) { //Check if need round up - int frac1 = dataconvert::string_to_ll(valStr.substr(dp + 1, 1), pushwarning); + int frac1 = dataconvert::string_to_ll(valStr.substr(dp + 1, 1), pushwarning); if ((!noRoundup) && frac1 >= 5) roundup = 1; @@ -281,11 +313,11 @@ int64_t number_int_value(const string& data, } } - int64_t intVal = dataconvert::string_to_ll(intStr, pushwarning); + intVal = dataconvert::string_to_ll(intStr, pushwarning); //@Bug 3350 negative value round up. intVal += intVal >= 0 ? roundup : -roundup; bool dummy = false; - int64_t frnVal = (frnStr.length() > 0) ? dataconvert::string_to_ll(frnStr, dummy) : 0; + T frnVal = (frnStr.length() > 0) ? dataconvert::string_to_ll(frnStr, dummy) : 0; if (frnVal != 0) pushwarning = true; @@ -406,6 +438,16 @@ int64_t number_int_value(const string& data, pushwarning = true; } } + else if (ct.colWidth == 16) + { + int128_t tmp; + utils::int128Min(tmp); + if (intVal < tmp + 2) // + 2 for NULL and EMPTY values + { + intVal = tmp + 2; + pushwarning = true; + } + } break; @@ -418,8 +460,20 @@ int64_t number_int_value(const string& data, (ct.colDataType == CalpontSystemCatalog::UDECIMAL) || (ct.scale > 0)) { - int64_t rangeUp = columnstore_precision[ct.precision]; - int64_t rangeLow = -rangeUp; + T rangeUp, rangeLow; + + if (ct.precision < 19) + { + rangeUp = (T) columnstore_precision[ct.precision]; + } + else + { + bool dummy = false; + char *ep = NULL; + rangeUp = (T) dataconvert::strtoll128(columnstore_big_precision[ct.precision - 19].c_str(), dummy, &ep); + } + + rangeLow = -rangeUp; if (intVal > rangeUp) { @@ -432,10 +486,23 @@ int64_t number_int_value(const string& data, pushwarning = true; } } - - return intVal; } +// Explicit template instantiation +template +void number_int_value(const std::string& data, + const execplan::CalpontSystemCatalog::ColType& ct, + bool& pushwarning, + bool noRoundup, + int64_t& intVal); + +template +void number_int_value(const std::string& data, + const execplan::CalpontSystemCatalog::ColType& ct, + bool& pushwarning, + bool noRoundup, + int128_t& intVal); + uint64_t number_uint_value(const string& data, const CalpontSystemCatalog::ColType& ct, bool& pushwarning, @@ -476,7 +543,7 @@ uint64_t number_uint_value(const string& data, // get the exponent string exp = valStr.substr(epos + 1); bool overflow = false; - int64_t exponent = dataconvert::string_to_ll(exp, overflow); + int64_t exponent = dataconvert::string_to_ll(exp, overflow); // if the exponent can not be held in 64-bit, not supported or saturated. if (overflow) @@ -596,11 +663,6 @@ uint64_t number_uint_value(const string& data, return uintVal; } -} // namespace anon - -namespace dataconvert -{ - /** * This function reads a decimal value from a string. It will stop processing * in 3 cases: @@ -1427,6 +1489,7 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, bool noRoundup, bool isUpdate) { boost::any value; + int64_t val64; // WIP std::string data( dataOrig ); pushWarning = false; @@ -1453,7 +1516,11 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, data.replace (x, 1, " "); } - if (number_int_value (data, colType, pushWarning, noRoundup)) + int64_t tmp = 0; + + number_int_value (data, colType, pushWarning, noRoundup, tmp); + + if (tmp) { bool bitvalue; @@ -1470,54 +1537,65 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, break; case CalpontSystemCatalog::TINYINT: - value = (char) number_int_value(data, colType, pushWarning, noRoundup); + number_int_value(data, colType, pushWarning, noRoundup, val64); + value = (char) val64; break; case CalpontSystemCatalog::SMALLINT: - value = (short) number_int_value(data, colType, pushWarning, noRoundup); + number_int_value(data, colType, pushWarning, noRoundup, val64); + value = (short) val64; break; case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: - value = (int) number_int_value(data, colType, pushWarning, noRoundup); + number_int_value(data, colType, pushWarning, noRoundup, val64); + value = (int) val64; break; case CalpontSystemCatalog::BIGINT: - value = (long long) number_int_value(data, colType, pushWarning, noRoundup); + number_int_value(data, colType, pushWarning, noRoundup, val64); + value = (long long) val64; break; - // MCOL-641 WIP - // Simplest form of a template will use colType and width as a parameter - // There will be lots specializations case CalpontSystemCatalog::DECIMAL: - // TODO MCOL-641 implement decimal38 version of number_int_value - if (colType.colWidth == 16) + if (colType.colWidth == 1) { - int128_t bigint; - // WIP - //atoi_(data, bigint); - atoi128(data, bigint); - value = bigint; + number_int_value(data, colType, pushWarning, noRoundup, val64); + value = (char) val64; } - else if (colType.colWidth == 1) - value = (char) number_int_value(data, colType, pushWarning, noRoundup); else if (colType.colWidth == 2) - value = (short) number_int_value(data, colType, pushWarning, noRoundup); + { + number_int_value(data, colType, pushWarning, noRoundup, val64); + value = (short) val64; + } else if (colType.colWidth == 4) - value = (int) number_int_value(data, colType, pushWarning, noRoundup); + { + number_int_value(data, colType, pushWarning, noRoundup, val64); + value = (int) val64; + } else if (colType.colWidth == 8) - value = (long long) number_int_value(data, colType, pushWarning, noRoundup); - else if (colType.colWidth == 32) - value = data; + { + number_int_value(data, colType, pushWarning, noRoundup, val64); + value = (long long) val64; + } + else if (colType.colWidth == 16) + { + int128_t val128; + number_int_value(data, colType, pushWarning, noRoundup, val128); + value = (int128_t) val128; + } + //else if (colType.colWidth == 32) + // value = data; break; - // MCOL-641 Implement UDECIMAL + case CalpontSystemCatalog::UDECIMAL: // UDECIMAL numbers may not be negative if (colType.colWidth == 1) { - char ival = (char) number_int_value(data, colType, pushWarning, noRoundup); + number_int_value(data, colType, pushWarning, noRoundup, val64); + char ival = (char) val64; if (ival < 0 && ival != static_cast(joblist::TINYINTEMPTYROW) && @@ -1531,7 +1609,8 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } else if (colType.colWidth == 2) { - short ival = (short) number_int_value(data, colType, pushWarning, noRoundup); + number_int_value(data, colType, pushWarning, noRoundup, val64); + short ival = (short) val64; if (ival < 0 && ival != static_cast(joblist::SMALLINTEMPTYROW) && @@ -1545,7 +1624,8 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } else if (colType.colWidth == 4) { - int ival = static_cast(number_int_value(data, colType, pushWarning, noRoundup)); + number_int_value(data, colType, pushWarning, noRoundup, val64); + int ival = static_cast(val64); if (ival < 0 && ival != static_cast(joblist::INTEMPTYROW) && @@ -1559,7 +1639,8 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } else if (colType.colWidth == 8) { - long long ival = static_cast(number_int_value(data, colType, pushWarning, noRoundup)); + number_int_value(data, colType, pushWarning, noRoundup, val64); + long long ival = static_cast(val64); if (ival < 0 && ival != static_cast(joblist::BIGINTEMPTYROW) && @@ -1571,6 +1652,21 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, value = ival; } + else if (colType.colWidth == 16) + { + int128_t val128; + number_int_value(data, colType, pushWarning, noRoundup, val128); + + if (val128 < 0 && + !utils::isWideDecimalNullValue(val128) && + !utils::isWideDecimalEmptyValue(val128)) + { + val128 = 0; + pushWarning = true; + } + + value = val128; + } break; diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 2301d75ec..67cb3265c 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -50,6 +50,9 @@ #include "calpontsystemcatalog.h" #include "columnresult.h" #include "exceptclasses.h" +#include "common/branchpred.h" + +#include "widedecimalutils.h" // remove this block if the htonll is defined in library #ifdef __linux__ @@ -822,8 +825,8 @@ void TimeStamp::reset() second = 0xFFFFFFFFFFF; } -inline -int64_t string_to_ll( const std::string& data, bool& bSaturate ) +template +inline T string_to_ll( const std::string& data, bool& bSaturate ) { // This function doesn't take into consideration our special values // for NULL and EMPTY when setting the saturation point. Should it? @@ -870,6 +873,13 @@ uint64_t string_to_ull( const std::string& data, bool& bSaturate ) return value; } +template +void number_int_value(const std::string& data, + const execplan::CalpontSystemCatalog::ColType& ct, + bool& pushwarning, + bool noRoundup, + T& intVal); + /** @brief DataConvert is a component for converting string data to Calpont format */ class DataConvert @@ -1044,9 +1054,7 @@ public: const uint8_t scale); template static size_t writeFractionalPart(T* dec, char* p, - const uint16_t buflen, const uint8_t scale); - - + const uint16_t buflen, const uint8_t scale); static inline void int128Max(int128_t& i) { @@ -1465,6 +1473,107 @@ inline std::string DataConvert::constructRegexp(const std::string& str) return ret; } +inline int128_t add128(int128_t a, int128_t b) +{ + return a + b; +} + +inline int128_t subtract128(int128_t a, int128_t b) +{ + return a - b; +} + +inline bool lessThan128(int128_t a, int128_t b) +{ + return a < b; +} + +inline bool greaterThan128(int128_t a, int128_t b) +{ + return a > b; +} + +// Naive __int128 version of strtoll +inline int128_t strtoll128(const char* data, bool& saturate, char** ep) +{ + int128_t res = 0; + + if (*data == '\0') + { + if (ep) + *ep = (char*)data; + return res; + } + + // skip leading whitespace characters + while (*data != '\0' && + (*data == ' ' || *data == '\t' || *data == '\n')) + data++; + + int128_t (*op)(int128_t, int128_t); + op = add128; + bool (*compare)(int128_t, int128_t); + compare = lessThan128; + + // check the -ve sign + bool is_neg = false; + if (*data == '-') + { + is_neg = true; + op = subtract128; + compare = greaterThan128; + data++; + } + + int128_t tmp; + + for (; *data != '\0' && isdigit(*data); data++) + { + tmp = op(res*10, *data - '0'); + + if (UNLIKELY(compare(tmp, res))) + { + saturate = true; + + if (is_neg) + utils::int128Min(res); + else + utils::int128Max(res); + + while (*data != '\0' && isdigit(*data)) + data++; + + if (ep) + *ep = (char*)data; + + return res; + } + + res = tmp; + } + + if (ep) + *ep = (char*)data; + + return res; +} + +template<> +inline int128_t string_to_ll ( const std::string& data, bool& bSaturate ) +{ + // This function doesn't take into consideration our special values + // for NULL and EMPTY when setting the saturation point. Should it? + char* ep = NULL; + const char* str = data.c_str(); + int128_t value = strtoll128(str, bSaturate, &ep); + + // (no digits) || (more chars) + if ((ep == str) || (*ep != '\0')) + throw logging::QueryDataExcept("value is not numerical.", logging::formatErr); + + return value; +} + } // namespace dataconvert #undef EXPORT diff --git a/writeengine/xml/we_xmljob.cpp b/writeengine/xml/we_xmljob.cpp index 863e5cbef..f93859b61 100644 --- a/writeengine/xml/we_xmljob.cpp +++ b/writeengine/xml/we_xmljob.cpp @@ -48,6 +48,7 @@ using namespace execplan; namespace WriteEngine { // Maximum saturation value for DECIMAL types based on precision +// TODO MCOL-641 add support here. see dataconvert.cpp const long long columnstore_precision[19] = { 0, From 61647c1f5be18f6a5b2a4d7ef2f6276a7c24b3a9 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 27 Feb 2020 17:58:58 +0000 Subject: [PATCH 22/78] MCOL-641 DataConvert::decimalToString() refactoring. --- utils/dataconvert/dataconvert-tests.cpp | 103 ++++++++++++ utils/dataconvert/dataconvert.cpp | 209 ++++++++---------------- utils/dataconvert/dataconvert.h | 5 +- utils/rowgroup/rowgroup-tests.cpp | 18 +- 4 files changed, 193 insertions(+), 142 deletions(-) diff --git a/utils/dataconvert/dataconvert-tests.cpp b/utils/dataconvert/dataconvert-tests.cpp index 3a18571e8..b184030cb 100644 --- a/utils/dataconvert/dataconvert-tests.cpp +++ b/utils/dataconvert/dataconvert-tests.cpp @@ -24,6 +24,10 @@ using namespace std; using namespace execplan; #include "dataconvert.h" using namespace dataconvert; +#include "joblisttypes.h" +#include "columnwidth.h" + +using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; TEST(DataConvertTest, Strtoll128) { @@ -540,3 +544,102 @@ TEST(DataConvertTest, NumberIntValue) EXPECT_EQ(b2, b4); EXPECT_TRUE(pushWarning); } + +TEST(DataConvertTest, atoiCheck) { + std::vector expected; + std::vector decimalAsStringVec; + decimalAsStringVec.push_back("99999999999999999999999999999999999999"); + decimalAsStringVec.push_back("-99999999999999999999999999999999999999"); + decimalAsStringVec.push_back("0"); + decimalAsStringVec.push_back("-0.042"); + expected.push_back(10000000000000000000ULL*(uint128_t)9999999999999999999ULL + +9999999999999999999ULL); + expected.push_back(-1*expected[0]); + expected.push_back(0); + expected.push_back(-42); + for (int i=0; i < expected.size(); i++) { + int128_t value; + dataconvert::atoi128(decimalAsStringVec[i], value); + EXPECT_EQ(expected[i], value); + EXPECT_NE(expected[i], value-1); + } +} + +TEST(DataConvertTest, DecimalToStringCheckScale3) { + std::vector expected; + std::vector decimalAsStringVec; + // scale 3 + uint8_t scale3 = 3; + decimalAsStringVec.push_back("99999999999999999999999999999999999999"); + decimalAsStringVec.push_back("-99999999999999999999999999999999999999"); + decimalAsStringVec.push_back("0"); + decimalAsStringVec.push_back("-0.042"); + expected.push_back("99999999999999999999999999999999999.999"); + expected.push_back("-99999999999999999999999999999999999.999"); + expected.push_back("0.000"); + expected.push_back("-0.042"); + char buf[42]; + CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL; + for (int i=0; i < expected.size(); i++) { + int128_t value = -4; + memset(buf, 0, 42); + dataconvert::atoi128(decimalAsStringVec[i], value); + dataconvert::DataConvert::decimalToString(&value, scale3, buf, + utils::precisionByWidth(sizeof(value))+4, type); + EXPECT_EQ(expected[i], std::string(buf)); + } +} + +TEST(DataConvertTest, DecimalToStringCheckScale37) { + std::vector expected; + std::vector decimalAsStringVec; + uint8_t scale37 = 37; + decimalAsStringVec.push_back("99999999999999999999999999999999999999"); + decimalAsStringVec.push_back("-99999999999999999999999999999999999999"); + decimalAsStringVec.push_back("0"); + decimalAsStringVec.push_back("-42"); + decimalAsStringVec.push_back("-19999999999999999999999999999999999999"); + expected.push_back("9.9999999999999999999999999999999999999"); + expected.push_back("-9.9999999999999999999999999999999999999"); + expected.push_back("0.0000000000000000000000000000000000000"); + expected.push_back("-0.0000000000000000000000000000000000042"); + expected.push_back("-1.9999999999999999999999999999999999999"); + char buf[42]; + CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL; + for (int i=0; i < expected.size(); i++) { + int128_t value = -4; + memset(buf, 0, 41); + dataconvert::atoi128(decimalAsStringVec[i], value); + dataconvert::DataConvert::decimalToString(&value, scale37, buf, + utils::precisionByWidth(sizeof(value))+4, type); + EXPECT_EQ(expected[i], std::string(buf)); + } +} + +TEST(DataConvertTest, DecimalToStringCheckScale38) { + std::vector expected; + std::vector decimalAsStringVec; + uint8_t scale38 = 38; + decimalAsStringVec.push_back("99999999999999999999999999999999999999"); + decimalAsStringVec.push_back("-99999999999999999999999999999999999999"); + decimalAsStringVec.push_back("0"); + decimalAsStringVec.push_back("-42"); + expected.push_back("0.99999999999999999999999999999999999999"); + expected.push_back("-0.99999999999999999999999999999999999999"); + expected.push_back("0.00000000000000000000000000000000000000"); + expected.push_back("-0.00000000000000000000000000000000000042"); + char buf[42]; + CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL; + for (int i=0; i < expected.size(); i++) { + int128_t value = -4; + memset(buf, 0, 41); + dataconvert::atoi128(decimalAsStringVec[i], value); + dataconvert::DataConvert::decimalToString(&value, scale38, buf, + utils::precisionByWidth(sizeof(value))+4, type); + EXPECT_EQ(expected[i], std::string(buf)); + } +} + +TEST(DataConvertTest, ConvertColumnData) { +} + diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 1273e134f..beaf5a7dd 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1227,16 +1227,16 @@ bool stringToTimestampStruct(const string& data, TimeStamp& timeStamp, const str } // WIP MCOL-641 -// Second DT is for POD representation. template size_t DataConvert::writeIntPart(T* dec, char* p, const uint16_t buflen, - const uint8_t scale) + const uint8_t scale) //don't need this { T intPart = *dec; if (scale) { - // optimize this + //TODO Use dictionary to produce divisor + // instead of a loop for (size_t i = 0; i < scale; i++) intPart /= 10; } @@ -1292,90 +1292,84 @@ size_t DataConvert::writeFractionalPart(T* dec, char* p, const uint16_t buflen, const uint8_t scale) { - T scaleDivisor = 10; - for (size_t i = 1; i < scale; i++) - scaleDivisor *= 10; + //TODO Use dictionary instead of multiplication. + T scaleDivisor = 10; + for (size_t i = 1; i < scale; i++) + scaleDivisor *= 10; - T fractionalPart = *dec % scaleDivisor; - return writeIntPart(&fractionalPart, p, buflen, 0); + T fractionalPart = *dec % scaleDivisor; + // divide by the base untill we have non-zero quotinent + size_t written = 0; + scaleDivisor /= 10; + while (scaleDivisor > 1 && *dec / scaleDivisor == 0) + { + *p++ = '0'; + written++; + scaleDivisor /= 10; + } + written += writeIntPart(&fractionalPart, p, buflen, 0); + if (written < scale) + { + p += written; + for (size_t left = written; left < scale; left++) + { + *p++ = '0'; + } + } + return scale; } -// WIP MCOL-641 -// Limit this to Decimal only -// Replace decimalToString with this one template -void DataConvert::toString1(T* dec, char *p, const uint16_t buflen, - const uint8_t scale) +void DataConvert::toString(T* dec, uint8_t scale, + char *p, unsigned int buflen) { - if (*dec < static_cast(0)) - { - *p++ = '-'; - *dec *= -1; - } + char* original_p = p; + size_t written = 0; + // Early return for NULL value + int128_t sign = 0; + // WIP use constants here + // WIP Treat both magics here + uint64_t* signPod = reinterpret_cast(&sign); + signPod[1] = utils::BINARYNULLVALUEHIGH; - char* original_p = p; - size_t written = 0; - written = writeIntPart(dec, p, buflen, scale); - p += written; + if (*dec == sign) + { + *p++ = '0'; + if (scale) + { + *p++ = '.'; + while (scale-- > 0) + *p++ = '0'; + } + return; + } - // WIP To be finished for 0.042 - if (scale) - { - *p++ = '.'; - p += writeFractionalPart(dec, p, p-original_p, scale); - } + if (*dec < static_cast(0)) + { + *p++ = '-'; + *dec *= -1; + } - if (buflen <= p-original_p) - { - throw QueryDataExcept("toString() char buffer overflow.", formatErr); - } + written = writeIntPart(dec, p, buflen, scale); + p += written; + + if (scale) + { + *p++ = '.'; + p += writeFractionalPart(dec, p, buflen-(p-original_p), scale); + } + + if (buflen <= p-original_p) + { + throw QueryDataExcept("toString() char buffer overflow.", formatErr); + } } -template -void DataConvert::toString(T* dec, char *p, size_t buflen) -{ - uint64_t div = 10000000000000000000ULL; - // template this - uint128_t high = *dec; - uint128_t low; - low = high % div; - high /= div; - uint128_t mid; - mid = high % div; - high /= div; - - // WIP How to treat PODs here ? - // use typeof - // Or a templated structure - // Use uint64* to access parts of uint128 and remove pods - Int128Pod_t *high_pod = reinterpret_cast(&high); - Int128Pod_t *mid_pod = reinterpret_cast(&mid); - Int128Pod_t *low_pod = reinterpret_cast(&low); - char* original_p = p; - int printed_chars = 0; - - // WIP replace snprintf with streams - if (high_pod->lo != 0) { - printed_chars = sprintf(p, "%lu", high_pod->lo); - p += printed_chars; - printed_chars = sprintf(p, "%019lu", mid_pod->lo); - p += printed_chars; - sprintf(p, "%019lu", low_pod->lo); - } else if (mid_pod->lo != 0) { - printed_chars = sprintf(p, "%lu", mid_pod->lo); - p += printed_chars; - sprintf(p, "%019lu", low_pod->lo); - } - else { - sprintf(p, "%lu", low_pod->lo); - } - if (buflen <= p-original_p) - std::cout << "DataConvert::toString char buffer overflow" << std::endl; -} +template +void DataConvert::toString(int128_t* dec, uint8_t scale, + char *p, unsigned int buflen); // WIP MCOL-641 -// Template this -// result must be calloc-ed void atoi128(const std::string& arg, int128_t& res) { res = 0; @@ -1386,14 +1380,12 @@ void atoi128(const std::string& arg, int128_t& res) if (LIKELY(arg[j]-'0' >= 0)) res = res*10 + arg[j] - '0'; } - // Use bit shift if possible if (idx) res *= -1; - //toString(res, buf); - //std::cerr << "atoi_ " << buf <(0)) - { - *ptr++ = '-'; - value *= -1; - idbassert(l1 >= 2); - l1--; - } - //we want to move the last scale chars right by one spot to insert the dp - //we want to move the trailing null as well, so it's really scale+1 chars - - toString(&value, ptr, buflen); - - // Biggest ColumnStore supports is DECIMAL(38,x), or 38 total digits+dp+sign for column - - if (scale == 0) - return; - - //need to make sure we have enough leading zeros for this to work... - //at this point scale is always > 0 - size_t l2 = 1; - - if ((unsigned)scale > l1) - { - const char* zeros = "00000000000000000000000000000000000000"; //38 0's - size_t diff = 0; - - if (*valuePtr != 0) - diff = scale - l1; //this will always be > 0 - else - diff = scale; - - memmove((ptr + diff), ptr, l1 + 1); //also move null - memcpy(ptr, zeros, diff); - - if (*valuePtr != 0) - l1 = 0; - else - l1 = 1; - } - else if ((unsigned)scale == l1) - { - l1 = 0; - l2 = 2; - } - else - { - l1 -= scale; - } - - memmove((ptr + l1 + l2), (ptr + l1), scale + 1); //also move null - - if (l2 == 2) - *(ptr + l1++) = '0'; - - *(ptr + l1) = '.'; + toString(valuePtr, scale, buf, buflen); } // Explicit instantiation template diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 67cb3265c..78e3b8639 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -1044,11 +1044,8 @@ public: template EXPORT static void decimalToString(T* value, uint8_t scale, char* buf, unsigned int buflen, cscDataType colDataType); - template - static void toString(T* dec, char *p, size_t buflen); template - EXPORT static void toString1(T* dec, char* p, const uint16_t buflen, - const uint8_t scale = 0); + static void toString(T* dec, uint8_t scale, char* p, unsigned int buflen); template static size_t writeIntPart(T* dec, char* p, const uint16_t buflen, const uint8_t scale); diff --git a/utils/rowgroup/rowgroup-tests.cpp b/utils/rowgroup/rowgroup-tests.cpp index 7b9708526..a18f8d38b 100644 --- a/utils/rowgroup/rowgroup-tests.cpp +++ b/utils/rowgroup/rowgroup-tests.cpp @@ -1,3 +1,20 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + #include // googletest header file #include @@ -195,7 +212,6 @@ TEST_F(RowDecimalTest, getBinaryFieldCheck) { } } -// TBD Need to add asserts when toString will be finished TEST_F(RowDecimalTest, toStringCheck) { std::vector exemplarVector; exemplarVector.push_back(std::string("0: NULL NULL NULL NULL NULL NULL ")); From 97ee1609b2dd73841df7d4e19a97cafadfdde6cb Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Mon, 2 Mar 2020 14:38:40 +0000 Subject: [PATCH 23/78] MCOL-641 Replaced NULL binary constants. DataConvert::decimalToString, toString, writeIntPart, writeFractionalPart are not templates anymore. --- dbcon/joblist/joblisttypes.h | 7 +- dbcon/mysql/ha_mcs_impl.cpp | 29 ++---- primitives/linux-port/column.cpp | 23 ++-- primitives/primproc/columncommand.cpp | 6 +- utils/common/columnwidth.h | 5 +- utils/common/nullvaluemanip.cpp | 13 ++- utils/common/widedecimalutils.h | 49 +++++++-- utils/dataconvert/dataconvert.cpp | 127 ++++++++++++----------- utils/dataconvert/dataconvert.h | 14 +-- utils/funcexp/funcexp.cpp | 4 +- utils/rowgroup/rowaggregation.cpp | 19 +--- utils/rowgroup/rowgroup-tests.cpp | 12 +-- utils/rowgroup/rowgroup.cpp | 32 +++--- utils/rowgroup/rowgroup.h | 13 ++- writeengine/server/we_dmlcommandproc.cpp | 28 ++--- writeengine/wrapper/writeengine.cpp | 12 ++- 16 files changed, 200 insertions(+), 193 deletions(-) diff --git a/dbcon/joblist/joblisttypes.h b/dbcon/joblist/joblisttypes.h index cfd3a72e8..ccdf13228 100644 --- a/dbcon/joblist/joblisttypes.h +++ b/dbcon/joblist/joblisttypes.h @@ -83,8 +83,11 @@ const uint16_t NULL_UINT16 = USMALLINTNULL; const uint32_t NULL_UINT32 = UINTNULL; const uint64_t NULL_UINT64 = UBIGINTNULL; -const uint64_t BINARYEMPTYROW = UBIGINTEMPTYROW; -const uint64_t BINARYNULL = UBIGINTNULL; +const uint64_t BINARYNULLVALUELOW = 0ULL; +const uint64_t BINARYNULLVALUEHIGH = 0x8000000000000000ULL; +const uint64_t BINARYEMPTYVALUELOW = 1ULL; +const uint64_t BINARYEMPTYVALUEHIGH = 0x8000000000000000ULL; +const uint64_t BINARYEMPTYROW = 0ULL; const std::string CPNULLSTRMARK("_CpNuLl_"); const std::string CPSTRNOTFOUND("_CpNoTf_"); diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index ea23b3f44..135c5edd9 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -805,36 +805,21 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - // WIP MCOL-641 - if (row.getPrecision(s) > 18) + if (colType.colWidth == 16) { // unset null bit first // Might be redundant if ((*f)->null_ptr) *(*f)->null_ptr &= ~(*f)->null_bit; - uint128_t* udec; int128_t* dec; - // We won't have more than 38 digits + sign + dp - // Make this precision based - char buf[41]; + // We won't have more than [+-][0][.] + up to 38 digits + char buf[utils::MAXLENGTH16BYTES]; - // This C-style cast doesn't look appropriate. - // Is there a way to use decltype instead of if? - if (colType.colDataType == CalpontSystemCatalog::DECIMAL) - { - dec = row.getBinaryField(s); - dataconvert::DataConvert::decimalToString(dec, - (unsigned)colType.scale, buf, - sizeof(buf), colType.colDataType); - } - else - { - udec = row.getBinaryField(s); - dataconvert::DataConvert::decimalToString(udec, - (unsigned)colType.scale, buf, - sizeof(buf), colType.colDataType); - } + dec = row.getBinaryField(s); + dataconvert::DataConvert::decimalToString(dec, + (unsigned)colType.scale, buf, + sizeof(buf), colType.colDataType); Field_new_decimal* f2 = (Field_new_decimal*)*f; f2->store(buf, strlen(buf), f2->charset()); diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 463764cd0..b8acd52d9 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -40,6 +40,7 @@ using namespace boost; #include "stats.h" #include "primproc.h" #include "dataconvert.h" +#include "widedecimalutils.h" using namespace logging; using namespace dbbc; using namespace primitives; @@ -277,16 +278,17 @@ template<> inline bool isEmptyVal<32>(uint8_t type, const uint8_t* ival) // For BINARY { const uint64_t* val = reinterpret_cast(ival); - return ((val[0] == joblist::BINARYEMPTYROW) && (val[1] == joblist::BINARYEMPTYROW) - && (val[2] == joblist::BINARYEMPTYROW) && (val[3] == joblist::BINARYEMPTYROW)); + return ((val[0] == joblist::BINARYEMPTYVALUELOW) + && (val[1] == joblist::BINARYNULLVALUELOW) + && (val[2] == joblist::BINARYNULLVALUELOW) + && (val[3] == joblist::BINARYEMPTYVALUEHIGH)); } template<> inline bool isEmptyVal<16>(uint8_t type, const uint8_t* ival) // For BINARY { - const uint64_t* val = reinterpret_cast(ival); - return ((val[0] == joblist::BINARYEMPTYROW) && (val[1] == joblist::BINARYEMPTYROW)); - + const int128_t* val = reinterpret_cast(ival); + return utils::isWideDecimalEmptyValue (*val); } template<> @@ -410,19 +412,22 @@ inline bool isEmptyVal<1>(uint8_t type, const uint8_t* ival) template inline bool isNullVal(uint8_t type, const uint8_t* val8); +// WIP This method only works for wide DECIMAL so far. template<> inline bool isNullVal<16>(uint8_t type, const uint8_t* ival) // For BINARY { - const uint64_t* val = reinterpret_cast(ival); - return ((val[0] == joblist::BINARYNULL) && (val[1] == joblist::BINARYEMPTYROW)); + const int128_t* val = reinterpret_cast(ival); + return utils::isWideDecimalNullValue (*val); } template<> inline bool isNullVal<32>(uint8_t type, const uint8_t* ival) // For BINARY { const uint64_t* val = reinterpret_cast(ival); - return ((val[0] == joblist::BINARYNULL) && (val[1] == joblist::BINARYEMPTYROW) - && (val[2] == joblist::BINARYEMPTYROW) && (val[3] == joblist::BINARYEMPTYROW)); + return ((val[0] == joblist::BINARYNULLVALUELOW) + && (val[1] == joblist::BINARYNULLVALUELOW) + && (val[2] == joblist::BINARYNULLVALUELOW) + && (val[3] == joblist::BINARYNULLVALUEHIGH)); } template<> diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index d2145807c..8a50fd3e0 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -40,6 +40,7 @@ using namespace std; #include "primitiveserver.h" #include "primproc.h" #include "stats.h" +#include "widedecimalutils.h" using namespace messageqcpp; using namespace rowgroup; @@ -1062,9 +1063,8 @@ const uint64_t ColumnCommand::getEmptyRowValue( const CSCDataType dataType, cons void ColumnCommand::getEmptyRowValue(const CSCDataType dataType, const int width, messageqcpp::ByteStream::hexbyte* space) const { - uint64_t *ptr = reinterpret_cast(space); - ptr[0] = joblist::BINARYEMPTYROW; - ptr[1] = joblist::BINARYEMPTYROW; + int128_t *val = reinterpret_cast(space); + utils::setWideDecimalEMptyValue(*val); } void ColumnCommand::getLBIDList(uint32_t loopCount, vector* lbids) diff --git a/utils/common/columnwidth.h b/utils/common/columnwidth.h index b16086494..7d5865514 100644 --- a/utils/common/columnwidth.h +++ b/utils/common/columnwidth.h @@ -18,11 +18,12 @@ #ifndef UTILS_COLWIDTH_H #define UTILS_COLWIDTH_H -#define MAXLEGACYWIDTH 8 -#define MAXCOLUMNWIDTH 16 namespace utils { + const uint8_t MAXLEGACYWIDTH = 8ULL; + const uint8_t MAXCOLUMNWIDTH = 16ULL; + inline bool isWide(uint8_t width) { return width > MAXLEGACYWIDTH; diff --git a/utils/common/nullvaluemanip.cpp b/utils/common/nullvaluemanip.cpp index 63b0c8693..adb8e108d 100644 --- a/utils/common/nullvaluemanip.cpp +++ b/utils/common/nullvaluemanip.cpp @@ -127,7 +127,7 @@ uint64_t getNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidth) return joblist::UBIGINTNULL; case CalpontSystemCatalog::BINARY: - return joblist::BINARYNULL; + return joblist::BINARYNULLVALUELOW; case CalpontSystemCatalog::VARBINARY: default: @@ -201,7 +201,6 @@ int64_t getSignedNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidt default: throw logic_error("getSignedNullValue() Can't return the NULL string"); } - break; } @@ -219,8 +218,14 @@ int64_t getSignedNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidt case 4 : return (int64_t) ((int32_t) joblist::INTNULL); - default: + case 8: return joblist::BIGINTNULL; + + default: + ostringstream os; + os << "getSignedNullValue(): got bad column width (" << t << + "). Width=" << colWidth << endl; + throw logic_error(os.str()); } break; @@ -243,7 +248,7 @@ int64_t getSignedNullValue(CalpontSystemCatalog::ColDataType t, uint32_t colWidt return (int64_t)joblist::LONGDOUBLENULL; case CalpontSystemCatalog::BINARY: - return (int64_t)joblist::BINARYNULL; + return (int64_t)joblist::BINARYNULLVALUELOW; case CalpontSystemCatalog::VARBINARY: default: diff --git a/utils/common/widedecimalutils.h b/utils/common/widedecimalutils.h index be0c742d9..67302f2bd 100644 --- a/utils/common/widedecimalutils.h +++ b/utils/common/widedecimalutils.h @@ -18,30 +18,58 @@ #ifndef WIDE_DECIMAL_UTILS_H #define WIDE_DECIMAL_UTILS_H -namespace utils -{ - using int128_t = __int128; using uint128_t = unsigned __int128; -const uint64_t BINARYNULLVALUELOW = 0ULL; -const uint64_t BINARYNULLVALUEHIGH = 0x8000000000000000ULL; -const uint64_t BINARYEMPTYVALUELOW = 1ULL; -const uint64_t BINARYEMPTYVALUEHIGH = 0x8000000000000000ULL; +namespace utils +{ + const uint64_t BINARYNULLVALUELOW = 0ULL; + const uint64_t BINARYNULLVALUEHIGH = 0x8000000000000000ULL; + const uint64_t BINARYEMPTYVALUELOW = 1ULL; + const uint64_t BINARYEMPTYVALUEHIGH = 0x8000000000000000ULL; + const uint8_t MAXLENGTH16BYTES = 42; - inline bool isWideDecimalNullValue(const int128_t val) + inline bool isWideDecimalNullValue(const int128_t& val) { const uint64_t* ptr = reinterpret_cast(&val); return (ptr[0] == BINARYNULLVALUELOW && ptr[1] == BINARYNULLVALUEHIGH); } - inline bool isWideDecimalEmptyValue(const int128_t val) + inline bool isWideDecimalEmptyValue(const int128_t& val) { const uint64_t* ptr = reinterpret_cast(&val); return (ptr[0] == BINARYEMPTYVALUELOW && ptr[1] == BINARYEMPTYVALUEHIGH); } - inline void int128Max(int128_t& val) + inline void setWideDecimalNullValue(int128_t& val) + { + uint64_t* ptr = reinterpret_cast(&val); + ptr[0] = BINARYNULLVALUELOW; + ptr[1] = BINARYNULLVALUEHIGH; + } + + inline void setWideDecimalEMptyValue(int128_t& val) + { + uint64_t* ptr = reinterpret_cast(&val); + ptr[0] = BINARYEMPTYVALUELOW; + ptr[1] = BINARYEMPTYVALUEHIGH; + } + + inline void setWideDecimalNullValue(int128_t* val) + { + uint64_t* ptr = reinterpret_cast(val); + ptr[0] = BINARYNULLVALUELOW; + ptr[1] = BINARYNULLVALUEHIGH; + } + + inline void setWideDecimalEMptyValue(int128_t* val) + { + uint64_t* ptr = reinterpret_cast(val); + ptr[0] = BINARYEMPTYVALUELOW; + ptr[1] = BINARYEMPTYVALUEHIGH; + } + + inline void int128Max(int128_t& val) { uint64_t* ptr = reinterpret_cast(&val); ptr[0] = 0xFFFFFFFFFFFFFFFF; @@ -61,7 +89,6 @@ const uint64_t BINARYEMPTYVALUEHIGH = 0x8000000000000000ULL; ptr[0] = 0xFFFFFFFFFFFFFFFF; ptr[1] = 0xFFFFFFFFFFFFFFFF; } - } #endif // WIDE_DECIMAL_UTILS_H diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index beaf5a7dd..42f4295fa 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -102,6 +102,29 @@ const string columnstore_big_precision[20] = "99999999999999999999999999999999999999" }; +const uint64_t columnstore_pow_10[20] = +{ + 1ULL, + 10ULL, + 100ULL, + 1000ULL, + 10000ULL, + 100000ULL, + 1000000ULL, + 10000000ULL, + 100000000ULL, + 1000000000ULL, + 10000000000ULL, + 100000000000ULL, + 1000000000000ULL, + 10000000000000ULL, + 100000000000000ULL, + 1000000000000000ULL, + 10000000000000000ULL, + 100000000000000000ULL, + 1000000000000000000ULL, + 10000000000000000000ULL +}; template bool from_string(T& t, const std::string& s, std::ios_base & (*f)(std::ios_base&)) { @@ -1226,32 +1249,37 @@ bool stringToTimestampStruct(const string& data, TimeStamp& timeStamp, const str } -// WIP MCOL-641 -template -size_t DataConvert::writeIntPart(T* dec, char* p, +size_t DataConvert::writeIntPart(int128_t* dec, char* p, const uint16_t buflen, - const uint8_t scale) //don't need this + const uint8_t scale) { - T intPart = *dec; + int128_t intPart = *dec; if (scale) { - //TODO Use dictionary to produce divisor - // instead of a loop - for (size_t i = 0; i < scale; i++) - intPart /= 10; + uint8_t maxPowOf10 = sizeof(columnstore_pow_10)/sizeof(columnstore_pow_10[0])-1; + switch (scale/maxPowOf10) + { + case(2): + intPart /= columnstore_pow_10[maxPowOf10]; + intPart /= columnstore_pow_10[maxPowOf10]; + break; + case(1): + intPart /= columnstore_pow_10[maxPowOf10]; + case(0): + intPart /= columnstore_pow_10[scale%maxPowOf10]; + } } - // optimize for less then uint64_t values uint64_t div = 10000000000000000000ULL; - T high = intPart; - T low; + int128_t high = intPart; + int128_t low; low = high % div; high /= div; - T mid; + int128_t mid; mid = high % div; high /= div; - // pod[0] is high 8 byte, pod[1] is low + // pod[0] is low 8 byte, pod[1] is high uint64_t* high_pod = reinterpret_cast(&high); uint64_t* mid_pod = reinterpret_cast(&mid); uint64_t* low_pod = reinterpret_cast(&low); @@ -1287,17 +1315,29 @@ size_t DataConvert::writeIntPart(T* dec, char* p, return p-original_p; } -template -size_t DataConvert::writeFractionalPart(T* dec, char* p, +size_t DataConvert::writeFractionalPart(int128_t* dec, char* p, const uint16_t buflen, const uint8_t scale) { - //TODO Use dictionary instead of multiplication. - T scaleDivisor = 10; - for (size_t i = 1; i < scale; i++) - scaleDivisor *= 10; + int128_t scaleDivisor = 1; + + uint8_t maxPowOf10 = sizeof(columnstore_pow_10)/sizeof(columnstore_pow_10[0])-1; + switch (scale/maxPowOf10) + { + case(2): + scaleDivisor *= columnstore_pow_10[maxPowOf10]; + scaleDivisor *= columnstore_pow_10[maxPowOf10]; + break; + case(1): + scaleDivisor *= columnstore_pow_10[maxPowOf10]; + case(0): + scaleDivisor *= columnstore_pow_10[scale%maxPowOf10]; + } + + //for (size_t i = 1; i < scale; i++) + // scaleDivisor *= 10; - T fractionalPart = *dec % scaleDivisor; + int128_t fractionalPart = *dec % scaleDivisor; // divide by the base untill we have non-zero quotinent size_t written = 0; scaleDivisor /= 10; @@ -1319,38 +1359,24 @@ size_t DataConvert::writeFractionalPart(T* dec, char* p, return scale; } -template -void DataConvert::toString(T* dec, uint8_t scale, +void DataConvert::toString(int128_t* dec, uint8_t scale, char *p, unsigned int buflen) { char* original_p = p; size_t written = 0; - // Early return for NULL value - int128_t sign = 0; - // WIP use constants here - // WIP Treat both magics here - uint64_t* signPod = reinterpret_cast(&sign); - signPod[1] = utils::BINARYNULLVALUEHIGH; - - if (*dec == sign) + // Raise exception on NULL and EMPTY value + if (utils::isWideDecimalNullValue(*dec) || utils::isWideDecimalEmptyValue(*dec)) { - *p++ = '0'; - if (scale) - { - *p++ = '.'; - while (scale-- > 0) - *p++ = '0'; - } - return; + throw QueryDataExcept("toString() char buffer overflow.", formatErr); } - if (*dec < static_cast(0)) + if (*dec < static_cast(0)) { *p++ = '-'; *dec *= -1; } - written = writeIntPart(dec, p, buflen, scale); + written = writeIntPart(dec, p, buflen, scale); p += written; if (scale) @@ -1365,10 +1391,6 @@ void DataConvert::toString(T* dec, uint8_t scale, } } -template -void DataConvert::toString(int128_t* dec, uint8_t scale, - char *p, unsigned int buflen); - // WIP MCOL-641 void atoi128(const std::string& arg, int128_t& res) { @@ -1397,25 +1419,14 @@ void atoi128(const std::string& arg, uint128_t& res) } } -// WIP MCOL-641 -// Doesn't work for -0.042 -template -void DataConvert::decimalToString(T* valuePtr, +void DataConvert::decimalToString(int128_t* valuePtr, uint8_t scale, char* buf, unsigned int buflen, cscDataType colDataType) // We don't need the last one { - - toString(valuePtr, scale, buf, buflen); + toString(valuePtr, scale, buf, buflen); } -// Explicit instantiation -template -void DataConvert::decimalToString(int128_t* value, uint8_t scale, - char* buf, unsigned int buflen, cscDataType colDataType); -template -void DataConvert::decimalToString(uint128_t* value, uint8_t scale, - char* buf, unsigned int buflen, cscDataType colDataType); boost::any DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 78e3b8639..822d36e52 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -1041,17 +1041,13 @@ public: EXPORT static bool isNullData(execplan::ColumnResult* cr, int rownum, execplan::CalpontSystemCatalog::ColType colType); static inline std::string decimalToString(int64_t value, uint8_t scale, cscDataType colDataType); static inline void decimalToString(int64_t value, uint8_t scale, char* buf, unsigned int buflen, cscDataType colDataType); - template - EXPORT static void decimalToString(T* value, uint8_t scale, char* buf, unsigned int buflen, cscDataType colDataType); + static void decimalToString(int128_t* value, uint8_t scale, char* buf, unsigned int buflen, cscDataType colDataType); - template - static void toString(T* dec, uint8_t scale, char* p, unsigned int buflen); - template - static size_t writeIntPart(T* dec, char* p, const uint16_t buflen, + static void toString(int128_t* dec, uint8_t scale, char* p, unsigned int buflen); + static size_t writeIntPart(int128_t* dec, char* p, const uint16_t buflen, + const uint8_t scale); + static size_t writeFractionalPart(int128_t* dec, char* p, const uint16_t buflen, const uint8_t scale); - template - static size_t writeFractionalPart(T* dec, char* p, - const uint16_t buflen, const uint8_t scale); static inline void int128Max(int128_t& i) { diff --git a/utils/funcexp/funcexp.cpp b/utils/funcexp/funcexp.cpp index 854fbd8ed..d20382d30 100644 --- a/utils/funcexp/funcexp.cpp +++ b/utils/funcexp/funcexp.cpp @@ -477,11 +477,11 @@ void FuncExp::evaluate(rowgroup::Row& row, std::vector& expressi { // WIP make this a separate function w and w/o overflow check if (expression[i]->resultType().colDataType == execplan::CalpontSystemCatalog::DECIMAL) - row.setBinaryField_offset(reinterpret_cast(&val.__v.__s128), + row.setBinaryField_offset(&val.__v.__s128, expression[i]->resultType().colWidth, row.getOffset(expression[i]->outputIndex())); else - row.setBinaryField_offset(reinterpret_cast(&val.__v.__u128), + row.setBinaryField_offset(&val.__v.__u128, expression[i]->resultType().colWidth, row.getOffset(expression[i]->outputIndex())); } diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index da216c177..ce52b8d2e 100755 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -53,6 +53,7 @@ #include "vlarray.h" #include "collation.h" +#include "widedecimalutils.h" //..comment out NDEBUG to enable assertions, uncomment NDEBUG to disable //#define NDEBUG @@ -222,19 +223,8 @@ inline string getStringNullValue() return joblist::CPNULLSTRMARK; } -inline uint64_t getBinaryNullValue() -{ - return joblist::BINARYNULL; } -inline uint64_t getBinaryEmptyValue() -{ - return joblist::BINARYEMPTYROW; -} - -} - - namespace rowgroup { const std::string typeStr(""); @@ -1176,14 +1166,11 @@ void RowAggregation::makeAggFieldsNull(Row& row) } else { - // WIP This is only 1st part of the value - uint64_t nullValue = getBinaryNullValue(); - uint64_t emptyValue = getBinaryEmptyValue(); + int128_t nullValue = 0; + utils::setWideDecimalNullValue(nullValue); uint32_t offset = row.getOffset(colOut); row.setBinaryField_offset(&nullValue, sizeof(nullValue), offset); - row.setBinaryField_offset(&emptyValue, sizeof(nullValue), - offset+sizeof(nullValue)); } break; } diff --git a/utils/rowgroup/rowgroup-tests.cpp b/utils/rowgroup/rowgroup-tests.cpp index a18f8d38b..8fbbe22fc 100644 --- a/utils/rowgroup/rowgroup-tests.cpp +++ b/utils/rowgroup/rowgroup-tests.cpp @@ -91,15 +91,9 @@ class RowDecimalTest : public ::testing::Test { int128_t nullValue = 0; int128_t bigValue = 0; uint64_t* uint128_pod = reinterpret_cast(&nullValue); - uint128_pod[0] = joblist::BINARYNULL; - uint128_pod[1] = joblist::BINARYEMPTYROW; + uint128_pod[0] = joblist::UBIGINTNULL; + uint128_pod[1] = joblist::UBIGINTEMPTYROW; bigValue = -static_cast(0xFFFFFFFF)*0xFFFFFFFFFFFFFFFF; - //char buf[utils::precisionByWidth(16)+3]; - //memset(&buf[0], 0, sizeof(buf)); - //int scale1 = 3; - //dataconvert::DataConvert::decimalToString(&bigValue, scale1, buf, - // utils::precisionByWidth(sizeof(bigValue))+3,types[0]); - //std::cout << buf << std::endl; sValueVector.push_back(nullValue); sValueVector.push_back(-42); sValueVector.push_back(bigValue); @@ -136,8 +130,6 @@ class RowDecimalTest : public ::testing::Test { s64ValueVector.push_back(0x81); s64ValueVector.push_back(joblist::BIGINTNULL-1); - //r.initToNull(); - //r.nextRow(rowSize); for(size_t i = 0; i < sValueVector.size(); i++) { r.setBinaryField_offset(&sValueVector[i], sizeof(sValueVector[0]), offsets[0]); diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 448c8ad5c..3fe84f92f 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -44,6 +44,7 @@ using namespace execplan; #include "rowgroup.h" #include "dataconvert.h" #include "columnwidth.h" +#include "widedecimalutils.h" #include "collation.h" @@ -850,9 +851,7 @@ void Row::initToNull() case 16 : { - uint64_t *dec = reinterpret_cast(&data[offsets[i]]); - dec[0] = joblist::BINARYNULL; - dec[1] = joblist::BINARYEMPTYROW; + utils::setWideDecimalNullValue(reinterpret_cast(data[offsets[i]])); break; } default: @@ -881,9 +880,7 @@ void Row::initToNull() break; case CalpontSystemCatalog::BINARY: { - uint64_t *dec = reinterpret_cast(&data[offsets[i]]); - dec[0] = joblist::BINARYNULL; - dec[1] = joblist::BINARYEMPTYROW; + utils::setWideDecimalNullValue(reinterpret_cast(data[offsets[i]])); } break; @@ -915,11 +912,11 @@ inline bool Row::isNullValue_offset( uint32_t offset) const { - const int64_t *intPtr = reinterpret_cast(&data[offset]); - return ((intPtr[0] == static_cast(joblist::BINARYNULL)) && - (intPtr[1] == static_cast(joblist::BINARYEMPTYROW)) && - (intPtr[2] == static_cast(joblist::BINARYEMPTYROW)) && - (intPtr[3] == static_cast(joblist::BINARYEMPTYROW))); + const uint64_t *intPtr = reinterpret_cast(&data[offset]); + return ((intPtr[0] == static_cast(utils::BINARYNULLVALUELOW)) && + (intPtr[1] == static_cast(utils::BINARYNULLVALUELOW)) && + (intPtr[2] == static_cast(utils::BINARYNULLVALUELOW)) && + (intPtr[3] == static_cast(utils::BINARYEMPTYVALUEHIGH))); } template<> @@ -927,9 +924,8 @@ inline bool Row::isNullValue_offset( uint32_t offset) const { - const int64_t *intPtr = reinterpret_cast(&data[offset]); - return ((intPtr[0] == static_cast(joblist::BINARYNULL)) - && (intPtr[1] == static_cast(joblist::BINARYEMPTYROW))); + const int128_t *intPtr = reinterpret_cast(&data[offset]); + return utils::isWideDecimalNullValue (*intPtr); } template<> @@ -937,9 +933,8 @@ inline bool Row::isNullValue_offset( uint32_t offset) const { - const int64_t *intPtr = reinterpret_cast(&data[offset]); - return ((intPtr[0] == static_cast(joblist::BINARYNULL)) - && (intPtr[1] == static_cast(joblist::BINARYEMPTYROW))); + const int128_t *intPtr = reinterpret_cast(&data[offset]); + return utils::isWideDecimalNullValue (*intPtr); } template<> @@ -1059,8 +1054,7 @@ bool Row::isNullValue(uint32_t colIndex) const case CalpontSystemCatalog::UDECIMAL: { // WIP MCOL-641 Allmighty hack. - int32_t width = getColumnWidth(colIndex); - switch (width) + switch (getColumnWidth(colIndex)) { // MCOL-641 case 16: diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 831fa6d0b..9fa0b98fc 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -814,22 +814,27 @@ inline uint32_t Row::getStringLength(uint32_t colIndex) const memcpy(&data[offset], strdata, length); }*/ -// WIP MCOL-641. This method can be applied to uint8_t* buffers. +// MCOL-641. This method can be applied to uint8_t* buffers. template inline void Row::setBinaryField(T* value, uint32_t width, uint32_t colIndex) { memcpy(&data[offsets[colIndex]], value, width); } -// WIP MCOL-641. This method !cannot! be applied to uint8_t* buffers. +// MCOL-641. This method !cannot! be applied to uint8_t* buffers. template inline void Row::setBinaryField_offset(T* value, uint32_t width, uint32_t offset) { - // WIP Compare performance. - //memcpy(&data[offset], value, width); + // WIP Compare performance. *reinterpret_cast(&data[offset]) = *value; } +template<> +inline void Row::setBinaryField_offset(uint8_t* value, uint32_t width, uint32_t offset) +{ + memcpy(&data[offset], value, width); +} + inline void Row::setStringField(const uint8_t* strdata, uint32_t length, uint32_t colIndex) { uint64_t offset; diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 2eaa1895d..83e75588d 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -51,8 +51,8 @@ using namespace BRM; #include "cacheutils.h" #include "IDBDataFile.h" #include "IDBPolicy.h" - #include "checks.h" +#include "columnwidth.h" namespace WriteEngine { @@ -3000,21 +3000,16 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, case CalpontSystemCatalog::UDECIMAL: { // WIP MCOL-641 - // decimal width > 8 cannot be stored in an integer - if (fetchColColwidths[fetchColPos] > 8) + if (fetchColColwidths[fetchColPos] == 16) { int128_t* dec; - char buf[41]; + char buf[utils::MAXLENGTH16BYTES]; dec = row.getBinaryField(fetchColPos); - dataconvert::DataConvert::decimalToString(dec, + dataconvert::DataConvert::decimalToString(dec, (unsigned)fetchColScales[fetchColPos], buf, sizeof(buf), fetchColTypes[fetchColPos]); - value = buf; - - //value = row.getStringField(fetchColPos); - //unsigned i = strlen(value.c_str()); - //value = value.substr(0, i); + value.assign(buf); break; } @@ -3368,22 +3363,17 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - // WIP MCOL-641 - // decimal width > 8 cannot be stored in an integer - if (fetchColColwidths[fetchColPos] > 8) + if (fetchColColwidths[fetchColPos] == 16) { + // WIP MCOL-641 int128_t* dec; - char buf[41]; + char buf[utils::MAXLENGTH16BYTES]; dec = row.getBinaryField(fetchColPos); - dataconvert::DataConvert::decimalToString(dec, + dataconvert::DataConvert::decimalToString(dec, (unsigned)fetchColScales[fetchColPos], buf, sizeof(buf), fetchColTypes[fetchColPos]); value = buf; - - //value = row.getStringField(fetchColPos); - //unsigned i = strlen(value.c_str()); - //value = value.substr(0, i); break; } diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index f08943a51..7ce12b5a3 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -58,6 +58,7 @@ using namespace execplan; #include "MonitorProcMem.h" using namespace idbdatafile; #include "dataconvert.h" +#include "widedecimalutils.h" #ifdef _MSC_VER #define isnan _isnan @@ -3973,11 +3974,16 @@ void WriteEngineWrapper::printInputValue(const ColStructList& colStructList, curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); else if (curTuple.data.type() == typeid(long long)) curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); + else if (curTuple.data.type() == typeid(int128_t)) + { + // WIP replace with a single call + char buf[utils::MAXLENGTH16BYTES]; + int128_t val = boost::any_cast(curTuple.data); + dataconvert::DataConvert::toString(&val, 0, buf, utils::MAXLENGTH16BYTES); + curStr.assign(buf); + } else if (curTuple.data.type() == typeid(double)) curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); -// else -// if (curTuple.data.type() == typeid(bool)) -// curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); else if (curTuple.data.type() == typeid(short)) curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); else if (curTuple.data.type() == typeid(char)) From 824615a55b5d1588cc23cae6d5fded9e9cab2b22 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Mon, 2 Mar 2020 13:23:07 -0500 Subject: [PATCH 24/78] MCOL-641 Refactor empty value implementation in writeengine. --- dbcon/mysql/ha_mcs_impl.cpp | 4 +- primitives/linux-port/column.cpp | 5 - primitives/primproc/columncommand.cpp | 126 +--- primitives/primproc/columncommand.h | 3 - primitives/primproc/passthrucommand.cpp | 2 +- utils/common/CMakeLists.txt | 3 +- utils/common/emptyvaluemanip.cpp | 115 ++++ utils/common/emptyvaluemanip.h | 31 + utils/common/widedecimalutils.h | 6 +- utils/dataconvert/dataconvert.cpp | 6 + writeengine/bulk/we_bulkload.cpp | 7 +- writeengine/bulk/we_colbuf.cpp | 7 +- writeengine/bulk/we_colbufcompressed.cpp | 8 +- writeengine/bulk/we_columninfo.cpp | 2 +- writeengine/bulk/we_columninfocompressed.cpp | 2 +- writeengine/server/we_dmlcommandproc.cpp | 10 +- writeengine/shared/we_blockop.cpp | 115 +--- writeengine/shared/we_blockop.h | 7 +- writeengine/shared/we_bulkrollbackfile.cpp | 3 +- .../shared/we_bulkrollbackfilecompressed.cpp | 3 +- writeengine/shared/we_chunkmanager.cpp | 5 +- writeengine/shared/we_chunkmanager.h | 2 +- writeengine/shared/we_convertor.cpp | 10 +- writeengine/shared/we_fileop.cpp | 22 +- writeengine/shared/we_fileop.h | 22 +- writeengine/shared/we_type.h | 3 +- writeengine/wrapper/we_colop.cpp | 146 ++--- writeengine/wrapper/we_colop.h | 2 +- writeengine/wrapper/we_colopcompress.cpp | 2 +- writeengine/wrapper/we_colopcompress.h | 2 +- writeengine/wrapper/writeengine.cpp | 549 ++++-------------- 31 files changed, 418 insertions(+), 812 deletions(-) create mode 100644 utils/common/emptyvaluemanip.cpp create mode 100644 utils/common/emptyvaluemanip.h diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 135c5edd9..0529523cc 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -1608,7 +1608,9 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c } // Exit early if there is nothing to update - if (colAssignmentListPtr->empty()) + if (colAssignmentListPtr->empty() && + (((thd->lex)->sql_command == SQLCOM_UPDATE) || + ((thd->lex)->sql_command == SQLCOM_UPDATE_MULTI))) { ci->affectedRows = 0; return 0; diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index b8acd52d9..1161f2cda 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -1537,11 +1537,6 @@ inline void p_Col_ridArray(NewColRequestHeader* in, #endif } - -// WIP MCOL-641 -using uint128_t = unsigned __int128; -using int128_t = __int128; - // for BINARY template inline void p_Col_bin_ridArray(NewColRequestHeader* in, diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index 8a50fd3e0..3dbd2fa61 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -48,6 +48,8 @@ using namespace rowgroup; #include "messageids.h" using namespace logging; +#include "emptyvaluemanip.h" + #ifdef _MSC_VER #define llabs labs #endif @@ -193,28 +195,34 @@ void ColumnCommand::loadData() { if (bPtr && colType.colWidth == 1) { - ByteStream::byte b = getEmptyRowValue(colType.colDataType, colType.colWidth); + ByteStream::byte b; + utils::getEmptyRowValue(colType.colDataType, colType.colWidth, (uint8_t*)&b); bPtr[idx] = b; } //@Bug 1812. Added two bytes column handling else if (dPtr && colType.colWidth == 2) { - ByteStream::doublebyte d = getEmptyRowValue(colType.colDataType, colType.colWidth); + ByteStream::doublebyte d; + utils::getEmptyRowValue(colType.colDataType, colType.colWidth, (uint8_t*)&d); dPtr[idx] = d; } else if (qPtr && colType.colWidth == 4) { - ByteStream::quadbyte q = getEmptyRowValue(colType.colDataType, colType.colWidth); + ByteStream::quadbyte q; + utils::getEmptyRowValue(colType.colDataType, colType.colWidth, (uint8_t*)&q); qPtr[idx] = q; } else if (oPtr && colType.colWidth == 8) { - ByteStream::octbyte o = getEmptyRowValue(colType.colDataType, colType.colWidth); + ByteStream::octbyte o; + utils::getEmptyRowValue(colType.colDataType, colType.colWidth, (uint8_t*)&o); oPtr[idx] = o; } else if (colType.colWidth == 16) { - getEmptyRowValue(colType.colDataType, colType.colWidth, &hPtr[idx]); + ByteStream::hexbyte h; + utils::getEmptyRowValue(colType.colDataType, colType.colWidth, (uint8_t*)&h); + hPtr[idx] = h; } } @@ -959,114 +967,6 @@ void ColumnCommand::enableFilters() prep(primMsg->OutputType, makeAbsRids); } - -/*********************************************************** -* DESCRIPTION: -* Get the value that represents empty row -* PARAMETERS: -* dataType - data type -* width - data width in byte -* RETURN: -* emptyVal - the value of empty row -***********************************************************/ -const uint64_t ColumnCommand::getEmptyRowValue( const CSCDataType dataType, const int width ) const -{ - uint64_t emptyVal = 0; - int offset; - - offset = ( dataType == execplan::CalpontSystemCatalog::VARCHAR ) ? -1 : 0; - - switch ( dataType ) - { - case execplan::CalpontSystemCatalog::TINYINT : - emptyVal = joblist::TINYINTEMPTYROW; - break; - - case execplan::CalpontSystemCatalog::SMALLINT: - emptyVal = joblist::SMALLINTEMPTYROW; - break; - - case execplan::CalpontSystemCatalog::MEDINT : - case execplan::CalpontSystemCatalog::INT : - emptyVal = joblist::INTEMPTYROW; - break; - - case execplan::CalpontSystemCatalog::BIGINT : - emptyVal = joblist::BIGINTEMPTYROW; - break; - - case execplan::CalpontSystemCatalog::UTINYINT : - emptyVal = joblist::UTINYINTEMPTYROW; - break; - - case execplan::CalpontSystemCatalog::USMALLINT: - emptyVal = joblist::USMALLINTEMPTYROW; - break; - - case execplan::CalpontSystemCatalog::UMEDINT : - case execplan::CalpontSystemCatalog::UINT : - emptyVal = joblist::UINTEMPTYROW; - break; - - case execplan::CalpontSystemCatalog::UBIGINT : - emptyVal = joblist::UBIGINTEMPTYROW; - break; - - case execplan::CalpontSystemCatalog::FLOAT : - case execplan::CalpontSystemCatalog::UFLOAT : - emptyVal = joblist::FLOATEMPTYROW; - break; - - case execplan::CalpontSystemCatalog::DOUBLE : - case execplan::CalpontSystemCatalog::UDOUBLE : - emptyVal = joblist::DOUBLEEMPTYROW; - break; - - case execplan::CalpontSystemCatalog::DECIMAL : - case execplan::CalpontSystemCatalog::UDECIMAL : - if ( width <= 1 ) - emptyVal = joblist::TINYINTEMPTYROW; - else if (width <= 2) - emptyVal = joblist::SMALLINTEMPTYROW; - else if ( width <= 4 ) - emptyVal = joblist::INTEMPTYROW; - else - emptyVal = joblist::BIGINTEMPTYROW; - - break; - - case execplan::CalpontSystemCatalog::CHAR : - case execplan::CalpontSystemCatalog::VARCHAR : - case execplan::CalpontSystemCatalog::DATE : - case execplan::CalpontSystemCatalog::DATETIME : - case execplan::CalpontSystemCatalog::TIMESTAMP : - case execplan::CalpontSystemCatalog::TIME : - case execplan::CalpontSystemCatalog::VARBINARY : - case execplan::CalpontSystemCatalog::BLOB : - case execplan::CalpontSystemCatalog::TEXT : - default: - emptyVal = joblist::CHAR1EMPTYROW; - - if ( width == (2 + offset) ) - emptyVal = joblist::CHAR2EMPTYROW; - else if ( width >= (3 + offset) && width <= ( 4 + offset ) ) - emptyVal = joblist::CHAR4EMPTYROW; - else if ( width >= (5 + offset) ) - emptyVal = joblist::CHAR8EMPTYROW; - - break; - } - - return emptyVal; -} - -void ColumnCommand::getEmptyRowValue(const CSCDataType dataType, - const int width, messageqcpp::ByteStream::hexbyte* space) const -{ - int128_t *val = reinterpret_cast(space); - utils::setWideDecimalEMptyValue(*val); -} - void ColumnCommand::getLBIDList(uint32_t loopCount, vector* lbids) { int64_t firstLBID = lbid, lastLBID = firstLBID + (loopCount * colType.colWidth) - 1, i; diff --git a/primitives/primproc/columncommand.h b/primitives/primproc/columncommand.h index 52544021f..c1db001b0 100644 --- a/primitives/primproc/columncommand.h +++ b/primitives/primproc/columncommand.h @@ -84,9 +84,6 @@ public: makeAbsRids = m; } bool willPrefetch(); - const uint64_t getEmptyRowValue( const CSCDataType dataType, const int width ) const; - void getEmptyRowValue(const CSCDataType dataType, - const int width, messageqcpp::ByteStream::hexbyte* space) const; const int64_t getLastLbid(); void getLBIDList(uint32_t loopCount, std::vector* lbids); diff --git a/primitives/primproc/passthrucommand.cpp b/primitives/primproc/passthrucommand.cpp index 7e362b1a6..6c7a0336f 100644 --- a/primitives/primproc/passthrucommand.cpp +++ b/primitives/primproc/passthrucommand.cpp @@ -167,7 +167,7 @@ void PassThruCommand::projectIntoRowGroup(RowGroup& rg, uint32_t col) << *(((int64_t*) bpp->values[i]) +1) << endl; // values[i] is 8 bytes so it contains the pointer to bpp->outputMsg set by ColumnCommand::process_OT_BOTH() - r.setBinaryField_offset((uint8_t*)bpp->values[i], 16, offset); + r.setBinaryField_offset((uint128_t*)bpp->values[i], 16, offset); r.nextRow(rowSize); } diff --git a/utils/common/CMakeLists.txt b/utils/common/CMakeLists.txt index c6a0fde01..e3dd4607e 100644 --- a/utils/common/CMakeLists.txt +++ b/utils/common/CMakeLists.txt @@ -10,7 +10,8 @@ set(common_LIB_SRCS MonitorProcMem.cpp nullvaluemanip.cpp threadnaming.cpp - utils_utf8.cpp) + utils_utf8.cpp + emptyvaluemanip.cpp) add_library(common SHARED ${common_LIB_SRCS}) diff --git a/utils/common/emptyvaluemanip.cpp b/utils/common/emptyvaluemanip.cpp new file mode 100644 index 000000000..fc97d2f14 --- /dev/null +++ b/utils/common/emptyvaluemanip.cpp @@ -0,0 +1,115 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include "widedecimalutils.h" +#include "emptyvaluemanip.h" + +namespace utils +{ + +void getEmptyRowValue(const execplan::CalpontSystemCatalog::ColDataType colDataType, + const int width, uint8_t* emptyVal) +{ + switch (colDataType) + { + case execplan::CalpontSystemCatalog::TINYINT: + *(uint8_t*)emptyVal = joblist::TINYINTEMPTYROW; + break; + + case execplan::CalpontSystemCatalog::SMALLINT: + *(uint16_t*)emptyVal = joblist::SMALLINTEMPTYROW; + break; + + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::INT: + *(uint32_t*)emptyVal = joblist::INTEMPTYROW; + break; + + case execplan::CalpontSystemCatalog::BIGINT: + *(uint64_t*)emptyVal = joblist::BIGINTEMPTYROW; + break; + + case execplan::CalpontSystemCatalog::UTINYINT: + *(uint8_t*)emptyVal = joblist::UTINYINTEMPTYROW; + break; + + case execplan::CalpontSystemCatalog::USMALLINT: + *(uint16_t*)emptyVal = joblist::USMALLINTEMPTYROW; + break; + + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UINT: + *(uint32_t*)emptyVal = joblist::UINTEMPTYROW; + break; + + case execplan::CalpontSystemCatalog::UBIGINT: + *(uint64_t*)emptyVal = joblist::UBIGINTEMPTYROW; + break; + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + *(uint32_t*)emptyVal = joblist::FLOATEMPTYROW; + break; + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + *(uint64_t*)emptyVal = joblist::DOUBLEEMPTYROW; + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + if (width <= 1) + *(uint8_t*)emptyVal = joblist::TINYINTEMPTYROW; + else if (width <= 2) + *(uint16_t*)emptyVal = joblist::SMALLINTEMPTYROW; + else if (width <= 4) + *(uint32_t*)emptyVal = joblist::INTEMPTYROW; + else if (width <= 8) + *(uint64_t*)emptyVal = joblist::BIGINTEMPTYROW; + else + setWideDecimalEmptyValue(*(reinterpret_cast(emptyVal))); + break; + + //case CalpontSystemCatalog::BINARY: + // emptyVal = joblist::BINARYEMPTYROW; + // break; + + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::DATE: + case execplan::CalpontSystemCatalog::DATETIME: + case execplan::CalpontSystemCatalog::TIMESTAMP: + case execplan::CalpontSystemCatalog::TIME: + case execplan::CalpontSystemCatalog::VARBINARY: + case execplan::CalpontSystemCatalog::BLOB: + case execplan::CalpontSystemCatalog::TEXT: + default: + *(uint8_t*)emptyVal = joblist::CHAR1EMPTYROW; + int offset = (colDataType == execplan::CalpontSystemCatalog::VARCHAR) ? -1 : 0; + + if (width == (2 + offset)) + *(uint16_t*)emptyVal = joblist::CHAR2EMPTYROW; + else if (width >= (3 + offset) && width <= (4 + offset)) + *(uint32_t*)emptyVal = joblist::CHAR4EMPTYROW; + else if (width >= (5 + offset)) + *(uint64_t*)emptyVal = joblist::CHAR8EMPTYROW; + + break; + } +} + +} // namespace utils diff --git a/utils/common/emptyvaluemanip.h b/utils/common/emptyvaluemanip.h new file mode 100644 index 000000000..2cf6222de --- /dev/null +++ b/utils/common/emptyvaluemanip.h @@ -0,0 +1,31 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#ifndef EMPTY_VALUE_MANIP_H +#define EMPTY_VALUE_MANIP_H + +#include "calpontsystemcatalog.h" + +namespace utils +{ + +void getEmptyRowValue(const execplan::CalpontSystemCatalog::ColDataType colDataType, + const int width, uint8_t* emptyVal); + +} // namespace utils + +#endif // EMPTY_VALUE_MANIP_H diff --git a/utils/common/widedecimalutils.h b/utils/common/widedecimalutils.h index 67302f2bd..7ae3d0ebd 100644 --- a/utils/common/widedecimalutils.h +++ b/utils/common/widedecimalutils.h @@ -18,6 +18,8 @@ #ifndef WIDE_DECIMAL_UTILS_H #define WIDE_DECIMAL_UTILS_H +#include + using int128_t = __int128; using uint128_t = unsigned __int128; @@ -48,7 +50,7 @@ namespace utils ptr[1] = BINARYNULLVALUEHIGH; } - inline void setWideDecimalEMptyValue(int128_t& val) + inline void setWideDecimalEmptyValue(int128_t& val) { uint64_t* ptr = reinterpret_cast(&val); ptr[0] = BINARYEMPTYVALUELOW; @@ -62,7 +64,7 @@ namespace utils ptr[1] = BINARYNULLVALUEHIGH; } - inline void setWideDecimalEMptyValue(int128_t* val) + inline void setWideDecimalEmptyValue(int128_t* val) { uint64_t* ptr = reinterpret_cast(val); ptr[0] = BINARYEMPTYVALUELOW; diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 42f4295fa..dac9e521e 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1920,6 +1920,12 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, long long eightbyte = joblist::BIGINTNULL; value = eightbyte; } + else if (colType.colWidth == 16) + { + int128_t val; + utils::setWideDecimalNullValue(val); + value = val; + } else { WriteEngine::Token nullToken; diff --git a/writeengine/bulk/we_bulkload.cpp b/writeengine/bulk/we_bulkload.cpp index e51b51237..0b4c4abb5 100644 --- a/writeengine/bulk/we_bulkload.cpp +++ b/writeengine/bulk/we_bulkload.cpp @@ -563,9 +563,10 @@ int BulkLoad::preProcess( Job& job, int tableNo, job.jobTableList[tableNo].colList[i].weType = curColStruct.colType; // set width to correct column width job.jobTableList[tableNo].colList[i].width = curColStruct.colWidth; - job.jobTableList[tableNo].colList[i].emptyVal = getEmptyRowValue( - job.jobTableList[tableNo].colList[i].dataType, - job.jobTableList[tableNo].colList[i].width ); + getEmptyRowValue( + job.jobTableList[tableNo].colList[i].dataType, + job.jobTableList[tableNo].colList[i].width, + (uint8_t*)&job.jobTableList[tableNo].colList[i].emptyVal); // check HWM for column file rc = BRMWrapper::getInstance()->getDbRootHWMInfo( curJobCol.mapOid, diff --git a/writeengine/bulk/we_colbuf.cpp b/writeengine/bulk/we_colbuf.cpp index 7be21a6ec..caa4f8535 100644 --- a/writeengine/bulk/we_colbuf.cpp +++ b/writeengine/bulk/we_colbuf.cpp @@ -115,12 +115,13 @@ int ColumnBuffer::writeToFile(int startOffset, int writeSize, bool fillUpWEmptie { BlockOp blockOp; newBuf = new unsigned char[BYTE_PER_BLOCK]; - uint64_t EmptyValue = blockOp.getEmptyRowValue(fColInfo->column.dataType, - fColInfo->column.width); + uint8_t* emptyVal = (uint8_t*) alloca(fColInfo->column.width); + blockOp.getEmptyRowValue(fColInfo->column.dataType, + fColInfo->column.width, emptyVal); ::memcpy(static_cast(newBuf), static_cast(fBuffer + startOffset), writeSize); blockOp.setEmptyBuf(newBuf + writeSize, BYTE_PER_BLOCK - writeSize, - EmptyValue, fColInfo->column.width); + emptyVal, fColInfo->column.width); } #ifdef PROFILE Stats::startParseEvent(WE_STATS_WRITE_COL); diff --git a/writeengine/bulk/we_colbufcompressed.cpp b/writeengine/bulk/we_colbufcompressed.cpp index 43534dc75..0e359e885 100644 --- a/writeengine/bulk/we_colbufcompressed.cpp +++ b/writeengine/bulk/we_colbufcompressed.cpp @@ -132,7 +132,7 @@ int ColumnBufferCompressed::resetToBeCompressedColBuf( BlockOp::setEmptyBuf( fToBeCompressedBuffer, IDBCompressInterface::UNCOMPRESSED_INBUF_LEN, - fColInfo->column.emptyVal, + (uint8_t*)&fColInfo->column.emptyVal, fColInfo->column.width ); if (fLog->isDebug( DEBUG_2 )) @@ -317,7 +317,7 @@ int ColumnBufferCompressed::writeToFile(int startOffset, int writeSize, // Start over again loading a new to-be-compressed buffer BlockOp::setEmptyBuf( fToBeCompressedBuffer, IDBCompressInterface::UNCOMPRESSED_INBUF_LEN, - fColInfo->column.emptyVal, + (uint8_t*)&fColInfo->column.emptyVal, fColInfo->column.width ); fToBeCompressedCapacity = @@ -628,7 +628,7 @@ int ColumnBufferCompressed::initToBeCompressedBuffer(long long& startFileOffset) new unsigned char[IDBCompressInterface::UNCOMPRESSED_INBUF_LEN]; BlockOp::setEmptyBuf( fToBeCompressedBuffer, IDBCompressInterface::UNCOMPRESSED_INBUF_LEN, - fColInfo->column.emptyVal, + (uint8_t*)&fColInfo->column.emptyVal, fColInfo->column.width ); bNewBuffer = true; } @@ -743,7 +743,7 @@ int ColumnBufferCompressed::initToBeCompressedBuffer(long long& startFileOffset) { BlockOp::setEmptyBuf( fToBeCompressedBuffer, IDBCompressInterface::UNCOMPRESSED_INBUF_LEN, - fColInfo->column.emptyVal, + (uint8_t*)&fColInfo->column.emptyVal, fColInfo->column.width ); } diff --git a/writeengine/bulk/we_columninfo.cpp b/writeengine/bulk/we_columninfo.cpp index 5a9e4afa0..f0e40fb9d 100644 --- a/writeengine/bulk/we_columninfo.cpp +++ b/writeengine/bulk/we_columninfo.cpp @@ -895,7 +895,7 @@ int ColumnInfo::extendColumnOldExtent( } rc = colOp->expandAbbrevColumnExtent( pFile, dbRootNext, - column.emptyVal, column.width); + (uint8_t*)&column.emptyVal, column.width); if (rc != NO_ERROR) { diff --git a/writeengine/bulk/we_columninfocompressed.cpp b/writeengine/bulk/we_columninfocompressed.cpp index 25af8bc22..4fdd7f7be 100644 --- a/writeengine/bulk/we_columninfocompressed.cpp +++ b/writeengine/bulk/we_columninfocompressed.cpp @@ -540,7 +540,7 @@ int ColumnInfoCompressed::extendColumnOldExtent( int rc = colOp->fillCompColumnExtentEmptyChunks( curCol.dataFile.fid, curCol.colWidth, - column.emptyVal, + (uint8_t*)&column.emptyVal, curCol.dataFile.fDbRoot, curCol.dataFile.fPartition, curCol.dataFile.fSegment, diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 83e75588d..ff274d677 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -370,8 +370,6 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: try { - // WIP - // make convertColumnData a template datavalue = DataConvert::convertColumnData(colType, indata, pushWarning, insertPkg.get_TimeZone(), isNULL, false, false); } catch (exception&) @@ -388,7 +386,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: return rc; } - if ( pushWarning) + if (pushWarning) { if (!isWarningSet) isWarningSet = true; @@ -546,7 +544,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: WErrorCodes ec; err = ec.errorString(error); } - else if ( error == ERR_BRM_VB_OVERFLOW ) + else if (error == ERR_BRM_VB_OVERFLOW) { rc = dmlpackageprocessor::DMLPackageProcessor::VB_OVERFLOW_ERROR; err = IDBErrorInfo::instance()->errorMsg(ERR_VERSIONBUFFER_OVERFLOW); @@ -561,9 +559,9 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: } std::map oids; - std::vector oidsToFlush; + std::vector oidsToFlush; - for ( unsigned i = 0; i < colStructs.size(); i++) + for (unsigned i = 0; i < colStructs.size(); i++) { oids[colStructs[i].dataOid] = colStructs[i].dataOid; oidsToFlush.push_back(colStructs[i].dataOid); diff --git a/writeengine/shared/we_blockop.cpp b/writeengine/shared/we_blockop.cpp index 01a8cd6f1..9cb1700eb 100644 --- a/writeengine/shared/we_blockop.cpp +++ b/writeengine/shared/we_blockop.cpp @@ -34,6 +34,8 @@ using namespace execplan; +#include "emptyvaluemanip.h" + namespace WriteEngine { @@ -83,110 +85,10 @@ bool BlockOp::calculateRowId( * emptyVal - the value of empty row ***********************************************************/ // TODO MCOL-641 Add support here -uint64_t BlockOp::getEmptyRowValue( - const CalpontSystemCatalog::ColDataType colDataType, const int width ) const +void BlockOp::getEmptyRowValue( + const CalpontSystemCatalog::ColDataType colDataType, const int width, uint8_t* emptyVal ) const { - uint64_t emptyVal = 0; - int offset = 0; - - switch ( colDataType ) - { - case CalpontSystemCatalog::TINYINT : - emptyVal = joblist::TINYINTEMPTYROW; - break; - - case CalpontSystemCatalog::SMALLINT: - emptyVal = joblist::SMALLINTEMPTYROW; - break; - - case CalpontSystemCatalog::MEDINT : - case CalpontSystemCatalog::INT : - emptyVal = joblist::INTEMPTYROW; - break; - - case CalpontSystemCatalog::BIGINT : - emptyVal = joblist::BIGINTEMPTYROW; - break; - - case CalpontSystemCatalog::FLOAT : - case CalpontSystemCatalog::UFLOAT : - emptyVal = joblist::FLOATEMPTYROW; - break; - - case CalpontSystemCatalog::DOUBLE : - case CalpontSystemCatalog::UDOUBLE : - emptyVal = joblist::DOUBLEEMPTYROW; - break; - - case CalpontSystemCatalog::DECIMAL : - case CalpontSystemCatalog::UDECIMAL : - - /* if( width <= 4 ) - emptyVal = joblist::SMALLINTEMPTYROW; - else - if( width <= 9 ) - emptyVal = 0x80000001; - else - if( width <= 18 ) - emptyVal = 0x8000000000000001LL; - else - emptyVal = 0xFFFFFFFFFFFFFFFFLL; - */ - // @bug 194 use the correct logic in handling empty value for decimal - if (width <= 1) - emptyVal = joblist::TINYINTEMPTYROW; - else if ( width <= 2 ) - emptyVal = joblist::SMALLINTEMPTYROW; - else if ( width <= 4 ) - emptyVal = joblist::INTEMPTYROW; - else if ( width <= 8 ) - emptyVal = joblist::BIGINTEMPTYROW; - else - emptyVal = joblist::BINARYEMPTYROW; - - break; - - case CalpontSystemCatalog::UTINYINT : - emptyVal = joblist::UTINYINTEMPTYROW; - break; - - case CalpontSystemCatalog::USMALLINT: - emptyVal = joblist::USMALLINTEMPTYROW; - break; - - case CalpontSystemCatalog::UMEDINT : - case CalpontSystemCatalog::UINT : - emptyVal = joblist::UINTEMPTYROW; - break; - - case CalpontSystemCatalog::UBIGINT : - emptyVal = joblist::UBIGINTEMPTYROW; - break; - - case CalpontSystemCatalog::BINARY : - emptyVal = joblist::BINARYEMPTYROW; - break; - - case CalpontSystemCatalog::CHAR : - case CalpontSystemCatalog::VARCHAR : - case CalpontSystemCatalog::DATE : - case CalpontSystemCatalog::DATETIME : - case CalpontSystemCatalog::TIMESTAMP : - default: - offset = ( colDataType == CalpontSystemCatalog::VARCHAR ) ? -1 : 0; - emptyVal = joblist::CHAR1EMPTYROW; - - if ( width == (2 + offset) ) - emptyVal = joblist::CHAR2EMPTYROW; - else if ( width >= (3 + offset) && width <= ( 4 + offset ) ) - emptyVal = joblist::CHAR4EMPTYROW; - else if ( width >= (5 + offset) ) - emptyVal = joblist::CHAR8EMPTYROW; - - break; - } - - return emptyVal; + utils::getEmptyRowValue(colDataType, width, emptyVal); } /*********************************************************** @@ -264,7 +166,7 @@ void BlockOp::resetBuf( unsigned char* buf, const int bufSize ) const ***********************************************************/ /* static */ void BlockOp::setEmptyBuf( - unsigned char* buf, const int bufSize, uint64_t emptyVal, const int width ) + unsigned char* buf, const int bufSize, uint8_t* emptyVal, const int width ) { const int ARRAY_COUNT = 128; const int NBYTES_IN_ARRAY = width * ARRAY_COUNT; @@ -275,10 +177,9 @@ void BlockOp::setEmptyBuf( // instead of individual values. This reduces the number of calls to // memcpy(). - int w = width > 8 ? 8: width; - for(uint8_t* pos = emptyValArray, * end = pos + NBYTES_IN_ARRAY; pos < end; pos += w) //FIXME for no loop + for(uint8_t* pos = emptyValArray, * end = pos + NBYTES_IN_ARRAY; pos < end; pos += width) //FIXME for no loop { - memcpy(pos, &emptyVal, w); + memcpy(pos, emptyVal, width); } int countFull128 = (bufSize / width) / ARRAY_COUNT; diff --git a/writeengine/shared/we_blockop.h b/writeengine/shared/we_blockop.h index eb88c5e38..10df5a8bb 100644 --- a/writeengine/shared/we_blockop.h +++ b/writeengine/shared/we_blockop.h @@ -89,8 +89,9 @@ public: /** * @brief Get an empty row value */ - EXPORT uint64_t getEmptyRowValue(const execplan::CalpontSystemCatalog::ColDataType colDataType, - const int width ) const; + EXPORT void getEmptyRowValue(const execplan::CalpontSystemCatalog::ColDataType colDataType, + const int width, + uint8_t* emptyVal ) const; /** * @brief Calculate row id @@ -116,7 +117,7 @@ public: */ EXPORT void static setEmptyBuf( unsigned char* buf, const int bufSize, - uint64_t emptyVal, const int width ); + uint8_t* emptyVal, const int width ); /** * @brief Set a value in a buffer diff --git a/writeengine/shared/we_bulkrollbackfile.cpp b/writeengine/shared/we_bulkrollbackfile.cpp index fbf4dfba7..ac4cdb4ee 100644 --- a/writeengine/shared/we_bulkrollbackfile.cpp +++ b/writeengine/shared/we_bulkrollbackfile.cpp @@ -306,7 +306,8 @@ void BulkRollbackFile::reInitTruncColumnExtent( } // Initialize the remainder of the extent after the HWM block - uint64_t emptyVal = fDbFile.getEmptyRowValue( colType, colWidth ); + uint8_t* emptyVal = (uint8_t*) alloca(colWidth); + fDbFile.getEmptyRowValue( colType, colWidth, emptyVal ); int rc = fDbFile.reInitPartialColumnExtent( pFile, startOffset, diff --git a/writeengine/shared/we_bulkrollbackfilecompressed.cpp b/writeengine/shared/we_bulkrollbackfilecompressed.cpp index 08954411d..4954930af 100644 --- a/writeengine/shared/we_bulkrollbackfilecompressed.cpp +++ b/writeengine/shared/we_bulkrollbackfilecompressed.cpp @@ -374,7 +374,8 @@ void BulkRollbackFileCompressed::reInitTruncColumnExtent( if (nBlocksToInit > 0) { - uint64_t emptyVal = fDbFile.getEmptyRowValue( colType, colWidth ); + uint8_t* emptyVal = (uint8_t*) alloca(colWidth); + fDbFile.getEmptyRowValue( colType, colWidth, emptyVal ); rc = fDbFile.reInitPartialColumnExtent( pFile, (chunkPtrs[chunkIndex].first + restoredChunkLen), nBlocksToInit, diff --git a/writeengine/shared/we_chunkmanager.cpp b/writeengine/shared/we_chunkmanager.cpp index 75a8feff8..a435ce1d4 100644 --- a/writeengine/shared/we_chunkmanager.cpp +++ b/writeengine/shared/we_chunkmanager.cpp @@ -821,7 +821,8 @@ int ChunkManager::fetchChunkFromFile(IDBDataFile* pFile, int64_t id, ChunkData*& void ChunkManager::initializeColumnChunk(char* buf, CompFileData* fileData) { int size = UNCOMPRESSED_CHUNK_SIZE; - uint64_t emptyVal = fFileOp->getEmptyRowValue(fileData->fColDataType, fileData->fColWidth); + uint8_t* emptyVal = (uint8_t*) alloca(fileData->fColWidth); + fFileOp->getEmptyRowValue(fileData->fColDataType, fileData->fColWidth, emptyVal); fFileOp->setEmptyBuf((unsigned char*)buf, size, emptyVal, fileData->fColWidth); } @@ -1342,7 +1343,7 @@ inline int ChunkManager::writeHeader_(CompFileData* fileData, int ptrSecSize) // For the specified segment file (pFile), read in an abbreviated/compressed // chunk extent, uncompress, and expand to a full chunk for a full extent. //------------------------------------------------------------------------------ -int ChunkManager::expandAbbrevColumnExtent(IDBDataFile* pFile, uint64_t emptyVal, int width) +int ChunkManager::expandAbbrevColumnExtent(IDBDataFile* pFile, uint8_t* emptyVal, int width) { map::iterator i = fFilePtrMap.find(pFile); diff --git a/writeengine/shared/we_chunkmanager.h b/writeengine/shared/we_chunkmanager.h index edf9b232f..6122537e3 100644 --- a/writeengine/shared/we_chunkmanager.h +++ b/writeengine/shared/we_chunkmanager.h @@ -214,7 +214,7 @@ public: void cleanUp(const std::map& columOids); // @brief Expand an initial column, not dictionary, extent to a full extent. - int expandAbbrevColumnExtent(IDBDataFile* pFile, uint64_t emptyVal, int width); + int expandAbbrevColumnExtent(IDBDataFile* pFile, uint8_t* emptyVal, int width); // @brief Update column extent int updateColumnExtent(IDBDataFile* pFile, int addBlockCount); diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index 9c5c63454..77fa8128d 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -639,8 +639,6 @@ void Convertor::convertColType(ColStruct* curStruct) break; default: - // WIP replace with BINARY - //*internalType = WriteEngine::WR_INT128; *internalType = WriteEngine::WR_BINARY; break; } @@ -710,14 +708,8 @@ void Convertor::convertColType(ColStruct* curStruct) // check whether width is in sync with the requirement *width = getCorrectRowWidth(dataType, *width); - - // This is the patch for the decimal thing, override -// if (dataType == CalpontSystemCatalog::DECIMAL) -// { -// *internalType = *width <= 4 ? -// WriteEngine::WR_INT : WriteEngine::WR_LONGLONG; -// } } + /******************************************************************************* * DESCRIPTION: diff --git a/writeengine/shared/we_fileop.cpp b/writeengine/shared/we_fileop.cpp index fe2eb2710..23e285e47 100644 --- a/writeengine/shared/we_fileop.cpp +++ b/writeengine/shared/we_fileop.cpp @@ -161,7 +161,7 @@ int FileOp::createDir( const char* dirName, mode_t mode ) const * ERR_FILE_CREATE if can not create the file ***********************************************************/ int FileOp::createFile( const char* fileName, int numOfBlock, - uint64_t emptyVal, int width, + uint8_t* emptyVal, int width, uint16_t dbRoot ) { IDBDataFile* pFile = @@ -228,7 +228,7 @@ int FileOp::createFile(FID fid, uint16_t dbRoot, uint32_t partition, execplan::CalpontSystemCatalog::ColDataType colDataType, - uint64_t emptyVal, + uint8_t* emptyVal, int width) { //std::cout << "Creating file oid: " << fid << @@ -569,7 +569,7 @@ bool FileOp::existsOIDDir( FID fid ) const ***********************************************************/ int FileOp::extendFile( OID oid, - uint64_t emptyVal, + uint8_t* emptyVal, int width, HWM hwm, BRM::LBID_t startLbid, @@ -875,7 +875,7 @@ int FileOp::extendFile( ***********************************************************/ int FileOp::addExtentExactFile( OID oid, - uint64_t emptyVal, + uint8_t* emptyVal, int width, int& allocSize, uint16_t dbRoot, @@ -1045,7 +1045,7 @@ int FileOp::initColumnExtent( IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, - uint64_t emptyVal, + uint8_t* emptyVal, int width, bool bNewFile, bool bExpandExtent, @@ -1225,7 +1225,7 @@ int FileOp::initAbbrevCompColumnExtent( IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, - uint64_t emptyVal, + uint8_t* emptyVal, int width) { // Reserve disk space for optimized abbreviated extent @@ -1285,7 +1285,7 @@ int FileOp::writeInitialCompColumnChunk( IDBDataFile* pFile, int nBlocksAllocated, int nRows, - uint64_t emptyVal, + uint8_t* emptyVal, int width, char* hdrs) { @@ -1366,7 +1366,7 @@ int FileOp::writeInitialCompColumnChunk( ***********************************************************/ int FileOp::fillCompColumnExtentEmptyChunks(OID oid, int colWidth, - uint64_t emptyVal, + uint8_t* emptyVal, uint16_t dbRoot, uint32_t partition, uint16_t segment, @@ -1671,7 +1671,7 @@ int FileOp::fillCompColumnExtentEmptyChunks(OID oid, ***********************************************************/ int FileOp::expandAbbrevColumnChunk( IDBDataFile* pFile, - uint64_t emptyVal, + uint8_t* emptyVal, int colWidth, const CompChunkPtr& chunkInPtr, CompChunkPtr& chunkOutPtr ) @@ -2036,7 +2036,7 @@ int FileOp::reInitPartialColumnExtent( IDBDataFile* pFile, long long startOffset, int nBlocks, - uint64_t emptyVal, + uint8_t* emptyVal, int width ) { int rc = setFileOffset( pFile, startOffset, SEEK_SET ); @@ -2845,7 +2845,7 @@ bool FileOp::isDiskSpaceAvail(const std::string& fileName, int nBlocks) const int FileOp::expandAbbrevColumnExtent( IDBDataFile* pFile, // FILE ptr to file where abbrev extent is to be expanded uint16_t dbRoot, // The DBRoot of the file with the abbreviated extent - uint64_t emptyVal,// Empty value to be used in expanding the extent + uint8_t* emptyVal,// Empty value to be used in expanding the extent int width ) // Width of the column (in bytes) { // Based on extent size, see how many blocks to add to fill the extent diff --git a/writeengine/shared/we_fileop.h b/writeengine/shared/we_fileop.h index 0b7d094e4..0615eaef8 100644 --- a/writeengine/shared/we_fileop.h +++ b/writeengine/shared/we_fileop.h @@ -92,7 +92,7 @@ public: int& allocSize, uint16_t dbRoot, uint32_t partition, execplan::CalpontSystemCatalog::ColDataType colDataType, - uint64_t emptyVal = 0, int width = 1 ) ; + uint8_t* emptyVal, int width = 1 ) ; /** @@ -100,7 +100,7 @@ public: * Changed to public for UT. */ int createFile( const char* fileName, int fileSize, - uint64_t emptyVal, int width, + uint8_t* emptyVal, int width, uint16_t dbRoot ); /** @@ -163,7 +163,7 @@ public: EXPORT virtual int expandAbbrevColumnExtent( IDBDataFile* pFile, uint16_t dbRoot, - uint64_t emptyVal, + uint8_t* emptyVal, int width ); /** @@ -198,7 +198,7 @@ public: * @param hdrs (in/out) Contents of headers, if file is compressed. * @return returns NO_ERROR if success. */ - EXPORT int extendFile(OID oid, uint64_t emptyVal, + EXPORT int extendFile(OID oid, uint8_t* emptyVal, int width, HWM hwm, BRM::LBID_t startLbid, @@ -226,7 +226,7 @@ public: * @param newFile (out) Indicates if a new file was created for the extent * @param hdrs (in/out) Contents of headers, if file is compressed. */ - EXPORT int addExtentExactFile(OID oid, uint64_t emptyVal, + EXPORT int addExtentExactFile(OID oid, uint8_t* emptyVal, int width, int& allocSize, uint16_t dbRoot, @@ -253,7 +253,7 @@ public: */ EXPORT int fillCompColumnExtentEmptyChunks(OID oid, int colWidth, - uint64_t emptyVal, + uint8_t* emptyVal, uint16_t dbRoot, uint32_t partition, uint16_t segment, @@ -433,7 +433,7 @@ public: EXPORT int reInitPartialColumnExtent( IDBDataFile* pFile, long long startOffset, int nBlocks, - uint64_t emptyVal, + uint8_t* emptyVal, int width ); /** @@ -497,7 +497,7 @@ public: int initColumnExtent( IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, - uint64_t emptyVal, + uint8_t* emptyVal, int width, bool bNewFile, bool bExpandExtent, @@ -519,7 +519,7 @@ private: FileOp& operator=(const FileOp& rhs); int expandAbbrevColumnChunk( IDBDataFile* pFile, - uint64_t emptyVal, + uint8_t* emptyVal, int colWidth, const compress::CompChunkPtr& chunkInPtr, compress::CompChunkPtr& chunkOutPt); @@ -527,7 +527,7 @@ private: int initAbbrevCompColumnExtent( IDBDataFile* pFile, uint16_t dbRoot, int nBlocks, - uint64_t emptyVal, + uint8_t* emptyVal, int width); static void initDbRootExtentMutexes(); @@ -536,7 +536,7 @@ private: int writeInitialCompColumnChunk( IDBDataFile* pFile, int nBlocksAllocated, int nRows, - uint64_t emptyVal, + uint8_t* emptyVal, int width, char* hdrs); diff --git a/writeengine/shared/we_type.h b/writeengine/shared/we_type.h index c2b20a1af..ba77b9409 100644 --- a/writeengine/shared/we_type.h +++ b/writeengine/shared/we_type.h @@ -57,6 +57,7 @@ typedef uint32_t FID; /** @brief File ID */ typedef uint64_t RID; /** @brief Row ID */ typedef uint32_t TxnID; /** @brief Transaction ID (New)*/ typedef uint32_t HWM; /** @brief high water mark */ +typedef unsigned __int128 uint128_t; /************************************************************************ * Type enumerations @@ -347,7 +348,7 @@ struct JobColumn /** @brief Job Column Structure */ execplan::CalpontSystemCatalog::ColDataType dataType; /** @brief column data type */ ColType weType; /** @brief write engine data type */ std::string typeName; /** @brief data type name */ - uint64_t emptyVal; /** @brief default empty value */ + uint128_t emptyVal; /** @brief default empty value */ int width; /** @brief column width; for a dictionary column, this is "eventually" the token width */ int definedWidth; /** @brief column width as defined in the table, used for non-dictionary strings */ int dctnryWidth; /** @brief dictionary width */ diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index de6119752..a997f0999 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -45,6 +45,7 @@ using namespace execplan; using namespace idbdatafile; +#include "emptyvaluemanip.h" namespace WriteEngine { @@ -54,6 +55,9 @@ struct RefcolInfo unsigned numExtents; }; +using int128_t = __int128; +using uint128_t = unsigned __int128; + /** * Constructor */ @@ -126,10 +130,8 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, Column newCol; unsigned char buf[BYTE_PER_BLOCK]; unsigned char* curVal; - int64_t emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); // Seems is ok have it here and just once - // TODO MCOL-641 consolidate the emptyvalue logic - //__int128 bigEmptyVal; - //dataconvert::DataConvert::uint128Max(bigEmptyVal); + uint8_t* emptyVal = (uint8_t*) alloca(column.colWidth); + getEmptyRowValue(column.colDataType, column.colWidth, emptyVal); if (useStartingExtent) { @@ -190,10 +192,12 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, { if (rc == ERR_FILE_EOF) { - uint64_t emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); + uint8_t* emptyVal = (uint8_t*) alloca(column.colWidth); + getEmptyRowValue(column.colDataType, column.colWidth, emptyVal); setEmptyBuf(buf, BYTE_PER_BLOCK, emptyVal, column.colWidth); RETURN_ON_ERROR(saveBlock(column.dataFile.pFile, buf, hwm)); - } else + } + else { return rc; } @@ -287,7 +291,8 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, if (newColStructList[i].fCompressionType > 0) { - uint64_t emptyVal = getEmptyRowValue(newColStructList[i].colDataType, newColStructList[i].colWidth); + uint8_t* emptyVal = (uint8_t*) alloca(newColStructList[i].colWidth); + getEmptyRowValue(newColStructList[i].colDataType, newColStructList[i].colWidth, emptyVal); string errorInfo; rc = fileOp.fillCompColumnExtentEmptyChunks(newColStructList[i].dataOid, newColStructList[i].colWidth, emptyVal, dbRoot, partition, segment, newHwm, segFile, errorInfo); @@ -313,7 +318,8 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, return rc; } - uint64_t emptyVal = getEmptyRowValue(newColStructList[i].colDataType, newColStructList[i].colWidth); + uint8_t* emptyVal = (uint8_t*) alloca(newColStructList[i].colWidth); + getEmptyRowValue(newColStructList[i].colDataType, newColStructList[i].colWidth, emptyVal); rc = fileOp.expandAbbrevColumnExtent( pFile, dbRoot, emptyVal, newColStructList[i].colWidth); //set hwm for this extent. fileOp.closeFile(pFile); @@ -385,7 +391,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, return rc; //create corresponding dictionary files - if (newFile ) + if (newFile) { boost::scoped_ptr we (new WriteEngineWrapper()); we->setTransId(txnid); @@ -406,7 +412,7 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, } } - we->flushDataFiles(rc, txnid, columnOids ); + we->flushDataFiles(rc, txnid, columnOids); } } @@ -490,10 +496,12 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, { if (rc == ERR_FILE_EOF) { - uint64_t emptyVal = getEmptyRowValue(newCol.colDataType, newCol.colWidth); + uint8_t* emptyVal = (uint8_t*) alloca(newCol.colWidth); + getEmptyRowValue(newCol.colDataType, newCol.colWidth, emptyVal); setEmptyBuf(buf, BYTE_PER_BLOCK, emptyVal, newCol.colWidth); RETURN_ON_ERROR(saveBlock(newCol.dataFile.pFile, buf, newHwm)); - } else + } + else { return rc; } @@ -531,10 +539,12 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent, { if (rc == ERR_FILE_EOF) { - uint64_t emptyVal = getEmptyRowValue(newCol.colDataType, newCol.colWidth); + uint8_t* emptyVal = (uint8_t*) alloca(newCol.colWidth); + getEmptyRowValue(newCol.colDataType, newCol.colWidth, emptyVal); setEmptyBuf(buf, BYTE_PER_BLOCK, emptyVal, newCol.colWidth); RETURN_ON_ERROR(saveBlock(newCol.dataFile.pFile, buf, newHwm)); - } else + } + else { return rc; } @@ -641,10 +651,10 @@ int ColumnOp::createColumn(Column& column, uint32_t partition) { int rc, newWidth, allocSize; - uint64_t emptyVal = 0; int compressionType = column.compressionType; setColParam(column, colNo, colWidth, colDataType, colType); - emptyVal = getEmptyRowValue(colDataType, colWidth); + uint8_t* emptyVal = (uint8_t*) alloca(colWidth); + getEmptyRowValue(colDataType, colWidth, emptyVal); newWidth = getCorrectRowWidth(colDataType, colWidth); column.dataFile.fid = dataFid; column.dataFile.fDbRoot = dbRoot; @@ -677,8 +687,6 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi HWM colHwm = 0; RID maxRowId = 0; int size = sizeof(Token); - uint64_t emptyVal; - uint64_t refEmptyVal; long long startColFbo = 0; long long startRefColFbo = 0; @@ -713,8 +721,10 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi config.initConfigCache(); std::vector rootList; config.getRootIdList( rootList ); - emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); - refEmptyVal = getEmptyRowValue(refCol.colDataType, refCol.colWidth); + uint8_t* emptyVal = (uint8_t*) alloca(column.colWidth); + uint8_t* refEmptyVal = (uint8_t*) alloca(refCol.colWidth); + getEmptyRowValue(column.colDataType, column.colWidth, emptyVal); + getEmptyRowValue(refCol.colDataType, refCol.colWidth, refEmptyVal); //find the dbroots which have rows for refrence column unsigned int i = 0, k = 0; @@ -938,7 +948,7 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi while (refBufOffset > 0) { - if (memcmp(&refColBuf[refBufOffset], &refEmptyVal, refCol.colWidth) != 0) + if (memcmp(&refColBuf[refBufOffset], refEmptyVal, refCol.colWidth) != 0) { maxRowId = maxRowId + (refBufOffset / refCol.colWidth); break; @@ -999,7 +1009,7 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi while ((tmpBufOffset + refCol.colWidth) <= BYTE_PER_BLOCK) { - if (memcmp(refColBuf + tmpBufOffset, &refEmptyVal, refCol.colWidth) != 0) //Find the number of nextVal needed. + if (memcmp(refColBuf + tmpBufOffset, refEmptyVal, refCol.colWidth) != 0) //Find the number of nextVal needed. { nexValNeeded++; //memcpy(colBuf + colBufOffset, defaultVal, column.colWidth); @@ -1028,7 +1038,7 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi while (((refBufOffset + refCol.colWidth) <= BYTE_PER_BLOCK) && ((colBufOffset + column.colWidth) <= BYTE_PER_BLOCK)) { - if (memcmp(refColBuf + refBufOffset, &refEmptyVal, refCol.colWidth) != 0) //Find the number of nextVal needed. + if (memcmp(refColBuf + refBufOffset, refEmptyVal, refCol.colWidth) != 0) //Find the number of nextVal needed. { memcpy(defaultVal, &nextVal, 8); nextVal++; @@ -1085,7 +1095,7 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi while (((refBufOffset + refCol.colWidth) <= BYTE_PER_BLOCK) && ((colBufOffset + column.colWidth) <= BYTE_PER_BLOCK)) { - if (memcmp(refColBuf + refBufOffset, &refEmptyVal, refCol.colWidth) != 0) + if (memcmp(refColBuf + refBufOffset, refEmptyVal, refCol.colWidth) != 0) { /*if (autoincrement) { @@ -1097,8 +1107,7 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi } else if (column.compressionType != 0) //@Bug 3866, fill the empty row value for compressed chunk { - for(int b = 0, w = column.colWidth; b < column.colWidth; b += 8, w = 8) //FIXME for no loop! - memcpy(colBuf + colBufOffset + b, &emptyVal, w); + memcpy(colBuf + colBufOffset, emptyVal, column.colWidth); dirty = true; } @@ -1327,9 +1336,8 @@ int ColumnOp::extendColumn( bool& newFile, char* hdrs) { - uint64_t emptyVal = 0; - - emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); + uint8_t* emptyVal = (uint8_t*) alloca(column.colWidth); + getEmptyRowValue(column.colDataType, column.colWidth, emptyVal); int rc = extendFile(column.dataFile.fid, emptyVal, column.colWidth, @@ -1370,9 +1378,8 @@ int ColumnOp::addExtent( int& allocSize, char* hdrs) { - uint64_t emptyVal = 0; - - emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); + uint8_t* emptyVal = (uint8_t*) alloca(column.colWidth); + getEmptyRowValue(column.colDataType, column.colWidth, emptyVal); int rc = addExtentExactFile(column.dataFile.fid, emptyVal, column.colWidth, @@ -1400,7 +1407,8 @@ int ColumnOp::addExtent( ***********************************************************/ int ColumnOp::expandAbbrevExtent(const Column& column) { - uint64_t emptyVal = getEmptyRowValue(column.colDataType, column.colWidth); + uint8_t* emptyVal = (uint8_t*) alloca(column.colWidth); + getEmptyRowValue(column.colDataType, column.colWidth, emptyVal); int rc = expandAbbrevColumnExtent(column.dataFile.pFile, column.dataFile.fDbRoot, emptyVal, @@ -1456,33 +1464,30 @@ void ColumnOp::initColumn(Column& column) const * RETURN: * true if success, false otherwise ***********************************************************/ - -// It is called at just 4 places on allocRowId() but all the time inside extend scanning loops -// WIP Template this method -// TODO MCOL-641 Add support here. -inline bool ColumnOp::isEmptyRow(uint64_t* curVal, uint64_t emptyVal, const int colWidth) +inline bool ColumnOp::isEmptyRow(uint64_t* curVal, uint8_t* emptyVal, const int colWidth) { + // colWidth is either 1, 2, 4, 8, or 16 (Convertor::getCorrectRowWidth) switch(colWidth){ case 1: - return *(uint8_t*)curVal == emptyVal; + return *(uint8_t*)curVal == *(uint8_t*)emptyVal; case 2: - return *(uint16_t*)curVal == emptyVal; + return *(uint16_t*)curVal == *(uint16_t*)emptyVal; case 4: - return *(uint32_t*)curVal == emptyVal; + return *(uint32_t*)curVal == *(uint32_t*)emptyVal; case 8: - return *curVal == emptyVal; + return *(uint64_t*)curVal == *(uint64_t*)emptyVal; case 16: - return ((curVal[0] == emptyVal) && (curVal[1] == emptyVal)); + return *(uint128_t*)curVal == *(uint128_t*)emptyVal; - case 32: - return ((curVal[0] == emptyVal) && (curVal[1] == emptyVal) - && (curVal[2] == emptyVal) && (curVal[3] == emptyVal)); + //case 32: + // return ((curVal[0] == emptyVal) && (curVal[1] == emptyVal) + // && (curVal[2] == emptyVal) && (curVal[3] == emptyVal)); } - // WIP + return false; } @@ -1610,11 +1615,6 @@ void ColumnOp::setColParam(Column& column, column.compressionType = compressionType; } -// WIP -using int128_t = __int128; -using uint128_t = unsigned __int128; - - /*********************************************************** * DESCRIPTION: * Write row(s) @@ -1637,7 +1637,7 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, bool bExit = false, bDataDirty = false; void* pVal = 0; char charTmpBuf[8]; - uint64_t emptyVal; + uint8_t* emptyVal = (uint8_t*) alloca(curCol.colWidth); int rc = NO_ERROR; uint16_t rowsInBlock = BYTE_PER_BLOCK / curCol.colWidth; @@ -1734,9 +1734,7 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, break; case WriteEngine::WR_BINARY: - // WIP CSCCol type - pVal = &((uint128_t*) valArray)[i]; - //pVal = (uint8_t*) valArray + i * curCol.colWidth; + if (!bDelete) pVal = &((int128_t*) valArray)[i]; break; default : @@ -1744,11 +1742,10 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, break; } - // TODO MCOL-641 do we need support here? if (bDelete) { - emptyVal = getEmptyRowValue(curCol.colDataType, curCol.colWidth); - pVal = &emptyVal; + utils::getEmptyRowValue(curCol.colDataType, curCol.colWidth, emptyVal); + pVal = emptyVal; } // This is the write stuff @@ -1793,16 +1790,9 @@ int ColumnOp::writeRows(Column& curCol, uint64_t totalRow, const RIDList& ridLis void* pVal = 0; //void* pOldVal; char charTmpBuf[8]; - uint64_t emptyVal; + uint8_t* emptyVal; int rc = NO_ERROR; - int w = 0, incr = 8; - - if (curCol.colType == WriteEngine::WR_BINARY) - w = incr = curCol.colWidth; - else - w = curCol.colWidth > 8 ? 8 : curCol.colWidth; - while (!bExit) { curRowId = ridList[i]; @@ -1903,26 +1893,12 @@ int ColumnOp::writeRows(Column& curCol, uint64_t totalRow, const RIDList& ridLis } else { - if (curCol.colType != WriteEngine::WR_BINARY) - { - emptyVal = getEmptyRowValue(curCol.colDataType, curCol.colWidth); - pVal = &emptyVal; - } - else - { - // fix this - uint128_t bigEmptyVal; - emptyVal = getEmptyRowValue(curCol.colDataType, curCol.colWidth); - *(reinterpret_cast(&bigEmptyVal)) = emptyVal; - *(reinterpret_cast(&bigEmptyVal) + 1) = emptyVal; - //dataconvert::DataConvert::uint128Max(bigEmptyVal); - pVal = &bigEmptyVal; - } + emptyVal = (uint8_t*) alloca(curCol.colWidth); + getEmptyRowValue(curCol.colDataType, curCol.colWidth, emptyVal); + pVal = emptyVal; } - // This is the write stuff - for (int b = 0; b < curCol.colWidth; b += incr) //FIXME for no loop - writeBufValue(dataBuf + dataBio + b, pVal, w); + writeBufValue(dataBuf + dataBio, pVal, curCol.colWidth); i++; diff --git a/writeengine/wrapper/we_colop.h b/writeengine/wrapper/we_colop.h index ad485a8e7..1b2d5dcc7 100644 --- a/writeengine/wrapper/we_colop.h +++ b/writeengine/wrapper/we_colop.h @@ -220,7 +220,7 @@ public: /** * @brief Check whether it is an empty row */ - EXPORT virtual bool isEmptyRow(uint64_t* curVal, uint64_t emptyVal, const int colWidth); + EXPORT virtual bool isEmptyRow(uint64_t* curVal, uint8_t* emptyVal, const int colWidth); /** * @brief Check whether it is a valid column diff --git a/writeengine/wrapper/we_colopcompress.cpp b/writeengine/wrapper/we_colopcompress.cpp index cb4fb05dd..4b06c429d 100644 --- a/writeengine/wrapper/we_colopcompress.cpp +++ b/writeengine/wrapper/we_colopcompress.cpp @@ -191,7 +191,7 @@ int ColumnOpCompress1::flushFile(int rc, std::map& columnOids) int ColumnOpCompress1::expandAbbrevColumnExtent( - IDBDataFile* pFile, uint16_t dbRoot, uint64_t emptyVal, int width) + IDBDataFile* pFile, uint16_t dbRoot, uint8_t* emptyVal, int width) { // update the uncompressed initial chunk to full chunk int rc = m_chunkManager->expandAbbrevColumnExtent(pFile, emptyVal, width); diff --git a/writeengine/wrapper/we_colopcompress.h b/writeengine/wrapper/we_colopcompress.h index 0203af320..5b6635dac 100644 --- a/writeengine/wrapper/we_colopcompress.h +++ b/writeengine/wrapper/we_colopcompress.h @@ -111,7 +111,7 @@ public: /** * @brief virtual method in FileOp */ - int expandAbbrevColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, uint64_t emptyVal, int width); + int expandAbbrevColumnExtent(IDBDataFile* pFile, uint16_t dbRoot, uint8_t* emptyVal, int width); /** * @brief virtual method in ColumnOp diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 7ce12b5a3..559dbe616 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -168,7 +168,6 @@ int WriteEngineWrapper::checkValid(const TxnID& txnid, const ColStructList& colS structListSize = colStructList.size() ; valListSize = colValueList.size(); -// if (colStructList.size() != colValueList.size()) if (structListSize != valListSize) return ERR_STRUCT_VALUE_NOT_MATCH; @@ -400,16 +399,16 @@ void WriteEngineWrapper::convertValue(const execplan::CalpontSystemCatalog::ColT } break; - // WIP MCOL-641 case WriteEngine::WR_BINARY: { size = cscColType.colWidth; - if (cscColType.colDataType == CalpontSystemCatalog::DECIMAL) + if (cscColType.colDataType == CalpontSystemCatalog::DECIMAL || + cscColType.colDataType == CalpontSystemCatalog::UDECIMAL) { int128_t val = boost::any_cast(data); memcpy(value, &val, size); } - else + else // for CalpontSystemCatalog::BINARY { char val = boost::any_cast(data); memcpy(value, &val, size); @@ -521,19 +520,19 @@ void WriteEngineWrapper::convertValue(const CalpontSystemCatalog::ColType& cscCo ((Token*)valArray)[pos] = boost::any_cast(data); break; + // WIP MCOL-641 case WriteEngine::WR_BINARY: - if (cscColType.colDataType != CalpontSystemCatalog::DECIMAL) - { - curStr = boost::any_cast(data); - // String length or column width? - memcpy((char*)valArray + pos * curStr.length(), curStr.c_str(), curStr.length()); - } - else + size_t size = cscColType.colWidth; + if (cscColType.colDataType == CalpontSystemCatalog::DECIMAL || + cscColType.colDataType == CalpontSystemCatalog::UDECIMAL) { int128_t val = boost::any_cast(data); - size_t size = cscColType.colWidth; - // WIP Why do we use memcpy here? - memcpy((uint8_t*)valArray+pos*size, &val, size); + memcpy((uint8_t*)valArray + pos * size, &val, size); + } + else // for CalpontSystemCatalog::BINARY + { + char val = boost::any_cast(data); + memcpy((uint8_t*)valArray + pos * size, &val, size); } break; @@ -602,17 +601,19 @@ void WriteEngineWrapper::convertValue(const CalpontSystemCatalog::ColType& cscCo case WriteEngine::WR_TOKEN: data = ((Token*)valArray)[pos]; break; - // WIP + // WIP MCOL-641 case WriteEngine::WR_BINARY : - if (cscColType.colDataType == CalpontSystemCatalog::DECIMAL) + if (cscColType.colDataType == CalpontSystemCatalog::DECIMAL || + cscColType.colDataType == CalpontSystemCatalog::UDECIMAL) { data = ((int128_t*)valArray)[pos]; } - else + else // for CalpontSystemCatalog::BINARY { // WIP do we need tmp here? - char *tmp = (char*) alloca (sizeof(char) * cscColType.colWidth); - memcpy(tmp, (char*)valArray + pos * cscColType.colWidth, cscColType.colWidth); + size_t size = cscColType.colWidth; + char *tmp = (char*) alloca (sizeof(char) * size); + memcpy(tmp, (uint8_t*)valArray + pos * size, size); curStr = tmp; data = curStr; } @@ -778,7 +779,6 @@ int WriteEngineWrapper::deleteRow(const TxnID& txnid, const vector DctnryValueList dctnryValueList; ColStructList colStructList; CSCTypesList cscColTypeList; - uint64_t emptyVal; int rc; string tmpStr(""); vector dctnryExtentsStruct; @@ -790,6 +790,8 @@ int WriteEngineWrapper::deleteRow(const TxnID& txnid, const vector setTransId(txnid); unsigned numExtents = colExtentsStruct.size(); + uint128_t emptyVal; + for (unsigned extent = 0; extent < numExtents; extent++) { colStructList = colExtentsStruct[extent]; @@ -802,23 +804,21 @@ int WriteEngineWrapper::deleteRow(const TxnID& txnid, const vector cscColType = cscColTypeList[i]; Convertor::convertColType(&curColStruct); - if (curColStruct.colType == WriteEngine::WR_BINARY) + /*if (curColStruct.colType == WriteEngine::WR_BINARY) { uint128_t bigEmptyVal; emptyVal = m_colOp[op(curColStruct.fCompressionType)]-> getEmptyRowValue(curColStruct.colDataType, curColStruct.colWidth); *(reinterpret_cast(&bigEmptyVal)) = emptyVal; *(reinterpret_cast(&bigEmptyVal) + 1) = emptyVal; - //dataconvert::DataConvert::uint128Max(bigEmptyVal); curTuple.data = bigEmptyVal; } else - { - emptyVal = m_colOp[op(curColStruct.fCompressionType)]-> - getEmptyRowValue(curColStruct.colDataType, curColStruct.colWidth); - + {*/ + m_colOp[op(curColStruct.fCompressionType)]-> + getEmptyRowValue(curColStruct.colDataType, curColStruct.colWidth, (uint8_t*)&emptyVal); curTuple.data = emptyVal; - } + //} curTupleList.push_back(curTuple); colValueList.push_back(curTupleList); @@ -850,6 +850,75 @@ int WriteEngineWrapper::deleteRow(const TxnID& txnid, const vector return rc; } +inline void allocateValArray(void*& valArray, ColTupleList::size_type totalRow, + ColType colType, int colWidth) +{ + valArray = calloc(totalRow, colWidth); + // TODO MCOL-641 is commenting out the switch statement below correct? +#if 0 + switch (colType) + { + case WriteEngine::WR_INT: + case WriteEngine::WR_MEDINT: + valArray = (int*) calloc(sizeof(int), totalRow); + break; + + case WriteEngine::WR_UINT: + case WriteEngine::WR_UMEDINT: + valArray = (uint32_t*) calloc(sizeof(uint32_t), totalRow); + break; + + case WriteEngine::WR_VARBINARY : // treat same as char for now + case WriteEngine::WR_CHAR: + case WriteEngine::WR_BLOB: + case WriteEngine::WR_TEXT: + valArray = (char*) calloc(sizeof(char), totalRow * MAX_COLUMN_BOUNDARY); + break; + + case WriteEngine::WR_FLOAT: + valArray = (float*) calloc(sizeof(float), totalRow); + break; + + case WriteEngine::WR_DOUBLE: + valArray = (double*) calloc(sizeof(double), totalRow); + break; + + case WriteEngine::WR_BYTE: + valArray = (char*) calloc(sizeof(char), totalRow); + break; + + case WriteEngine::WR_UBYTE: + valArray = (uint8_t*) calloc(sizeof(uint8_t), totalRow); + break; + + case WriteEngine::WR_SHORT: + valArray = (short*) calloc(sizeof(short), totalRow); + break; + + case WriteEngine::WR_USHORT: + valArray = (uint16_t*) calloc(sizeof(uint16_t), totalRow); + break; + + case WriteEngine::WR_LONGLONG: + valArray = (long long*) calloc(sizeof(long long), totalRow); + break; + + case WriteEngine::WR_ULONGLONG: + valArray = (uint64_t*) calloc(sizeof(uint64_t), totalRow); + break; + + case WriteEngine::WR_TOKEN: + valArray = (Token*) calloc(sizeof(Token), totalRow); + break; + + case WriteEngine::WR_BINARY: + valArray = calloc(totalRow, colWidth); + break; + + } +#endif +} + int WriteEngineWrapper::deleteBadRows(const TxnID& txnid, ColStructList& colStructs, RIDList& ridList, DctnryStructList& dctnryStructList) { @@ -889,69 +958,7 @@ int WriteEngineWrapper::deleteBadRows(const TxnID& txnid, ColStructList& colStru throw std::runtime_error(oss.str()); } - switch (colStructs[i].colType) - { - case WriteEngine::WR_INT: - case WriteEngine::WR_MEDINT: - valArray = (int*) calloc(sizeof(int), 1); - break; - - case WriteEngine::WR_UINT: - case WriteEngine::WR_UMEDINT: - valArray = (uint32_t*) calloc(sizeof(uint32_t), 1); - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR: - case WriteEngine::WR_BLOB: - case WriteEngine::WR_TEXT: - valArray = (char*) calloc(sizeof(char), 1 * MAX_COLUMN_BOUNDARY); - break; - - case WriteEngine::WR_FLOAT: - valArray = (float*) calloc(sizeof(float), 1); - break; - - case WriteEngine::WR_DOUBLE: - valArray = (double*) calloc(sizeof(double), 1); - break; - - case WriteEngine::WR_BYTE: - valArray = (char*) calloc(sizeof(char), 1); - break; - - case WriteEngine::WR_UBYTE: - valArray = (uint8_t*) calloc(sizeof(uint8_t), 1); - break; - - case WriteEngine::WR_SHORT: - valArray = (short*) calloc(sizeof(short), 1); - break; - - case WriteEngine::WR_USHORT: - valArray = (uint16_t*) calloc(sizeof(uint16_t), 1); - break; - - case WriteEngine::WR_LONGLONG: - valArray = (long long*) calloc(sizeof(long long), 1); - break; - - case WriteEngine::WR_ULONGLONG: - valArray = (uint64_t*) calloc(sizeof(uint64_t), 1); - break; - - case WriteEngine::WR_TOKEN: - valArray = (Token*) calloc(sizeof(Token), 1); - break; - - case WriteEngine::WR_BINARY: - //case WriteEngine::WR_INT128: - // WIP use column width here - // remove all C-casts from above - valArray = calloc(1, 16); - break; - - } + allocateValArray(valArray, 1, colStructs[i].colType, colStructs[i].colWidth); rc = colOp->writeRows(curCol, ridList.size(), ridList, valArray, 0, true); @@ -3270,8 +3277,6 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, printInputValue(colStructList, colValueList, ridList); } - // end - //Convert data type and column width to write engine specific for (i = 0; i < colStructList.size(); i++) Convertor::convertColType(&colStructList[i]); @@ -3299,7 +3304,7 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, //-------------------------------------------------------------------------- // allocate row id(s) //-------------------------------------------------------------------------- - curColStruct = colStructList[colId]; + curColStruct = colStructList[colId]; colOp = m_colOp[op(curColStruct.fCompressionType)]; colOp->initColumn(curCol); @@ -3336,7 +3341,7 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, oldHwm = hwm; //Save this info for rollback //need to pass real dbRoot, partition, and segment to setColParam - colOp->setColParam(curCol, colId, curColStruct.colWidth, curColStruct.colDataType, + colOp->setColParam(curCol, colId, curColStruct.colWidth, curColStruct.colDataType, curColStruct.colType, curColStruct.dataOid, curColStruct.fCompressionType, dbRoot, partitionNum, segmentNum); @@ -3455,12 +3460,12 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, // if totalRow == rowsLeft, then not adding rows to 1st extent, so skip it. //-------------------------------------------------------------------------- // DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated? - if ((colStructList[colId].fColPartition == 0) && - (colStructList[colId].fColSegment == 0) && - ((totalRow - rowsLeft) > 0) && - (rowIdArray[totalRow - rowsLeft - 1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK)) + if ((colStructList[colId].fColPartition == 0) && + (colStructList[colId].fColSegment == 0) && + ((totalRow - rowsLeft) > 0) && + (rowIdArray[totalRow - rowsLeft - 1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK)) { - for (unsigned k=0; kcalculateRowId(lastRid, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); - //cout << "insertcolumnrec oid:rid:fbo:hwm = " << - //colStructList[i].dataOid << ":" << lastRid << ":" << - //curFbo << ":" << hwm << endl; if (succFlag) { if ((HWM)curFbo > oldHwm) @@ -3811,7 +3812,6 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid, } } - //cout << "lbids size = " << lbids.size()<< endl; if (lbids.size() > 0) rc = BRMWrapper::getInstance()->markExtentsInvalid(lbids, colDataTypes); @@ -4516,68 +4516,7 @@ int WriteEngineWrapper::writeColumnRecords(const TxnID& txnid, break; } - switch (curColStruct.colType) - { - case WriteEngine::WR_INT: - case WriteEngine::WR_MEDINT: - valArray = (int*) calloc(sizeof(int), totalRow); - break; - - case WriteEngine::WR_UINT: - case WriteEngine::WR_UMEDINT: - valArray = (uint32_t*) calloc(sizeof(uint32_t), totalRow); - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR: - case WriteEngine::WR_BLOB: - case WriteEngine::WR_TEXT: - valArray = (char*) calloc(sizeof(char), totalRow * MAX_COLUMN_BOUNDARY); - break; - - case WriteEngine::WR_FLOAT: - valArray = (float*) calloc(sizeof(float), totalRow); - break; - - case WriteEngine::WR_DOUBLE: - valArray = (double*) calloc(sizeof(double), totalRow); - break; - - case WriteEngine::WR_BYTE: - valArray = (char*) calloc(sizeof(char), totalRow); - break; - - case WriteEngine::WR_UBYTE: - valArray = (uint8_t*) calloc(sizeof(uint8_t), totalRow); - break; - - case WriteEngine::WR_SHORT: - valArray = (short*) calloc(sizeof(short), totalRow); - break; - - case WriteEngine::WR_USHORT: - valArray = (uint16_t*) calloc(sizeof(uint16_t), totalRow); - break; - - case WriteEngine::WR_LONGLONG: - valArray = (long long*) calloc(sizeof(long long), totalRow); - break; - - case WriteEngine::WR_ULONGLONG: - valArray = (uint64_t*) calloc(sizeof(uint64_t), totalRow); - break; - - case WriteEngine::WR_TOKEN: - valArray = (Token*) calloc(sizeof(Token), totalRow); - break; - - // WIP MCOL-641 - case WriteEngine::WR_BINARY: - // Use column width and remove all C-casts from above - valArray = calloc(totalRow, curColType.colWidth); - break; - - } + allocateValArray(valArray, totalRow, curColStruct.colType, curColStruct.colWidth); // convert values to valArray bExcp = false; @@ -4679,16 +4618,11 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, StopWatch timer; #endif + totalRow1 = colValueList[0].size(); + totalRow2 = 0; + if (newColValueList.size() > 0) - { - totalRow1 = colValueList[0].size(); totalRow2 = newColValueList[0].size(); - } - else - { - totalRow1 = colValueList[0].size(); - totalRow2 = 0; - } TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid); @@ -4763,74 +4697,10 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, // WIP We can allocate based on column size and not colType // have to init the size here - valArray = calloc(totalRow1, colStructList[i].colWidth); -#if 0 - switch (colStructList[i].colType) - { - // WIP we don't need type cast here only size - case WriteEngine::WR_INT: - case WriteEngine::WR_MEDINT: - valArray = (int*) calloc(sizeof(int), totalRow1); - break; - - case WriteEngine::WR_UINT: - case WriteEngine::WR_UMEDINT: - valArray = (uint32_t*) calloc(sizeof(uint32_t), totalRow1); - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR: - case WriteEngine::WR_BLOB: - case WriteEngine::WR_TEXT: - valArray = (char*) calloc(sizeof(char), totalRow1 * MAX_COLUMN_BOUNDARY); - break; - - case WriteEngine::WR_FLOAT: - valArray = (float*) calloc(sizeof(float), totalRow1); - break; - - case WriteEngine::WR_DOUBLE: - valArray = (double*) calloc(sizeof(double), totalRow1); - break; - - case WriteEngine::WR_BYTE: - valArray = (char*) calloc(sizeof(char), totalRow1); - break; - - case WriteEngine::WR_UBYTE: - valArray = (uint8_t*) calloc(sizeof(uint8_t), totalRow1); - break; - - case WriteEngine::WR_SHORT: - valArray = (short*) calloc(sizeof(short), totalRow1); - break; - - case WriteEngine::WR_USHORT: - valArray = (uint16_t*) calloc(sizeof(uint16_t), totalRow1); - break; - - case WriteEngine::WR_LONGLONG: - valArray = (long long*) calloc(sizeof(long long), totalRow1); - break; - - case WriteEngine::WR_ULONGLONG: - valArray = (uint64_t*) calloc(sizeof(uint64_t), totalRow1); - break; - - case WriteEngine::WR_TOKEN: - valArray = (Token*) calloc(sizeof(Token), totalRow1); - break; - - // WIP - case WriteEngine::WR_BINARY: - valArray = calloc(totalRow1, colStructList[i].colWidth); - break; - } -#endif + allocateValArray(valArray, totalRow1, colStructList[i].colType, colStructList[i].colWidth); // convert values to valArray - // WIP - // Is this m_opType ever set to DELETE? + // WIP Is m_opType ever set to DELETE? if (m_opType != DELETE) { bExcp = false; @@ -4857,7 +4727,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, } #ifdef PROFILE - iimer.start("writeRow "); + timer.start("writeRow "); #endif rc = colOp->writeRow(curCol, totalRow1, firstPart, valArray); #ifdef PROFILE @@ -4950,71 +4820,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, } } - // have to init the size here - // TODO MCOL-641 is commenting out the switch statement below correct? - valArray = calloc(totalRow2, newColStructList[i].colWidth); - /*switch (newColStructList[i].colType) - { - case WriteEngine::WR_INT: - case WriteEngine::WR_MEDINT: - valArray = (int*) calloc(sizeof(int), totalRow2); - break; - - case WriteEngine::WR_UINT: - case WriteEngine::WR_UMEDINT: - valArray = (uint32_t*) calloc(sizeof(uint32_t), totalRow2); - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR: - case WriteEngine::WR_BLOB: - case WriteEngine::WR_TEXT: - valArray = (char*) calloc(sizeof(char), totalRow2 * MAX_COLUMN_BOUNDARY); - break; - - case WriteEngine::WR_FLOAT: - valArray = (float*) calloc(sizeof(float), totalRow2); - break; - - case WriteEngine::WR_DOUBLE: - valArray = (double*) calloc(sizeof(double), totalRow2); - break; - - case WriteEngine::WR_BYTE: - valArray = (char*) calloc(sizeof(char), totalRow2); - break; - - case WriteEngine::WR_UBYTE: - valArray = (uint8_t*) calloc(sizeof(uint8_t), totalRow2); - break; - - case WriteEngine::WR_SHORT: - valArray = (short*) calloc(sizeof(short), totalRow2); - break; - - case WriteEngine::WR_USHORT: - valArray = (uint16_t*) calloc(sizeof(uint16_t), totalRow2); - break; - - case WriteEngine::WR_LONGLONG: - valArray = (long long*) calloc(sizeof(long long), totalRow2); - break; - - case WriteEngine::WR_ULONGLONG: - valArray = (uint64_t*) calloc(sizeof(uint64_t), totalRow2); - break; - - case WriteEngine::WR_TOKEN: - valArray = (Token*) calloc(sizeof(Token), totalRow2); - break; - - case WriteEngine::WR_BINARY: - //case WriteEngine::WR_INT128: - // WIP - valArray = calloc(totalRow2, 16); - break; - - }*/ + allocateValArray(valArray, totalRow2, newColStructList[i].colType, newColStructList[i].colWidth); // convert values to valArray if (m_opType != DELETE) @@ -5131,71 +4937,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, } } - // have to init the size here - // shared pointers or memory in a stack - // TODO MCOL-641 is commenting out the switch statement below correct? - valArray = calloc(totalRow1, colStructList[i].colWidth); - // WIP - /*switch (colStructList[i].colType) - { - case WriteEngine::WR_INT: - case WriteEngine::WR_MEDINT: - valArray = (int*) calloc(sizeof(int), totalRow1); - break; - - case WriteEngine::WR_UINT: - case WriteEngine::WR_UMEDINT: - valArray = (uint32_t*) calloc(sizeof(uint32_t), totalRow1); - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR: - case WriteEngine::WR_BLOB: - case WriteEngine::WR_TEXT: - valArray = (char*) calloc(sizeof(char), totalRow1 * MAX_COLUMN_BOUNDARY); - break; - - case WriteEngine::WR_FLOAT: - valArray = (float*) calloc(sizeof(float), totalRow1); - break; - - case WriteEngine::WR_DOUBLE: - valArray = (double*) calloc(sizeof(double), totalRow1); - break; - - case WriteEngine::WR_BYTE: - valArray = (char*) calloc(sizeof(char), totalRow1); - break; - - case WriteEngine::WR_UBYTE: - valArray = (uint8_t*) calloc(sizeof(uint8_t), totalRow1); - break; - - case WriteEngine::WR_SHORT: - valArray = (short*) calloc(sizeof(short), totalRow1); - break; - - case WriteEngine::WR_USHORT: - valArray = (uint16_t*) calloc(sizeof(uint16_t), totalRow1); - break; - - case WriteEngine::WR_LONGLONG: - valArray = (long long*) calloc(sizeof(long long), totalRow1); - break; - - case WriteEngine::WR_ULONGLONG: - valArray = (uint64_t*) calloc(sizeof(uint64_t), totalRow1); - break; - - case WriteEngine::WR_TOKEN: - valArray = (Token*) calloc(sizeof(Token), totalRow1); - break; - - case WriteEngine::WR_BINARY: - //case WriteEngine::WR_INT128: - valArray = calloc(colStructList[i].colWidth, totalRow1); - break; - }*/ + allocateValArray(valArray, totalRow1, colStructList[i].colType, colStructList[i].colWidth); // convert values to valArray if (m_opType != DELETE) @@ -5415,9 +5157,8 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, ((uint16_t*)valArray)[j] = tmp16; break; - case WriteEngine::WR_BINARY: // WIP - //case WriteEngine::WR_INT128: + case WriteEngine::WR_BINARY: ((uint64_t*)valArray)[j] = curValue; //FIXME maybe break; @@ -5565,9 +5306,8 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid, ((uint16_t*)valArray)[j] = tmp16; break; - case WriteEngine::WR_BINARY: // WIP - //case WriteEngine::WR_INT128: + case WriteEngine::WR_BINARY: ((uint64_t*)valArray)[j] = curValue; // FIXME maybe break; } @@ -5786,64 +5526,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, break; } - switch (curColStruct.colType) - { - case WriteEngine::WR_INT: - case WriteEngine::WR_MEDINT: - valArray = (int*) calloc(sizeof(int), 1); - break; - - case WriteEngine::WR_UINT: - case WriteEngine::WR_UMEDINT: - valArray = (uint32_t*) calloc(sizeof(uint32_t), 1); - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR: - case WriteEngine::WR_BLOB: - case WriteEngine::WR_TEXT: - valArray = (char*) calloc(sizeof(char), 1 * MAX_COLUMN_BOUNDARY); - break; - - case WriteEngine::WR_FLOAT: - valArray = (float*) calloc(sizeof(float), 1); - break; - - case WriteEngine::WR_DOUBLE: - valArray = (double*) calloc(sizeof(double), 1); - break; - - case WriteEngine::WR_BYTE: - valArray = (char*) calloc(sizeof(char), 1); - break; - - case WriteEngine::WR_UBYTE: - valArray = (uint8_t*) calloc(sizeof(uint8_t), 1); - break; - - case WriteEngine::WR_SHORT: - valArray = (short*) calloc(sizeof(short), 1); - break; - - case WriteEngine::WR_USHORT: - valArray = (uint16_t*) calloc(sizeof(uint16_t), 1); - break; - - case WriteEngine::WR_LONGLONG: - valArray = (long long*) calloc(sizeof(long long), 1); - break; - - case WriteEngine::WR_ULONGLONG: - valArray = (uint64_t*) calloc(sizeof(uint64_t), 1); - break; - - case WriteEngine::WR_TOKEN: - valArray = (Token*) calloc(sizeof(Token), 1); - break; - case WriteEngine::WR_BINARY: - valArray = calloc(1, curColStruct.colWidth); - break; - } + allocateValArray(valArray, 1, curColStruct.colType, curColStruct.colWidth); // convert values to valArray if (m_opType != DELETE) From 238386bf63db5a2f90af71e98d9e051c9a5e059c Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 3 Mar 2020 15:51:55 +0000 Subject: [PATCH 25/78] MCOL-641 Replaced IDB_Decima.__v union with int128_t attribute. Moved all tests into ./test Introduced ./datatypes directory --- CMakeLists.txt | 6 +- datatypes/CMakeLists.txt | 9 + datatypes/decimal.cpp | 20 ++ datatypes/decimal.h | 32 +++ dbcon/execplan/arithmeticoperator.cpp | 64 ----- dbcon/execplan/arithmeticoperator.h | 8 +- dbcon/execplan/simplecolumn.cpp | 12 +- dbcon/execplan/treenode.h | 11 +- dbcon/mysql/ha_mcs_impl.cpp | 1 + tests/CMakeLists.txt | 19 ++ tests/arithmeticoperator-tests.cpp | 27 ++ .../dataconvert-tests.cpp | 25 ++ tests/rowgroup-tests.cpp | 243 ++++++++++++++++++ utils/dataconvert/CMakeLists.txt | 2 +- utils/dataconvert/dataconvert.cpp | 7 +- utils/funcexp/funcexp.cpp | 16 +- utils/rowgroup/CMakeLists.txt | 2 +- 17 files changed, 398 insertions(+), 106 deletions(-) create mode 100644 datatypes/CMakeLists.txt create mode 100644 datatypes/decimal.cpp create mode 100644 datatypes/decimal.h create mode 100644 tests/CMakeLists.txt create mode 100644 tests/arithmeticoperator-tests.cpp rename {utils/dataconvert => tests}/dataconvert-tests.cpp (96%) create mode 100644 tests/rowgroup-tests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 7a084400d..d4fe952ad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -205,6 +205,7 @@ SET (ENGINE_OAM_LIBS oamcpp alarmmanager) SET (ENGINE_BRM_LIBS brm idbdatafile cacheutils rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS}) SET (ENGINE_EXEC_LIBS joblist execplan windowfunction joiner rowgroup funcexp udfsdk regr dataconvert common compress querystats querytele thrift threadpool ${ENGINE_BRM_LIBS}) SET (ENGINE_WRITE_LIBS ddlpackageproc ddlpackage dmlpackageproc dmlpackage writeengine writeengineclient idbdatafile cacheutils ${ENGINE_EXEC_LIBS}) +SET (ENGINE_DATATYPES_LIBS datatypes) SET (ENGINE_COMMON_LDFLAGS "") @@ -284,6 +285,7 @@ SET (ENGINE_UTILS_DDLCLEANUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/ddlclea SET (ENGINE_UTILS_QUERYSTATS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/querystats") SET (ENGINE_UTILS_LIBMYSQL_CL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/libmysql_client") SET (ENGINE_WE_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/xml") +SET (ENGINE_DATATYPES_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/datatypes") SET (ENGINE_SERVER_SQL_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/sql") SET (ENGINE_SERVER_INCLUDE_INCLUDE "${SERVER_SOURCE_ROOT_DIR}/include") IF (PCRE_INCLUDES) @@ -305,7 +307,7 @@ ELSE () SET (ENGINE_READLINE_LIBRARY "readline") ENDIF () -SET (ENGINE_COMMON_INCLUDES ${ENGINE_DEFAULT_INCLUDES} ${Boost_INCLUDE_DIR} ${LIBXML2_INCLUDE_DIR} ${ENGINE_UTILS_MESSAGEQCPP_INCLUDE} ${ENGINE_WE_SHARED_INCLUDE} ${ENGINE_UTILS_IDBDATAFILE_INCLUDE} ${ENGINE_UTILS_LOGGINGCPP_INCLUDE} ${ENGINE_UTILS_CONFIGCPP_INCLUDE} ${ENGINE_UTILS_COMPRESS_INCLUDE} ${ENGINE_VERSIONING_BRM_INCLUDE} ${ENGINE_UTILS_ROWGROUP_INCLUDE} ${ENGINE_UTILS_COMMON_INCLUDE} ${ENGINE_UTILS_DATACONVERT_INCLUDE} ${ENGINE_UTILS_RWLOCK_INCLUDE} ${ENGINE_UTILS_FUNCEXP_INCLUDE} ${ENGINE_OAMAPPS_ALARMMANAGER_INCLUDE} ${ENGINE_UTILS_INCLUDE} ${ENGINE_OAM_OAMCPP_INCLUDE} ${ENGINE_DBCON_DDLPKGPROC_INCLUDE} ${ENGINE_DBCON_DDLPKG_INCLUDE} ${ENGINE_DBCON_EXECPLAN_INCLUDE} ${ENGINE_UTILS_STARTUP_INCLUDE} ${ENGINE_DBCON_JOBLIST_INCLUDE} ${ENGINE_WE_WRAPPER_INCLUDE} ${ENGINE_WE_SERVER_INCLUDE} ${ENGINE_DBCON_DMLPKG_INCLUDE} ${ENGINE_WE_CLIENT_INCLUDE} ${ENGINE_DBCON_DMLPKGPROC_INCLUDE} ${ENGINE_UTILS_CACHEUTILS_INCLUDE} ${ENGINE_UTILS_MYSQLCL_INCLUDE} ${ENGINE_UTILS_QUERYTELE_INCLUDE} ${ENGINE_UTILS_THRIFT_INCLUDE} ${ENGINE_UTILS_JOINER_INCLUDE} ${ENGINE_UTILS_THREADPOOL_INCLUDE} ${ENGINE_UTILS_BATCHLDR_INCLUDE} ${ENGINE_UTILS_DDLCLEANUP_INCLUDE} ${ENGINE_UTILS_QUERYSTATS_INCLUDE} ${ENGINE_WE_CONFIGCPP_INCLUDE} ${ENGINE_SERVER_SQL_INCLUDE} ${ENGINE_SERVER_INCLUDE_INCLUDE} ${ENGINE_SERVER_PCRE_INCLUDE} ${ENGINE_SERVER_WSREP_API_INCLUDE} ${ENGINE_SERVER_WSREP_INCLUDE} ${ENGINE_UTILS_UDFSDK_INCLUDE} ${ENGINE_UTILS_LIBMYSQL_CL_INCLUDE}) +SET (ENGINE_COMMON_INCLUDES ${ENGINE_DEFAULT_INCLUDES} ${Boost_INCLUDE_DIR} ${LIBXML2_INCLUDE_DIR} ${ENGINE_UTILS_MESSAGEQCPP_INCLUDE} ${ENGINE_WE_SHARED_INCLUDE} ${ENGINE_UTILS_IDBDATAFILE_INCLUDE} ${ENGINE_UTILS_LOGGINGCPP_INCLUDE} ${ENGINE_UTILS_CONFIGCPP_INCLUDE} ${ENGINE_UTILS_COMPRESS_INCLUDE} ${ENGINE_VERSIONING_BRM_INCLUDE} ${ENGINE_UTILS_ROWGROUP_INCLUDE} ${ENGINE_UTILS_COMMON_INCLUDE} ${ENGINE_UTILS_DATACONVERT_INCLUDE} ${ENGINE_UTILS_RWLOCK_INCLUDE} ${ENGINE_UTILS_FUNCEXP_INCLUDE} ${ENGINE_OAMAPPS_ALARMMANAGER_INCLUDE} ${ENGINE_UTILS_INCLUDE} ${ENGINE_OAM_OAMCPP_INCLUDE} ${ENGINE_DBCON_DDLPKGPROC_INCLUDE} ${ENGINE_DBCON_DDLPKG_INCLUDE} ${ENGINE_DBCON_EXECPLAN_INCLUDE} ${ENGINE_UTILS_STARTUP_INCLUDE} ${ENGINE_DBCON_JOBLIST_INCLUDE} ${ENGINE_WE_WRAPPER_INCLUDE} ${ENGINE_WE_SERVER_INCLUDE} ${ENGINE_DBCON_DMLPKG_INCLUDE} ${ENGINE_WE_CLIENT_INCLUDE} ${ENGINE_DBCON_DMLPKGPROC_INCLUDE} ${ENGINE_UTILS_CACHEUTILS_INCLUDE} ${ENGINE_UTILS_MYSQLCL_INCLUDE} ${ENGINE_UTILS_QUERYTELE_INCLUDE} ${ENGINE_UTILS_THRIFT_INCLUDE} ${ENGINE_UTILS_JOINER_INCLUDE} ${ENGINE_UTILS_THREADPOOL_INCLUDE} ${ENGINE_UTILS_BATCHLDR_INCLUDE} ${ENGINE_UTILS_DDLCLEANUP_INCLUDE} ${ENGINE_UTILS_QUERYSTATS_INCLUDE} ${ENGINE_WE_CONFIGCPP_INCLUDE} ${ENGINE_SERVER_SQL_INCLUDE} ${ENGINE_SERVER_INCLUDE_INCLUDE} ${ENGINE_SERVER_PCRE_INCLUDE} ${ENGINE_SERVER_WSREP_API_INCLUDE} ${ENGINE_SERVER_WSREP_INCLUDE} ${ENGINE_UTILS_UDFSDK_INCLUDE} ${ENGINE_UTILS_LIBMYSQL_CL_INCLUDE} ${ENGINE_DATATYPES_INCLUDE}) ADD_SUBDIRECTORY(dbcon/mysql) IF(NOT TARGET columnstore) @@ -336,6 +338,8 @@ ADD_SUBDIRECTORY(writeengine/server) ADD_SUBDIRECTORY(writeengine/bulk) ADD_SUBDIRECTORY(writeengine/splitter) ADD_SUBDIRECTORY(storage-manager) +ADD_SUBDIRECTORY(datatypes) +ADD_SUBDIRECTORY(tests) # WriteEngine component tests IF( WITH_SHARED_COMP_TESTS ) diff --git a/datatypes/CMakeLists.txt b/datatypes/CMakeLists.txt new file mode 100644 index 000000000..433cefa5c --- /dev/null +++ b/datatypes/CMakeLists.txt @@ -0,0 +1,9 @@ + +include_directories( ${ENGINE_COMMON_INCLUDES} ) +set(datatypes_LIB_SRCS + decimal.cpp) + +add_library(datatypes SHARED ${datatypes_LIB_SRCS}) + +install(TARGETS datatypes DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) + diff --git a/datatypes/decimal.cpp b/datatypes/decimal.cpp new file mode 100644 index 000000000..e5c44d395 --- /dev/null +++ b/datatypes/decimal.cpp @@ -0,0 +1,20 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include "decimal.h" + +const datatypes::Decimal someDecimal; diff --git a/datatypes/decimal.h b/datatypes/decimal.h new file mode 100644 index 000000000..104c4db5c --- /dev/null +++ b/datatypes/decimal.h @@ -0,0 +1,32 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#ifndef H_DECIMALDATATYPE +#define H_DECIMALDATATYPE + +namespace datatypes +{ + +class Decimal +{ + public: + Decimal() { }; + ~Decimal() { }; +}; + +} //end of namespace +#endif diff --git a/dbcon/execplan/arithmeticoperator.cpp b/dbcon/execplan/arithmeticoperator.cpp index 230786800..d968ceb5b 100644 --- a/dbcon/execplan/arithmeticoperator.cpp +++ b/dbcon/execplan/arithmeticoperator.cpp @@ -116,70 +116,6 @@ bool ArithmeticOperator::operator!=(const TreeNode* t) const return (!(*this == t)); } -#if 0 -void ArithmeticOperator::operationType(const Type& l, const Type& r) -{ - if (l.colDataType == execplan::CalpontSystemCatalog::DECIMAL) - { - switch (r.colDataType) - { - case execplan::CalpontSystemCatalog::DECIMAL: - { - // should follow the result type that MySQL gives - fOperationType = fResultType; - break; - } - - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::BIGINT: - fOperationType.colDataType = execplan::CalpontSystemCatalog::DECIMAL; - fOperationType.scale = l.scale; - break; - - default: - fOperationType.colDataType = execplan::CalpontSystemCatalog::DOUBLE; - } - } - else if (r.colDataType == execplan::CalpontSystemCatalog::DECIMAL) - { - switch (l.colDataType) - { - case execplan::CalpontSystemCatalog::DECIMAL: - { - // should following the result type that MySQL gives based on the following logic? - // @NOTE is this trustable? - fOperationType = fResultType; - break; - } - - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::BIGINT: - fOperationType.colDataType = execplan::CalpontSystemCatalog::DECIMAL; - fOperationType.scale = r.scale; - break; - - default: - fOperationType.colDataType = execplan::CalpontSystemCatalog::DOUBLE; - } - } - else if ((l.colDataType == execplan::CalpontSystemCatalog::INT || - l.colDataType == execplan::CalpontSystemCatalog::MEDINT || - l.colDataType == execplan::CalpontSystemCatalog::TINYINT || - l.colDataType == execplan::CalpontSystemCatalog::BIGINT) && - (r.colDataType == execplan::CalpontSystemCatalog::INT || - r.colDataType == execplan::CalpontSystemCatalog::MEDINT || - r.colDataType == execplan::CalpontSystemCatalog::TINYINT || - r.colDataType == execplan::CalpontSystemCatalog::BIGINT)) - fOperationType.colDataType = execplan::CalpontSystemCatalog::BIGINT; - else - fOperationType.colDataType = execplan::CalpontSystemCatalog::DOUBLE; -} -#endif - void ArithmeticOperator::adjustResultType(const CalpontSystemCatalog::ColType& m) { if (m.colDataType != CalpontSystemCatalog::DECIMAL) diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index 4660387fe..479b9e645 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -288,13 +288,9 @@ inline void ArithmeticOperator::execute(IDB_Decimal& result, IDB_Decimal op1, ID switch (fOp) { case OP_ADD: - if (resultCscType.precision > 18) + if (resultCscType.colWidth == 16) { - // WIP make this a separate function w and w/o overflow check - if (resultCscType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) - result.__v.__s128 = op1.__v.__s128 + op2.__v.__s128; - else - result.__v.__u128 = op1.__v.__u128 + op2.__v.__u128; + result.s128Value = op1.s128Value + op2.s128Value; break; } diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index eeb287e53..34e02a4df 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -635,16 +635,8 @@ void SimpleColumn::evaluate(Row& row, bool& isNull) { case 16: { - if (fResultType.colDataType == CalpontSystemCatalog::UDECIMAL) - { - fResult.decimalVal.__v.__u128 = - *row.getBinaryField_offset(fInputOffset); - } - else - { - fResult.decimalVal.__v.__s128 = - *row.getBinaryField_offset(fInputOffset); - } + fResult.decimalVal.s128Value = + *row.getBinaryField_offset(fInputOffset); fResult.decimalVal.scale = (unsigned)fResultType.scale; break; } diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 760760a9d..6448d12e2 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -37,7 +37,6 @@ #include "dataconvert.h" using int128_t = __int128; -using uint128_t = unsigned __int128; namespace messageqcpp { @@ -66,14 +65,14 @@ struct IDB_Decimal { IDB_Decimal(): value(0), scale(0), precision(0) { - __v.__s128 = 0; + s128Value = 0; } IDB_Decimal(int64_t val, int8_t s, uint8_t p) : value (val), scale(s), precision(p) { - __v.__s128 = 0; + s128Value = 0; } int decimalComp(const IDB_Decimal& d) const @@ -158,11 +157,7 @@ struct IDB_Decimal return (decimalComp(rhs) != 0); } - union { - uint128_t __u128; - int128_t __s128; - int64_t __s64[2]; - } __v; + int128_t s128Value; int64_t value; int8_t scale; // 0~38 uint8_t precision; // 1~38 diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 0529523cc..fa2c4a77d 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -820,6 +820,7 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h dataconvert::DataConvert::decimalToString(dec, (unsigned)colType.scale, buf, sizeof(buf), colType.colDataType); + std::cout << buf << std::endl; Field_new_decimal* f2 = (Field_new_decimal*)*f; f2->store(buf, strlen(buf), f2->charset()); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 000000000..5bc8f600a --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,19 @@ +include_directories( ${ENGINE_COMMON_INCLUDES} ) + +if (WITH_ROWGROUP_UT) + add_executable(rowgroup_tests rowgroup-tests.cpp) + target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) + install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) +endif() + +if (WITH_ARITHMETICOPERATOR_UT) + add_executable(arithmeticoperator_tests arithmeticoperator-tests.cpp) + target_link_libraries(arithmeticoperator_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) + install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) +endif() + +if (WITH_DATACONVERT_UT) + add_executable(dataconvert_tests dataconvert-tests.cpp) + target_link_libraries(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) + install(TARGETS dataconvert_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) +endif() diff --git a/tests/arithmeticoperator-tests.cpp b/tests/arithmeticoperator-tests.cpp new file mode 100644 index 000000000..dc192fbd1 --- /dev/null +++ b/tests/arithmeticoperator-tests.cpp @@ -0,0 +1,27 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include // googletest header file + +using int128_t = __int128; +using uint128_t = unsigned __int128; +//using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; + +TEST(SimpleCheck, check1) +{ + EXPECT_EQ(true, true); +} diff --git a/utils/dataconvert/dataconvert-tests.cpp b/tests/dataconvert-tests.cpp similarity index 96% rename from utils/dataconvert/dataconvert-tests.cpp rename to tests/dataconvert-tests.cpp index b184030cb..461ca28f1 100644 --- a/utils/dataconvert/dataconvert-tests.cpp +++ b/tests/dataconvert-tests.cpp @@ -590,6 +590,31 @@ TEST(DataConvertTest, DecimalToStringCheckScale3) { } } +TEST(DataConvertTest, DecimalToStringCheckScale10) { + std::vector expected; + std::vector decimalAsStringVec; + // scale 3 + uint8_t scale3 = 10; + decimalAsStringVec.push_back("10000000009"); + decimalAsStringVec.push_back("-10000000009"); + decimalAsStringVec.push_back("0"); + decimalAsStringVec.push_back("-10000000010"); + expected.push_back("1.0000000009"); + expected.push_back("-1.0000000009"); + expected.push_back("0.0000000000"); + expected.push_back("-1.0000000010"); + char buf[42]; + CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL; + for (int i=0; i < expected.size(); i++) { + int128_t value = -4; + memset(buf, 0, 42); + dataconvert::atoi128(decimalAsStringVec[i], value); + dataconvert::DataConvert::decimalToString(&value, scale3, buf, + utils::precisionByWidth(sizeof(value))+4, type); + EXPECT_EQ(expected[i], std::string(buf)); + } +} + TEST(DataConvertTest, DecimalToStringCheckScale37) { std::vector expected; std::vector decimalAsStringVec; diff --git a/tests/rowgroup-tests.cpp b/tests/rowgroup-tests.cpp new file mode 100644 index 000000000..8fbbe22fc --- /dev/null +++ b/tests/rowgroup-tests.cpp @@ -0,0 +1,243 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include // googletest header file +#include + +#include "rowgroup.h" +#include "columnwidth.h" +#include "joblisttypes.h" +#include "dataconvert.h" + +#define WIDE_DEC_PRECISION 38U +#define INITIAL_ROW_OFFSET 2 + +using int128_t = __int128; +using uint128_t = unsigned __int128; +using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; + +class RowDecimalTest : public ::testing::Test { + protected: + void SetUp() override { + uint32_t precision = WIDE_DEC_PRECISION; + uint32_t oid =3001; + + std::vectortypes; + std::vectorprecisionVec; + std::vector roids, tkeys, cscale; + types.push_back(execplan::CalpontSystemCatalog::DECIMAL); + types.push_back(execplan::CalpontSystemCatalog::UDECIMAL); + for (size_t i=0; i <= 3; i++) { + types.push_back(execplan::CalpontSystemCatalog::DECIMAL); + } + precisionVec.push_back(precision); + precisionVec.push_back(precision); + precisionVec.push_back(18); + precisionVec.push_back(9); + precisionVec.push_back(4); + precisionVec.push_back(2); + std::vectorwidthVec; + uint32_t offset = INITIAL_ROW_OFFSET; + offsets.push_back(offset); + for (size_t i=0; i < types.size(); i++) { + uint8_t width = utils::widthByPrecision(precisionVec[i]); + widthVec.push_back(width); + offset += width; + offsets.push_back(offset); + roids.push_back(oid+i); + tkeys.push_back(i+1); cscale.push_back(0); + } + /*offsets.push_back(INITIAL_ROW_OFFSET); + offsets.push_back(16+INITIAL_ROW_OFFSET); + offsets.push_back(16*2+INITIAL_ROW_OFFSET); + roids.push_back(oid); roids.push_back(oid+1); + tkeys.push_back(1); tkeys.push_back(1); + cscale.push_back(0); cscale.push_back(0);*/ + + rowgroup::RowGroup inRG(roids.size(), //column count + offsets, //oldOffset + roids, // column oids + tkeys, //keys + types, // types + cscale, //scale + precisionVec, // precision + 20, // sTableThreshold + false //useStringTable + ); + + rg = inRG; + rgD.reinit(rg); + rg.setData(&rgD); + + rg.initRow(&r); + rg.initRow(&rOutMappingCheck); + rowSize = r.getSize(); + rg.getRow(0, &r); + + int128_t nullValue = 0; + int128_t bigValue = 0; + uint64_t* uint128_pod = reinterpret_cast(&nullValue); + uint128_pod[0] = joblist::UBIGINTNULL; + uint128_pod[1] = joblist::UBIGINTEMPTYROW; + bigValue = -static_cast(0xFFFFFFFF)*0xFFFFFFFFFFFFFFFF; + sValueVector.push_back(nullValue); + sValueVector.push_back(-42); + sValueVector.push_back(bigValue); + sValueVector.push_back(0); + sValueVector.push_back(nullValue-1); + + uValueVector.push_back(nullValue); + uValueVector.push_back(42); + uValueVector.push_back(bigValue); + uValueVector.push_back(0); + uValueVector.push_back(nullValue-1); + + s8ValueVector.push_back(joblist::TINYINTNULL); + s8ValueVector.push_back(-0x79); + s8ValueVector.push_back(0); + s8ValueVector.push_back(0x81); + s8ValueVector.push_back(joblist::TINYINTNULL-1); + + s16ValueVector.push_back(joblist::SMALLINTNULL); + s16ValueVector.push_back(-0x79); + s16ValueVector.push_back(0); + s16ValueVector.push_back(0x81); + s16ValueVector.push_back(joblist::SMALLINTNULL-1); + + s32ValueVector.push_back(joblist::INTNULL); + s32ValueVector.push_back(-0x79); + s32ValueVector.push_back(0); + s32ValueVector.push_back(0x81); + s32ValueVector.push_back(joblist::INTNULL-1); + + s64ValueVector.push_back(joblist::BIGINTNULL); + s64ValueVector.push_back(-0x79); + s64ValueVector.push_back(0); + s64ValueVector.push_back(0x81); + s64ValueVector.push_back(joblist::BIGINTNULL-1); + + for(size_t i = 0; i < sValueVector.size(); i++) { + r.setBinaryField_offset(&sValueVector[i], + sizeof(sValueVector[0]), offsets[0]); + r.setBinaryField_offset(&uValueVector[i], + sizeof(uValueVector[0]), offsets[1]); + r.setIntField(s64ValueVector[i], 2); + r.setIntField(s32ValueVector[i], 3); + r.setIntField(s16ValueVector[i], 4); + r.setIntField(s8ValueVector[i], 5); + r.nextRow(rowSize); + } + rowCount = sValueVector.size(); + } + // void TearDown() override {} + + rowgroup::Row r; + rowgroup::Row rOutMappingCheck; + rowgroup::RowGroup rg; + rowgroup::RGData rgD; + uint32_t rowSize; + size_t rowCount; + std::vector sValueVector; + std::vector uValueVector; + std::vector s8ValueVector; + std::vector s16ValueVector; + std::vector s32ValueVector; + std::vector s64ValueVector; + std::vector offsets; +}; + +TEST_F(RowDecimalTest, NonNULLValuesCheck) { + rg.getRow(1, &r); + for (size_t i = 1; i <= sValueVector.size(); i++) { + EXPECT_FALSE(r.isNullValue(0)); + EXPECT_FALSE(r.isNullValue(1)); + EXPECT_FALSE(r.isNullValue(2)); + EXPECT_FALSE(r.isNullValue(3)); + EXPECT_FALSE(r.isNullValue(4)); + EXPECT_FALSE(r.isNullValue(5)); + r.nextRow(rowSize); + } +} + +TEST_F(RowDecimalTest, initToNullANDisNullValueValueCheck) { + rg.getRow(0, &r); + r.initToNull(); + EXPECT_TRUE(r.isNullValue(0)); + EXPECT_TRUE(r.isNullValue(1)); + EXPECT_TRUE(r.isNullValue(2)); + EXPECT_TRUE(r.isNullValue(3)); + EXPECT_TRUE(r.isNullValue(4)); + EXPECT_TRUE(r.isNullValue(5)); +} + +TEST_F(RowDecimalTest, getBinaryFieldCheck) { + rg.getRow(0, &r); + uint128_t* u128Value; + int128_t* s128Value; +// std::remove_reference::type uType; +// std::remove_reference::type sType; + + for (size_t i = 0; i < sValueVector.size(); i++) { + s128Value = r.getBinaryField(0); + EXPECT_EQ(sValueVector[i], *s128Value); + u128Value = r.getBinaryField(1); + EXPECT_EQ(uValueVector[i], *u128Value); + //EXPECT_EQ(s64ValueVector[i], r.getIntField(2)); + //EXPECT_EQ(s32ValueVector[i],r.getIntField(3)); + //EXPECT_EQ(r.getIntField(4),s16ValueVector[i]); + //EXPECT_EQ(r.getIntField(5),s8ValueVector[i]); + r.nextRow(rowSize); + } +} + +TEST_F(RowDecimalTest, toStringCheck) { + std::vector exemplarVector; + exemplarVector.push_back(std::string("0: NULL NULL NULL NULL NULL NULL ")); + exemplarVector.push_back(std::string("0: -42 42 -121 -121 -121 -121 ")); + exemplarVector.push_back(std::string("0: -79228162495817593515539431425 -79228162495817593515539431425 0 0 0 0 ")); + exemplarVector.push_back(std::string("0: 0 0 129 129 129 -127 ")); + exemplarVector.push_back(std::string("0: -3 -3 9223372036854775807 2147483647 32767 127 ")); + + rg.getRow(0, &r); + r.initToNull(); + for (auto &el: exemplarVector) { + EXPECT_EQ(el, r.toString()); + r.nextRow(rowSize); + } +} + +TEST_F(RowDecimalTest, toCSVCheck) { +} + +TEST_F(RowDecimalTest, applyMappingCheck) { + int mapping[] = {0, 1, -1, -1, -1, -1}; + rg.getRow(1, &r); + rg.getRow(2, &rOutMappingCheck); + int128_t* s128Value = rOutMappingCheck.getBinaryField(0); + uint128_t* u128Value = rOutMappingCheck.getBinaryField(1); + EXPECT_NE(sValueVector[1], *s128Value); + EXPECT_NE(uValueVector[1], *u128Value); + applyMapping(mapping, r, &rOutMappingCheck); + s128Value = rOutMappingCheck.getBinaryField(0); + EXPECT_EQ(sValueVector[1], *s128Value); + u128Value = rOutMappingCheck.getBinaryField(1); + EXPECT_EQ(uValueVector[1], *u128Value); +} + +// WIP +TEST_F(RowDecimalTest, RowEqualsCheck) { +} diff --git a/utils/dataconvert/CMakeLists.txt b/utils/dataconvert/CMakeLists.txt index 168efdf80..9a25dfb1b 100644 --- a/utils/dataconvert/CMakeLists.txt +++ b/utils/dataconvert/CMakeLists.txt @@ -15,5 +15,5 @@ install(TARGETS dataconvert DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-e if (WITH_DATACONVERT_UT) add_executable(dataconvert_tests dataconvert-tests.cpp) target_link_libraries(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) - install(TARGETS dataconvert_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) + install(TARGETS dataconvert_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) endif() diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index dac9e521e..cb4d7fe27 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1334,14 +1334,11 @@ size_t DataConvert::writeFractionalPart(int128_t* dec, char* p, scaleDivisor *= columnstore_pow_10[scale%maxPowOf10]; } - //for (size_t i = 1; i < scale; i++) - // scaleDivisor *= 10; - int128_t fractionalPart = *dec % scaleDivisor; // divide by the base untill we have non-zero quotinent size_t written = 0; scaleDivisor /= 10; - while (scaleDivisor > 1 && *dec / scaleDivisor == 0) + while (scaleDivisor > 1 && fractionalPart/scaleDivisor == 0) { *p++ = '0'; written++; @@ -1385,6 +1382,8 @@ void DataConvert::toString(int128_t* dec, uint8_t scale, p += writeFractionalPart(dec, p, buflen-(p-original_p), scale); } + *p = '\0'; + if (buflen <= p-original_p) { throw QueryDataExcept("toString() char buffer overflow.", formatErr); diff --git a/utils/funcexp/funcexp.cpp b/utils/funcexp/funcexp.cpp index d20382d30..1499c9744 100644 --- a/utils/funcexp/funcexp.cpp +++ b/utils/funcexp/funcexp.cpp @@ -472,18 +472,12 @@ void FuncExp::evaluate(rowgroup::Row& row, std::vector& expressi { IDB_Decimal val = expression[i]->getDecimalVal(row, isNull); - // WIP - if (expression[i]->resultType().precision > 18) + // WIP check for null and overflow here. + if (expression[i]->resultType().colWidth == 16) { - // WIP make this a separate function w and w/o overflow check - if (expression[i]->resultType().colDataType == execplan::CalpontSystemCatalog::DECIMAL) - row.setBinaryField_offset(&val.__v.__s128, - expression[i]->resultType().colWidth, - row.getOffset(expression[i]->outputIndex())); - else - row.setBinaryField_offset(&val.__v.__u128, - expression[i]->resultType().colWidth, - row.getOffset(expression[i]->outputIndex())); + row.setBinaryField_offset(&val.s128Value, + expression[i]->resultType().colWidth, + row.getOffset(expression[i]->outputIndex())); } else { diff --git a/utils/rowgroup/CMakeLists.txt b/utils/rowgroup/CMakeLists.txt index 86c5fa9db..3fcde10df 100644 --- a/utils/rowgroup/CMakeLists.txt +++ b/utils/rowgroup/CMakeLists.txt @@ -19,5 +19,5 @@ install(TARGETS rowgroup DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engi if (WITH_ROWGROUP_UT) add_executable(rowgroup_tests rowgroup-tests.cpp) target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) - install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) + install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) endif() From 2e8e7d52c38221389fb89c0c498b4e8459fbefac Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 5 Mar 2020 03:27:17 +0000 Subject: [PATCH 26/78] Renamed datatypes/decimal.* into csdecimal to avoid collision with MDB. --- CMakeLists.txt | 3 +- datatypes/CMakeLists.txt | 2 +- datatypes/csdecimal.cpp | 30 ++++++ datatypes/{decimal.h => csdecimal.h} | 9 +- dbcon/execplan/CMakeLists.txt | 2 +- dbcon/execplan/treenode.h | 98 ++++++++++++++++---- dbcon/mysql/ha_mcs_impl.cpp | 4 +- datatypes/decimal.cpp => tests/csdecimal.cpp | 7 +- utils/common/columnwidth.h | 6 +- 9 files changed, 133 insertions(+), 28 deletions(-) create mode 100644 datatypes/csdecimal.cpp rename datatypes/{decimal.h => csdecimal.h} (84%) rename datatypes/decimal.cpp => tests/csdecimal.cpp (89%) diff --git a/CMakeLists.txt b/CMakeLists.txt index d4fe952ad..abd646c2f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -200,12 +200,11 @@ ENDIF() SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed") -SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client) +SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client datatypes) SET (ENGINE_OAM_LIBS oamcpp alarmmanager) SET (ENGINE_BRM_LIBS brm idbdatafile cacheutils rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS}) SET (ENGINE_EXEC_LIBS joblist execplan windowfunction joiner rowgroup funcexp udfsdk regr dataconvert common compress querystats querytele thrift threadpool ${ENGINE_BRM_LIBS}) SET (ENGINE_WRITE_LIBS ddlpackageproc ddlpackage dmlpackageproc dmlpackage writeengine writeengineclient idbdatafile cacheutils ${ENGINE_EXEC_LIBS}) -SET (ENGINE_DATATYPES_LIBS datatypes) SET (ENGINE_COMMON_LDFLAGS "") diff --git a/datatypes/CMakeLists.txt b/datatypes/CMakeLists.txt index 433cefa5c..c9616b1e6 100644 --- a/datatypes/CMakeLists.txt +++ b/datatypes/CMakeLists.txt @@ -1,7 +1,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} ) set(datatypes_LIB_SRCS - decimal.cpp) + csdecimal.cpp) add_library(datatypes SHARED ${datatypes_LIB_SRCS}) diff --git a/datatypes/csdecimal.cpp b/datatypes/csdecimal.cpp new file mode 100644 index 000000000..7f961f4ce --- /dev/null +++ b/datatypes/csdecimal.cpp @@ -0,0 +1,30 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include "csdecimal.h" + +const datatypes::Decimal someDecimal; + +namespace datatypes +{ + + int Decimal::compare(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r) + { + return 0; + } + +} // end of namespace diff --git a/datatypes/decimal.h b/datatypes/csdecimal.h similarity index 84% rename from datatypes/decimal.h rename to datatypes/csdecimal.h index 104c4db5c..a567ce252 100644 --- a/datatypes/decimal.h +++ b/datatypes/csdecimal.h @@ -18,14 +18,21 @@ #ifndef H_DECIMALDATATYPE #define H_DECIMALDATATYPE +#include + +namespace execplan +{ + struct IDB_Decimal; +} + namespace datatypes { - class Decimal { public: Decimal() { }; ~Decimal() { }; + static int compare(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r); }; } //end of namespace diff --git a/dbcon/execplan/CMakeLists.txt b/dbcon/execplan/CMakeLists.txt index 8b7911838..2a1c759a2 100755 --- a/dbcon/execplan/CMakeLists.txt +++ b/dbcon/execplan/CMakeLists.txt @@ -1,5 +1,5 @@ -include_directories( ${ENGINE_COMMON_INCLUDES} ${ENGINE_UTILS_UDFSDK_INCLUDE} ) +include_directories( ${ENGINE_COMMON_INCLUDES} ${ENGINE_UTILS_UDFSDK_INCLUDE} ${ENGINE_DATATYPES_INCLUDE}) ########### next target ############### diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 6448d12e2..1c2f6b05a 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -35,6 +35,8 @@ #include "calpontsystemcatalog.h" #include "exceptclasses.h" #include "dataconvert.h" +#include "columnwidth.h" +#include "csdecimal.h" using int128_t = __int128; @@ -116,45 +118,105 @@ struct IDB_Decimal bool operator==(const IDB_Decimal& rhs) const { - if (scale == rhs.scale) - return value == rhs.value; + if (utils::widthByPrecision(precision) == 16) + { + if (scale == rhs.scale) + return s128Value == rhs.s128Value; + else + return (datatypes::Decimal::compare(*this, rhs) == 0); + } else - return (decimalComp(rhs) == 0); + { + if (scale == rhs.scale) + return value == rhs.value; + else + return (decimalComp(rhs) == 0); + } } bool operator>(const IDB_Decimal& rhs) const { - if (scale == rhs.scale) - return value > rhs.value; + if (utils::widthByPrecision(precision) == 16) + { + if (scale == rhs.scale) + return s128Value > rhs.s128Value; + else + return (datatypes::Decimal::compare(*this, rhs) > 0); + } else - return (decimalComp(rhs) > 0); + { + if (scale == rhs.scale) + return value > rhs.value; + else + return (decimalComp(rhs) > 0); + } } bool operator<(const IDB_Decimal& rhs) const { - if (scale == rhs.scale) - return value < rhs.value; + if (utils::widthByPrecision(precision) == 16) + { + if (scale == rhs.scale) + return s128Value < rhs.s128Value; + else + return (datatypes::Decimal::compare(*this, rhs) < 0); + } else - return (decimalComp(rhs) < 0); + { + if (scale == rhs.scale) + return value < rhs.value; + else + return (decimalComp(rhs) < 0); + } } bool operator>=(const IDB_Decimal& rhs) const { - if (scale == rhs.scale) - return value >= rhs.value; + if (utils::widthByPrecision(precision) == 16) + { + if (scale == rhs.scale) + return s128Value >= rhs.s128Value; + else + return (datatypes::Decimal::compare(*this, rhs) >= 0); + } else - return (decimalComp(rhs) >= 0); + { + if (scale == rhs.scale) + return value >= rhs.value; + else + return (decimalComp(rhs) >= 0); + } } bool operator<=(const IDB_Decimal& rhs) const { - if (scale == rhs.scale) - return value <= rhs.value; + if (utils::widthByPrecision(precision) == 16) + { + if (scale == rhs.scale) + return s128Value <= rhs.s128Value; + else + return (datatypes::Decimal::compare(*this, rhs) <= 0); + } else - return (decimalComp(rhs) <= 0); + { + if (scale == rhs.scale) + return value <= rhs.value; + else + return (decimalComp(rhs) <= 0); + } } bool operator!=(const IDB_Decimal& rhs) const { - if (scale == rhs.scale) - return value != rhs.value; + if (utils::widthByPrecision(precision) == 16) + { + if (scale == rhs.scale) + return s128Value != rhs.s128Value; + else + return (datatypes::Decimal::compare(*this, rhs) != 0); + } else - return (decimalComp(rhs) != 0); + { + if (scale == rhs.scale) + return value != rhs.value; + else + return (decimalComp(rhs) != 0); + } } int128_t s128Value; diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index fa2c4a77d..e798ead3f 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -143,10 +143,10 @@ namespace cal_impl_if extern bool nonConstFunc(Item_func* ifp); } +using int128_t = __int128; + namespace { - using int128_t = __int128; - using uint128_t = unsigned __int128; // Calpont vtable non-support error message const string infinidb_autoswitch_warning = "The query includes syntax that is not supported by MariaDB Columnstore distributed mode. The execution was switched to standard mode with downgraded performance."; diff --git a/datatypes/decimal.cpp b/tests/csdecimal.cpp similarity index 89% rename from datatypes/decimal.cpp rename to tests/csdecimal.cpp index e5c44d395..91bddcd42 100644 --- a/datatypes/decimal.cpp +++ b/tests/csdecimal.cpp @@ -15,6 +15,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -#include "decimal.h" +#include "gtest/gtest.h" -const datatypes::Decimal someDecimal; +TEST(DataConvertTest, Strtoll128) +{ + EXPECT_TRUE(true); +} diff --git a/utils/common/columnwidth.h b/utils/common/columnwidth.h index 7d5865514..9d02a7312 100644 --- a/utils/common/columnwidth.h +++ b/utils/common/columnwidth.h @@ -18,6 +18,7 @@ #ifndef UTILS_COLWIDTH_H #define UTILS_COLWIDTH_H +#include "branchpred.h" namespace utils { @@ -34,10 +35,12 @@ namespace utils return width <= MAXLEGACYWIDTH; } - // WIP MCOL-641 Replace with template /** @brief Map a DECIMAL precision to data width in bytes */ inline uint8_t widthByPrecision(unsigned p) { + if (LIKELY(p > 18 && p < 39)) + return 16; + switch (p) { case 1: @@ -70,6 +73,7 @@ namespace utils return 16; } } + inline uint8_t precisionByWidth(unsigned w) { switch(w) From 9b714274db4b25186db29810f2e06bee2fdc9d86 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Tue, 3 Mar 2020 11:14:21 -0500 Subject: [PATCH 27/78] MCOL-641 1. Minor refactoring of decimalToString for int128_t. 2. Update unit tests for decimalToString. 3. Allow support for wide decimal in TupleConstantStep::fillInConstants(). --- dbcon/joblist/jlf_execplantojoblist.cpp | 4 +- dbcon/joblist/pcolscan.cpp | 3 +- dbcon/joblist/pcolstep.cpp | 3 +- dbcon/joblist/tupleconstantstep.cpp | 6 +- tests/dataconvert-tests.cpp | 289 +++++++++++++--------- tests/rowgroup-tests.cpp | 299 ++++++++++++++--------- utils/dataconvert/dataconvert.cpp | 216 ++++++++-------- utils/dataconvert/dataconvert.h | 12 +- utils/rowgroup/rowgroup.cpp | 13 +- utils/rowgroup/rowgroup.h | 9 + writeengine/bulk/we_bulkloadbuffer.cpp | 4 +- writeengine/server/we_dmlcommandproc.cpp | 18 +- writeengine/server/we_readthread.cpp | 3 - writeengine/wrapper/writeengine.cpp | 25 +- 14 files changed, 504 insertions(+), 400 deletions(-) diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 3460b0491..8f646de55 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -1893,7 +1893,9 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) if (ct.colDataType == CalpontSystemCatalog::DECIMAL && ct.colWidth == 16) { - dataconvert::atoi128(constval, val128); + bool saturate = false; + val128 = dataconvert::string_to_ll(constval, saturate); + // TODO MCOL-641 check saturate } else { diff --git a/dbcon/joblist/pcolscan.cpp b/dbcon/joblist/pcolscan.cpp index d22fbf8e8..c870eca56 100644 --- a/dbcon/joblist/pcolscan.cpp +++ b/dbcon/joblist/pcolscan.cpp @@ -176,7 +176,8 @@ pColScanStep::pColScanStep( // MCOL-641 WIP else if (fColType.colWidth > 8 && fColType.colDataType != CalpontSystemCatalog::BINARY - && fColType.colDataType != CalpontSystemCatalog::DECIMAL) + && fColType.colDataType != CalpontSystemCatalog::DECIMAL + && fColType.colDataType != CalpontSystemCatalog::UDECIMAL) { fColType.colWidth = 8; fIsDict = true; diff --git a/dbcon/joblist/pcolstep.cpp b/dbcon/joblist/pcolstep.cpp index 296452819..a0b779a8a 100644 --- a/dbcon/joblist/pcolstep.cpp +++ b/dbcon/joblist/pcolstep.cpp @@ -180,7 +180,8 @@ pColStep::pColStep( // WIP MCOL-641 else if (fColType.colWidth > 8 && fColType.colDataType != CalpontSystemCatalog::BINARY - && fColType.colDataType != CalpontSystemCatalog::DECIMAL) + && fColType.colDataType != CalpontSystemCatalog::DECIMAL + && fColType.colDataType != CalpontSystemCatalog::UDECIMAL) { fColType.colWidth = 8; fIsDict = true; diff --git a/dbcon/joblist/tupleconstantstep.cpp b/dbcon/joblist/tupleconstantstep.cpp index 15e5927a1..f6601cb1e 100644 --- a/dbcon/joblist/tupleconstantstep.cpp +++ b/dbcon/joblist/tupleconstantstep.cpp @@ -528,7 +528,11 @@ void TupleConstantStep::fillInConstants(const rowgroup::Row& rowIn, rowgroup::Ro //fRowConst.copyField(rowOut.getData()+2, 0); // hardcoded 2 for rid length for (uint32_t i = 1; i < rowOut.getColumnCount(); i++) - rowIn.copyField(rowOut, i, i - 1); + // WIP MCOL-641 implement copyBinaryField inside copyField + if (UNLIKELY(rowIn.getColumnWidth(i - 1) == 16)) + rowIn.copyBinaryField(rowOut, i, i - 1); + else + rowIn.copyField(rowOut, i, i - 1); //memcpy(rowOut.getData()+rowOut.getOffset(1), rowIn.getData()+2, n); } diff --git a/tests/dataconvert-tests.cpp b/tests/dataconvert-tests.cpp index 461ca28f1..3e7f0d987 100644 --- a/tests/dataconvert-tests.cpp +++ b/tests/dataconvert-tests.cpp @@ -545,126 +545,189 @@ TEST(DataConvertTest, NumberIntValue) EXPECT_TRUE(pushWarning); } -TEST(DataConvertTest, atoiCheck) { - std::vector expected; - std::vector decimalAsStringVec; - decimalAsStringVec.push_back("99999999999999999999999999999999999999"); - decimalAsStringVec.push_back("-99999999999999999999999999999999999999"); - decimalAsStringVec.push_back("0"); - decimalAsStringVec.push_back("-0.042"); - expected.push_back(10000000000000000000ULL*(uint128_t)9999999999999999999ULL - +9999999999999999999ULL); - expected.push_back(-1*expected[0]); - expected.push_back(0); - expected.push_back(-42); - for (int i=0; i < expected.size(); i++) { - int128_t value; - dataconvert::atoi128(decimalAsStringVec[i], value); - EXPECT_EQ(expected[i], value); - EXPECT_NE(expected[i], value-1); - } +TEST(DataConvertTest, DecimalToStringCheckScale0) +{ + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::DECIMAL; + char buf[42]; + string input, expected; + ct.precision = 38; + ct.scale = 0; + int128_t res; + + // test simple values + res = 0; + expected = "0"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = 2; + expected = "2"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -2; + expected = "-2"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = 123; + expected = "123"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -123; + expected = "-123"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + + // test max/min decimal (i.e. 38 9's) + res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + expected = "99999999999999999999999999999999999999"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -res; + expected = "-99999999999999999999999999999999999999"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + + // test trailing zeros + res = 123000; + expected = "123000"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -res; + expected = "-123000"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); +} +TEST(DataConvertTest, DecimalToStringCheckScale10) +{ + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::DECIMAL; + char buf[42]; + string input, expected; + ct.precision = 38; + ct.scale = 10; + int128_t res; + + // test simple values + res = 0; + expected = "0.0000000000"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = 2; + expected = "0.0000000002"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -2; + expected = "-0.0000000002"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = 123; + expected = "0.0000000123"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -123; + expected = "-0.0000000123"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = 12345678901; + expected = "1.2345678901"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -12345678901; + expected = "-1.2345678901"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + + // test max/min decimal (i.e. 38 9's) + res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + expected = "9999999999999999999999999999.9999999999"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -res; + expected = "-9999999999999999999999999999.9999999999"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + + // test trailing zeros + res = 123000; + expected = "0.0000123000"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -res; + expected = "-0.0000123000"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + + // test leading zeros + res = 10000000009; + expected = "1.0000000009"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -res; + expected = "-1.0000000009"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); } -TEST(DataConvertTest, DecimalToStringCheckScale3) { - std::vector expected; - std::vector decimalAsStringVec; - // scale 3 - uint8_t scale3 = 3; - decimalAsStringVec.push_back("99999999999999999999999999999999999999"); - decimalAsStringVec.push_back("-99999999999999999999999999999999999999"); - decimalAsStringVec.push_back("0"); - decimalAsStringVec.push_back("-0.042"); - expected.push_back("99999999999999999999999999999999999.999"); - expected.push_back("-99999999999999999999999999999999999.999"); - expected.push_back("0.000"); - expected.push_back("-0.042"); +TEST(DataConvertTest, DecimalToStringCheckScale38) +{ + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::DECIMAL; char buf[42]; - CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL; - for (int i=0; i < expected.size(); i++) { - int128_t value = -4; - memset(buf, 0, 42); - dataconvert::atoi128(decimalAsStringVec[i], value); - dataconvert::DataConvert::decimalToString(&value, scale3, buf, - utils::precisionByWidth(sizeof(value))+4, type); - EXPECT_EQ(expected[i], std::string(buf)); - } -} + string input, expected; + ct.precision = 38; + ct.scale = 38; + int128_t res; -TEST(DataConvertTest, DecimalToStringCheckScale10) { - std::vector expected; - std::vector decimalAsStringVec; - // scale 3 - uint8_t scale3 = 10; - decimalAsStringVec.push_back("10000000009"); - decimalAsStringVec.push_back("-10000000009"); - decimalAsStringVec.push_back("0"); - decimalAsStringVec.push_back("-10000000010"); - expected.push_back("1.0000000009"); - expected.push_back("-1.0000000009"); - expected.push_back("0.0000000000"); - expected.push_back("-1.0000000010"); - char buf[42]; - CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL; - for (int i=0; i < expected.size(); i++) { - int128_t value = -4; - memset(buf, 0, 42); - dataconvert::atoi128(decimalAsStringVec[i], value); - dataconvert::DataConvert::decimalToString(&value, scale3, buf, - utils::precisionByWidth(sizeof(value))+4, type); - EXPECT_EQ(expected[i], std::string(buf)); - } -} + // test simple values + res = 0; + expected = "0.00000000000000000000000000000000000000"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = 2; + expected = "0.00000000000000000000000000000000000002"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -2; + expected = "-0.00000000000000000000000000000000000002"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = 123; + expected = "0.00000000000000000000000000000000000123"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -123; + expected = "-0.00000000000000000000000000000000000123"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = ((((((int128_t)1234567890 * 10000000000) + 1234567890) * 10000000000) + 1234567890) * 100000000 ) + 12345678; + expected = "0.12345678901234567890123456789012345678"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -res; + expected = "-0.12345678901234567890123456789012345678"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); -TEST(DataConvertTest, DecimalToStringCheckScale37) { - std::vector expected; - std::vector decimalAsStringVec; - uint8_t scale37 = 37; - decimalAsStringVec.push_back("99999999999999999999999999999999999999"); - decimalAsStringVec.push_back("-99999999999999999999999999999999999999"); - decimalAsStringVec.push_back("0"); - decimalAsStringVec.push_back("-42"); - decimalAsStringVec.push_back("-19999999999999999999999999999999999999"); - expected.push_back("9.9999999999999999999999999999999999999"); - expected.push_back("-9.9999999999999999999999999999999999999"); - expected.push_back("0.0000000000000000000000000000000000000"); - expected.push_back("-0.0000000000000000000000000000000000042"); - expected.push_back("-1.9999999999999999999999999999999999999"); - char buf[42]; - CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL; - for (int i=0; i < expected.size(); i++) { - int128_t value = -4; - memset(buf, 0, 41); - dataconvert::atoi128(decimalAsStringVec[i], value); - dataconvert::DataConvert::decimalToString(&value, scale37, buf, - utils::precisionByWidth(sizeof(value))+4, type); - EXPECT_EQ(expected[i], std::string(buf)); - } -} + // test max/min decimal (i.e. 38 9's) + res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + expected = "0.99999999999999999999999999999999999999"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -res; + expected = "-0.99999999999999999999999999999999999999"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); -TEST(DataConvertTest, DecimalToStringCheckScale38) { - std::vector expected; - std::vector decimalAsStringVec; - uint8_t scale38 = 38; - decimalAsStringVec.push_back("99999999999999999999999999999999999999"); - decimalAsStringVec.push_back("-99999999999999999999999999999999999999"); - decimalAsStringVec.push_back("0"); - decimalAsStringVec.push_back("-42"); - expected.push_back("0.99999999999999999999999999999999999999"); - expected.push_back("-0.99999999999999999999999999999999999999"); - expected.push_back("0.00000000000000000000000000000000000000"); - expected.push_back("-0.00000000000000000000000000000000000042"); - char buf[42]; - CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL; - for (int i=0; i < expected.size(); i++) { - int128_t value = -4; - memset(buf, 0, 41); - dataconvert::atoi128(decimalAsStringVec[i], value); - dataconvert::DataConvert::decimalToString(&value, scale38, buf, - utils::precisionByWidth(sizeof(value))+4, type); - EXPECT_EQ(expected[i], std::string(buf)); - } + // test trailing zeros + res = 123000; + expected = "0.00000000000000000000000000000000123000"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -res; + expected = "-0.00000000000000000000000000000000123000"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); } TEST(DataConvertTest, ConvertColumnData) { } - diff --git a/tests/rowgroup-tests.cpp b/tests/rowgroup-tests.cpp index 8fbbe22fc..7d7604b63 100644 --- a/tests/rowgroup-tests.cpp +++ b/tests/rowgroup-tests.cpp @@ -20,6 +20,7 @@ #include "rowgroup.h" #include "columnwidth.h" +#include "widedecimalutils.h" #include "joblisttypes.h" #include "dataconvert.h" @@ -30,125 +31,138 @@ using int128_t = __int128; using uint128_t = unsigned __int128; using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; -class RowDecimalTest : public ::testing::Test { - protected: - void SetUp() override { - uint32_t precision = WIDE_DEC_PRECISION; - uint32_t oid =3001; +class RowDecimalTest : public ::testing::Test +{ +protected: + void SetUp() override + { + uint32_t precision = WIDE_DEC_PRECISION; + uint32_t oid = 3001; - std::vectortypes; - std::vectorprecisionVec; - std::vector roids, tkeys, cscale; - types.push_back(execplan::CalpontSystemCatalog::DECIMAL); - types.push_back(execplan::CalpontSystemCatalog::UDECIMAL); - for (size_t i=0; i <= 3; i++) { + std::vector types; + std::vector precisionVec; + std::vector roids, tkeys, cscale; + std::vector widthVec; types.push_back(execplan::CalpontSystemCatalog::DECIMAL); - } - precisionVec.push_back(precision); - precisionVec.push_back(precision); - precisionVec.push_back(18); - precisionVec.push_back(9); - precisionVec.push_back(4); - precisionVec.push_back(2); - std::vectorwidthVec; - uint32_t offset = INITIAL_ROW_OFFSET; - offsets.push_back(offset); - for (size_t i=0; i < types.size(); i++) { - uint8_t width = utils::widthByPrecision(precisionVec[i]); - widthVec.push_back(width); - offset += width; - offsets.push_back(offset); - roids.push_back(oid+i); - tkeys.push_back(i+1); cscale.push_back(0); - } - /*offsets.push_back(INITIAL_ROW_OFFSET); - offsets.push_back(16+INITIAL_ROW_OFFSET); - offsets.push_back(16*2+INITIAL_ROW_OFFSET); - roids.push_back(oid); roids.push_back(oid+1); - tkeys.push_back(1); tkeys.push_back(1); - cscale.push_back(0); cscale.push_back(0);*/ + types.push_back(execplan::CalpontSystemCatalog::UDECIMAL); + + for (size_t i = 0; i <= 3; i++) + types.push_back(execplan::CalpontSystemCatalog::DECIMAL); + + precisionVec.push_back(precision); + precisionVec.push_back(precision); + precisionVec.push_back(18); + precisionVec.push_back(9); + precisionVec.push_back(4); + precisionVec.push_back(2); + + uint32_t offset = INITIAL_ROW_OFFSET; + offsets.push_back(offset); + + for (size_t i = 0; i < types.size(); i++) + { + uint8_t width = utils::widthByPrecision(precisionVec[i]); + widthVec.push_back(width); + offset += width; + offsets.push_back(offset); + roids.push_back(oid + i); + tkeys.push_back(i + 1); + cscale.push_back(0); + } + + /*offsets.push_back(INITIAL_ROW_OFFSET); + offsets.push_back(16+INITIAL_ROW_OFFSET); + offsets.push_back(16*2+INITIAL_ROW_OFFSET); + roids.push_back(oid); roids.push_back(oid+1); + tkeys.push_back(1); tkeys.push_back(1); + cscale.push_back(0); cscale.push_back(0);*/ - rowgroup::RowGroup inRG(roids.size(), //column count - offsets, //oldOffset - roids, // column oids - tkeys, //keys - types, // types - cscale, //scale - precisionVec, // precision - 20, // sTableThreshold - false //useStringTable - ); + rowgroup::RowGroup inRG(roids.size(), // column count + offsets, // oldOffset + roids, // column oids + tkeys, // keys + types, // types + cscale, // scale + precisionVec, // precision + 20, // sTableThreshold + false // useStringTable + ); - rg = inRG; - rgD.reinit(rg); - rg.setData(&rgD); + rg = rgOut = inRG; + rgD.reinit(rg); + rgDOut.reinit(rgOut); + rg.setData(&rgD); + rgOut.setData(&rgDOut); - rg.initRow(&r); - rg.initRow(&rOutMappingCheck); - rowSize = r.getSize(); - rg.getRow(0, &r); + rg.initRow(&r); + rg.initRow(&rOutMappingCheck); + rowSize = r.getSize(); + rg.getRow(0, &r); - int128_t nullValue = 0; - int128_t bigValue = 0; - uint64_t* uint128_pod = reinterpret_cast(&nullValue); - uint128_pod[0] = joblist::UBIGINTNULL; - uint128_pod[1] = joblist::UBIGINTEMPTYROW; - bigValue = -static_cast(0xFFFFFFFF)*0xFFFFFFFFFFFFFFFF; - sValueVector.push_back(nullValue); - sValueVector.push_back(-42); - sValueVector.push_back(bigValue); - sValueVector.push_back(0); - sValueVector.push_back(nullValue-1); + int128_t nullValue = 0; + int128_t bigValue = 0; - uValueVector.push_back(nullValue); - uValueVector.push_back(42); - uValueVector.push_back(bigValue); - uValueVector.push_back(0); - uValueVector.push_back(nullValue-1); + utils::setWideDecimalNullValue(nullValue); + bigValue = -static_cast(0xFFFFFFFF)*0xFFFFFFFFFFFFFFFF; - s8ValueVector.push_back(joblist::TINYINTNULL); - s8ValueVector.push_back(-0x79); - s8ValueVector.push_back(0); - s8ValueVector.push_back(0x81); - s8ValueVector.push_back(joblist::TINYINTNULL-1); + sValueVector.push_back(nullValue); + sValueVector.push_back(-42); + sValueVector.push_back(bigValue); + sValueVector.push_back(0); + sValueVector.push_back(nullValue-1); - s16ValueVector.push_back(joblist::SMALLINTNULL); - s16ValueVector.push_back(-0x79); - s16ValueVector.push_back(0); - s16ValueVector.push_back(0x81); - s16ValueVector.push_back(joblist::SMALLINTNULL-1); + uValueVector.push_back(nullValue); + uValueVector.push_back(42); + uValueVector.push_back(bigValue); + uValueVector.push_back(0); + uValueVector.push_back(nullValue-1); - s32ValueVector.push_back(joblist::INTNULL); - s32ValueVector.push_back(-0x79); - s32ValueVector.push_back(0); - s32ValueVector.push_back(0x81); - s32ValueVector.push_back(joblist::INTNULL-1); + s8ValueVector.push_back(joblist::TINYINTNULL); + s8ValueVector.push_back(-0x79); + s8ValueVector.push_back(0); + s8ValueVector.push_back(0x81); + s8ValueVector.push_back(joblist::TINYINTNULL-1); - s64ValueVector.push_back(joblist::BIGINTNULL); - s64ValueVector.push_back(-0x79); - s64ValueVector.push_back(0); - s64ValueVector.push_back(0x81); - s64ValueVector.push_back(joblist::BIGINTNULL-1); + s16ValueVector.push_back(joblist::SMALLINTNULL); + s16ValueVector.push_back(-0x79); + s16ValueVector.push_back(0); + s16ValueVector.push_back(0x81); + s16ValueVector.push_back(joblist::SMALLINTNULL-1); - for(size_t i = 0; i < sValueVector.size(); i++) { - r.setBinaryField_offset(&sValueVector[i], - sizeof(sValueVector[0]), offsets[0]); - r.setBinaryField_offset(&uValueVector[i], - sizeof(uValueVector[0]), offsets[1]); - r.setIntField(s64ValueVector[i], 2); - r.setIntField(s32ValueVector[i], 3); - r.setIntField(s16ValueVector[i], 4); - r.setIntField(s8ValueVector[i], 5); - r.nextRow(rowSize); + s32ValueVector.push_back(joblist::INTNULL); + s32ValueVector.push_back(-0x79); + s32ValueVector.push_back(0); + s32ValueVector.push_back(0x81); + s32ValueVector.push_back(joblist::INTNULL-1); + + s64ValueVector.push_back(joblist::BIGINTNULL); + s64ValueVector.push_back(-0x79); + s64ValueVector.push_back(0); + s64ValueVector.push_back(0x81); + s64ValueVector.push_back(joblist::BIGINTNULL-1); + + for (size_t i = 0; i < sValueVector.size(); i++) + { + r.setBinaryField_offset(&sValueVector[i], + sizeof(sValueVector[0]), offsets[0]); + r.setBinaryField_offset(&uValueVector[i], + sizeof(uValueVector[0]), offsets[1]); + r.setIntField(s64ValueVector[i], 2); + r.setIntField(s32ValueVector[i], 3); + r.setIntField(s16ValueVector[i], 4); + r.setIntField(s8ValueVector[i], 5); + r.nextRow(rowSize); + } + + rowCount = sValueVector.size(); } - rowCount = sValueVector.size(); - } - // void TearDown() override {} - rowgroup::Row r; +// void TearDown() override {} + + rowgroup::Row r, rOut; rowgroup::Row rOutMappingCheck; - rowgroup::RowGroup rg; - rowgroup::RGData rgD; + rowgroup::RowGroup rg, rgOut; + rowgroup::RGData rgD, rgDOut; uint32_t rowSize; size_t rowCount; std::vector sValueVector; @@ -160,9 +174,12 @@ class RowDecimalTest : public ::testing::Test { std::vector offsets; }; -TEST_F(RowDecimalTest, NonNULLValuesCheck) { +TEST_F(RowDecimalTest, NonNullValueCheck) +{ rg.getRow(1, &r); - for (size_t i = 1; i <= sValueVector.size(); i++) { + + for (size_t i = 1; i <= sValueVector.size(); i++) + { EXPECT_FALSE(r.isNullValue(0)); EXPECT_FALSE(r.isNullValue(1)); EXPECT_FALSE(r.isNullValue(2)); @@ -173,9 +190,11 @@ TEST_F(RowDecimalTest, NonNULLValuesCheck) { } } -TEST_F(RowDecimalTest, initToNullANDisNullValueValueCheck) { +TEST_F(RowDecimalTest, InitToNullAndIsNullValueCheck) +{ rg.getRow(0, &r); r.initToNull(); + EXPECT_TRUE(r.isNullValue(0)); EXPECT_TRUE(r.isNullValue(1)); EXPECT_TRUE(r.isNullValue(2)); @@ -184,14 +203,16 @@ TEST_F(RowDecimalTest, initToNullANDisNullValueValueCheck) { EXPECT_TRUE(r.isNullValue(5)); } -TEST_F(RowDecimalTest, getBinaryFieldCheck) { +TEST_F(RowDecimalTest, GetBinaryFieldCheck) +{ rg.getRow(0, &r); uint128_t* u128Value; int128_t* s128Value; // std::remove_reference::type uType; // std::remove_reference::type sType; - for (size_t i = 0; i < sValueVector.size(); i++) { + for (size_t i = 0; i < sValueVector.size(); i++) + { s128Value = r.getBinaryField(0); EXPECT_EQ(sValueVector[i], *s128Value); u128Value = r.getBinaryField(1); @@ -204,26 +225,32 @@ TEST_F(RowDecimalTest, getBinaryFieldCheck) { } } -TEST_F(RowDecimalTest, toStringCheck) { +TEST_F(RowDecimalTest, ToStringCheck) +{ std::vector exemplarVector; + exemplarVector.push_back(std::string("0: NULL NULL NULL NULL NULL NULL ")); exemplarVector.push_back(std::string("0: -42 42 -121 -121 -121 -121 ")); exemplarVector.push_back(std::string("0: -79228162495817593515539431425 -79228162495817593515539431425 0 0 0 0 ")); exemplarVector.push_back(std::string("0: 0 0 129 129 129 -127 ")); - exemplarVector.push_back(std::string("0: -3 -3 9223372036854775807 2147483647 32767 127 ")); + exemplarVector.push_back(std::string("0: 170141183460469231731687303715884105727 170141183460469231731687303715884105727 9223372036854775807 2147483647 32767 127 ")); rg.getRow(0, &r); r.initToNull(); - for (auto &el: exemplarVector) { + + for (auto& el: exemplarVector) + { EXPECT_EQ(el, r.toString()); r.nextRow(rowSize); } } -TEST_F(RowDecimalTest, toCSVCheck) { +TEST_F(RowDecimalTest, ToCSVCheck) +{ } -TEST_F(RowDecimalTest, applyMappingCheck) { +TEST_F(RowDecimalTest, ApplyMappingCheck) +{ int mapping[] = {0, 1, -1, -1, -1, -1}; rg.getRow(1, &r); rg.getRow(2, &rOutMappingCheck); @@ -233,11 +260,49 @@ TEST_F(RowDecimalTest, applyMappingCheck) { EXPECT_NE(uValueVector[1], *u128Value); applyMapping(mapping, r, &rOutMappingCheck); s128Value = rOutMappingCheck.getBinaryField(0); - EXPECT_EQ(sValueVector[1], *s128Value); u128Value = rOutMappingCheck.getBinaryField(1); + EXPECT_EQ(sValueVector[1], *s128Value); EXPECT_EQ(uValueVector[1], *u128Value); } -// WIP -TEST_F(RowDecimalTest, RowEqualsCheck) { +TEST_F(RowDecimalTest, CopyBinaryFieldCheck) +{ + int128_t constVal = 1; + int128_t *col1Out, *col1In; + uint128_t *col2Out, *col2In; + rgOut.getRow(0, &rOut); + + for (size_t i = 0; i < sValueVector.size(); i++) + { + rOut.setBinaryField_offset(&constVal, 16, offsets[0]); + rOut.setBinaryField_offset((uint128_t*)&constVal, 16, offsets[1]); + rOut.nextRow(rowSize); + } + + rgOut.initRow(&rOut); + rgOut.getRow(0, &rOut); + rg.getRow(0, &r); + + for (size_t i = 0; i < sValueVector.size(); i++) + { + col1In = r.getBinaryField(0); + col1Out = rOut.getBinaryField(0); + col2In = r.getBinaryField(1); + col2Out = rOut.getBinaryField(1); + EXPECT_NE(*col1In, *col1Out); + EXPECT_NE(*col2In, *col2Out); + r.copyBinaryField(rOut, 0, 0); + r.copyBinaryField(rOut, 1, 1); + col1Out = rOut.getBinaryField(0); + col2Out = rOut.getBinaryField(1); + EXPECT_EQ(*col1In, *col1Out); + EXPECT_EQ(*col2In, *col2Out); + r.nextRow(rowSize); + rOut.nextRow(rowSize); + } +} + +// WIP +TEST_F(RowDecimalTest, RowEqualsCheck) +{ } diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index cb4d7fe27..4bffee3d5 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -125,6 +125,7 @@ const uint64_t columnstore_pow_10[20] = 1000000000000000000ULL, 10000000000000000000ULL }; + template bool from_string(T& t, const std::string& s, std::ios_base & (*f)(std::ios_base&)) { @@ -1249,115 +1250,144 @@ bool stringToTimestampStruct(const string& data, TimeStamp& timeStamp, const str } -size_t DataConvert::writeIntPart(int128_t* dec, char* p, - const uint16_t buflen, +size_t DataConvert::writeIntPart(int128_t* dec, + char* p, + const unsigned int buflen, const uint8_t scale) { - int128_t intPart = *dec; - if (scale) - { - uint8_t maxPowOf10 = sizeof(columnstore_pow_10)/sizeof(columnstore_pow_10[0])-1; - switch (scale/maxPowOf10) + int128_t intPart = *dec; + int128_t high = 0, mid = 0, low = 0; + uint64_t div = 10000000000000000000ULL; + + if (scale) { - case(2): - intPart /= columnstore_pow_10[maxPowOf10]; - intPart /= columnstore_pow_10[maxPowOf10]; - break; - case(1): - intPart /= columnstore_pow_10[maxPowOf10]; - case(0): - intPart /= columnstore_pow_10[scale%maxPowOf10]; + const uint8_t maxPowOf10 = + (sizeof(columnstore_pow_10) / sizeof(columnstore_pow_10[0])) - 1; + + // Assuming scale = [0, 56] + switch (scale / maxPowOf10) + { + case 2: // scale = [38, 56] + intPart /= columnstore_pow_10[maxPowOf10]; + intPart /= columnstore_pow_10[maxPowOf10]; + low = intPart; + break; + case 1: // scale = [19, 37] + intPart /= columnstore_pow_10[maxPowOf10]; + intPart /= columnstore_pow_10[scale % maxPowOf10]; + low = intPart % div; + mid = intPart / div; + break; + case 0: // scale = [0, 18] + intPart /= columnstore_pow_10[scale % maxPowOf10]; + low = intPart % div; + intPart /= div; + mid = intPart % div; + high = intPart / div; + break; + default: + throw QueryDataExcept("writeIntPart() bad scale", formatErr); + } + } + else + { + low = intPart % div; + intPart /= div; + mid = intPart % div; + high = intPart / div; } - } - uint64_t div = 10000000000000000000ULL; - int128_t high = intPart; - int128_t low; - low = high % div; - high /= div; - int128_t mid; - mid = high % div; - high /= div; + // pod[0] is low 8 bytes, pod[1] is high 8 bytes + uint64_t* high_pod = reinterpret_cast(&high); + uint64_t* mid_pod = reinterpret_cast(&mid); + uint64_t* low_pod = reinterpret_cast(&low); + char* original_p = p; - // pod[0] is low 8 byte, pod[1] is high - uint64_t* high_pod = reinterpret_cast(&high); - uint64_t* mid_pod = reinterpret_cast(&mid); - uint64_t* low_pod = reinterpret_cast(&low); - char* original_p = p; - int written = 0; + // WIP replace sprintf with streams + if (high_pod[0] != 0) + { + p += sprintf(p, "%lu", high_pod[0]); + p += sprintf(p, "%019lu", mid_pod[0]); + p += sprintf(p, "%019lu", low_pod[0]); + } + else if (mid_pod[0] != 0) + { + p += sprintf(p, "%lu", mid_pod[0]); + p += sprintf(p, "%019lu", low_pod[0]); + } + else + { + p += sprintf(p, "%lu", low_pod[0]); + } - // WIP replace snprintf with streams - if (high_pod[0] != 0) - { - written = sprintf(p, "%lu", high_pod[0]); - p += written; - written = sprintf(p, "%019lu", mid_pod[0]); - p += written; - sprintf(p, "%019lu", low_pod[0]); - } - else if (mid_pod[0] != 0) - { - written = sprintf(p, "%lu", mid_pod[0]); - p += written; - written = sprintf(p, "%019lu", low_pod[0]); - p += written; - } - else - { - written = sprintf(p, "%lu", low_pod[0]); - p += written; - } + size_t written = p - original_p; - if (buflen <= p-original_p) - { - throw QueryDataExcept("writeIntPart() char buffer overflow.", formatErr); - } - return p-original_p; + if (buflen <= written) + { + throw QueryDataExcept("writeIntPart() char buffer overflow.", formatErr); + } + + return written; } -size_t DataConvert::writeFractionalPart(int128_t* dec, char* p, - const uint16_t buflen, +size_t DataConvert::writeFractionalPart(int128_t* dec, + char* p, + const unsigned int buflen, const uint8_t scale) { int128_t scaleDivisor = 1; - uint8_t maxPowOf10 = sizeof(columnstore_pow_10)/sizeof(columnstore_pow_10[0])-1; - switch (scale/maxPowOf10) + const uint8_t maxPowOf10 = + (sizeof(columnstore_pow_10) / sizeof(columnstore_pow_10[0])) - 1; + + switch (scale / maxPowOf10) { - case(2): + case 2: scaleDivisor *= columnstore_pow_10[maxPowOf10]; scaleDivisor *= columnstore_pow_10[maxPowOf10]; break; - case(1): + case 1: scaleDivisor *= columnstore_pow_10[maxPowOf10]; - case(0): - scaleDivisor *= columnstore_pow_10[scale%maxPowOf10]; + case 0: + scaleDivisor *= columnstore_pow_10[scale % maxPowOf10]; } int128_t fractionalPart = *dec % scaleDivisor; - // divide by the base untill we have non-zero quotinent + + // divide by the base until we have non-zero quotient size_t written = 0; scaleDivisor /= 10; - while (scaleDivisor > 1 && fractionalPart/scaleDivisor == 0) + + char* original_p = p; + + while (scaleDivisor > 1 && fractionalPart / scaleDivisor == 0) { *p++ = '0'; written++; scaleDivisor /= 10; } - written += writeIntPart(&fractionalPart, p, buflen, 0); + + p += writeIntPart(&fractionalPart, p, buflen - written, 0); + + written = p - original_p; + + // this should never be true if (written < scale) { - p += written; for (size_t left = written; left < scale; left++) { *p++ = '0'; } } - return scale; + + return written; } -void DataConvert::toString(int128_t* dec, uint8_t scale, - char *p, unsigned int buflen) +void DataConvert::decimalToString(int128_t* dec, + const uint8_t scale, + char *p, + const unsigned int buflen, + cscDataType colDataType) // colDataType is redundant { char* original_p = p; size_t written = 0; @@ -1379,54 +1409,20 @@ void DataConvert::toString(int128_t* dec, uint8_t scale, if (scale) { *p++ = '.'; - p += writeFractionalPart(dec, p, buflen-(p-original_p), scale); + written = p - original_p; + p += writeFractionalPart(dec, p, buflen - written, scale); } *p = '\0'; - if (buflen <= p-original_p) + written = p - original_p; + + if (buflen <= written) { throw QueryDataExcept("toString() char buffer overflow.", formatErr); } } -// WIP MCOL-641 -void atoi128(const std::string& arg, int128_t& res) -{ - res = 0; - size_t idx = (arg[0] == '-') ? 1 : 0; - for (size_t j = idx; j < arg.size(); j++) - { - // WIP Optimize this - if (LIKELY(arg[j]-'0' >= 0)) - res = res*10 + arg[j] - '0'; - } - if (idx) - res *= -1; -} - -// WIP MCOL-641 -// remove this as we don't need this for wide-DECIMAL -void atoi128(const std::string& arg, uint128_t& res) -{ - res = 0; - for (size_t j = 0; j < arg.size(); j++) - { - // WIP Optimize this - if (LIKELY(arg[j]-'0' >= 0)) - res = res*10 + arg[j] - '0'; - } -} - -void DataConvert::decimalToString(int128_t* valuePtr, - uint8_t scale, - char* buf, - unsigned int buflen, - cscDataType colDataType) // We don't need the last one -{ - toString(valuePtr, scale, buf, buflen); -} - boost::any DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, const std::string& dataOrig, bool& pushWarning, diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 822d36e52..f70ec7f87 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -152,8 +152,6 @@ struct Int128Pod_struct typedef Int128Pod_struct Int128Pod_t; -void atoi128(const std::string& arg, int128_t& res); - enum CalpontDateTimeFormat { CALPONTDATE_ENUM = 1, // date format is: "YYYY-MM-DD" @@ -177,9 +175,6 @@ struct MySQLTime } }; -void atoi128(const std::string& arg, int128_t& res); -void atoi128(const std::string& arg, uint128_t& res); - /** * This function converts the timezone represented as a string * in the format "+HH:MM" or "-HH:MM" to a signed offset in seconds @@ -1041,12 +1036,11 @@ public: EXPORT static bool isNullData(execplan::ColumnResult* cr, int rownum, execplan::CalpontSystemCatalog::ColType colType); static inline std::string decimalToString(int64_t value, uint8_t scale, cscDataType colDataType); static inline void decimalToString(int64_t value, uint8_t scale, char* buf, unsigned int buflen, cscDataType colDataType); - static void decimalToString(int128_t* value, uint8_t scale, char* buf, unsigned int buflen, cscDataType colDataType); - static void toString(int128_t* dec, uint8_t scale, char* p, unsigned int buflen); - static size_t writeIntPart(int128_t* dec, char* p, const uint16_t buflen, + static void decimalToString(int128_t* dec, const uint8_t scale, char* buf, const unsigned int buflen, cscDataType colDataType); + static size_t writeIntPart(int128_t* dec, char* p, const unsigned int buflen, const uint8_t scale); - static size_t writeFractionalPart(int128_t* dec, char* p, const uint16_t buflen, + static size_t writeFractionalPart(int128_t* dec, char* p, const unsigned int buflen, const uint8_t scale); static inline void int128Max(int128_t& i) diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 3fe84f92f..9ee1b5fbe 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -633,22 +633,23 @@ string Row::toString() const os << " " << dec; break; } + // WIP MCOL-641 case CalpontSystemCatalog::BINARY: std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; break; - // WIP case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - if (colWidths[i] == sizeof(int128_t)) + if (utils::isWide(colWidths[i])) { - char *buf = (char*)alloca(precision[i] + 3); + unsigned int buflen = precision[i] + 3; + char *buf = (char*)alloca(buflen); // empty the buffer dataconvert::DataConvert::decimalToString(getBinaryField(i), - scale[i], buf, precision[i]+3, types[i]); + scale[i], buf, buflen, types[i]); os << buf << " "; break; } - // fallback if the the legacy DECIMAL + // fallthrough if the legacy DECIMAL default: os << getIntField(i) << " "; break; @@ -850,10 +851,8 @@ void Row::initToNull() break; case 16 : - { utils::setWideDecimalNullValue(reinterpret_cast(data[offsets[i]])); break; - } default: *((int64_t*) &data[offsets[i]]) = static_cast(joblist::BIGINTNULL); break; diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 9fa0b98fc..f3c7fec39 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -475,6 +475,9 @@ public: // that's not string-table safe, this one is inline void copyField(Row& dest, uint32_t destIndex, uint32_t srcIndex) const; + // WIP MCOL-641 + inline void copyBinaryField(Row& dest, uint32_t destIndex, uint32_t srcIndex) const; + std::string toString() const; std::string toCSV() const; @@ -1229,6 +1232,12 @@ inline void Row::copyField(Row& out, uint32_t destIndex, uint32_t srcIndex) cons out.setIntField(getIntField(srcIndex), destIndex); } +// WIP MCOL-641 +inline void Row::copyBinaryField(Row& out, uint32_t destIndex, uint32_t srcIndex) const +{ + out.setBinaryField(getBinaryField(srcIndex), 16, destIndex); +} + inline void Row::setRid(uint64_t rid) { *((uint16_t*) data) = rid & 0xffff; diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index 1f199fac8..b899dcd8b 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -994,7 +994,9 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, } else { - dataconvert::atoi128(string(field), bigllVal); + bool saturate = false; + bigllVal = dataconvert::string_to_ll(string(field), saturate); + // TODO MCOL-641 check saturate } } else diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index ff274d677..3f3f732a2 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -815,24 +815,19 @@ uint8_t WE_DMLCommandProc::rollbackVersion(ByteStream& bs, std::string& err) uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId) { int rc = 0; - //cout << "processBatchInsert received bytestream length " << bs.length() << endl; InsertDMLPackage insertPkg; ByteStream::quadbyte tmp32; bs >> tmp32; - //cout << "processBatchInsert got transaction id " << tmp32 << endl; bs >> PMId; - //cout << "processBatchInsert gor PMId " << PMId << endl; insertPkg.read( bs); uint32_t sessionId = insertPkg.get_SessionID(); - //cout << " processBatchInsert for session " << sessionId << endl; DMLTable* tablePtr = insertPkg.get_Table(); bool isAutocommitOn = insertPkg.get_isAutocommitOn(); if (idbdatafile::IDBPolicy::useHdfs()) isAutocommitOn = true; - //cout << "This session isAutocommitOn is " << isAutocommitOn << endl; BRM::TxnID txnid; txnid.id = tmp32; txnid.valid = true; @@ -858,7 +853,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std:: try { ridList = systemCatalogPtr->columnRIDs(tableName, true); - roPair = systemCatalogPtr->tableRID( tableName); + roPair = systemCatalogPtr->tableRID(tableName); } catch (std::exception& ex) { @@ -867,7 +862,6 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std:: return rc; } - std::vector dctnryStoreOids(ridList.size()) ; std::vector columns; DctnryStructList dctnryList; @@ -919,14 +913,10 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std:: if (i == 0) { rc = pDBRootExtentTracker->selectFirstSegFile(dbRootExtent, bFirstExtentOnThisPM, bEmptyPM, trkErrMsg); - /* cout << "bEmptyPM = " << (int) bEmptyPM << " bFirstExtentOnThisPM= " << (int)bFirstExtentOnThisPM << - " oid:dbroot:hwm = " << ridList[i].objnum << ":"< 0) { if (colValuesList[0].size() > 0) @@ -1361,7 +1348,6 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std:: if ( isWarningSet && ( rc == NO_ERROR ) ) { rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; - //cout << "Got warning" << endl; Message::Args args; string cols = "'" + colNames[0] + "'"; diff --git a/writeengine/server/we_readthread.cpp b/writeengine/server/we_readthread.cpp index f27ed9795..f262817f5 100644 --- a/writeengine/server/we_readthread.cpp +++ b/writeengine/server/we_readthread.cpp @@ -154,10 +154,7 @@ void DmlReadThread::operator()() case WE_SVR_BATCH_INSERT: { - //timer.start("processBatchInsert"); rc = fWeDMLprocessor->processBatchInsert(ibs, errMsg, PMId); - //timer.stop("processBatchInsert"); - //cout << "fWeDMLprocessor " << fWeDMLprocessor << " is processing batchinsert ..." << endl; break; } diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 559dbe616..f4ba6c314 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -1129,7 +1129,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, //---------------------------------------------------------------------- if (bFirstExtentOnThisPM) { - //cout << "bFirstExtentOnThisPM is " << bFirstExtentOnThisPM << endl; std::vector cols; BRM::CreateStripeColumnExtentsArgIn createStripeColumnExtentsArgIn; @@ -1234,8 +1233,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, colStructList[i].fColPartition = tmpExtentInfo[currentDBrootIdx].fPartition; colStructList[i].fColSegment = tmpExtentInfo[currentDBrootIdx].fSegment; colStructList[i].fColDbRoot = tmpExtentInfo[currentDBrootIdx].fDbRoot; - //cout << "Load from dbrootExtenttracker oid:dbroot:part:seg = " <getDBRootExtentList(); aExt.isNewExt = false; aExt.hwm = tmpExtentInfo[currentDBrootIdx].fLocalHwm; - //cout << "oid " << colStructList[i].dataOid << " gets hwm " << aExt.hwm << endl; } aExt.current = true; aColExtsInfo.push_back(aExt); - //cout << "get from extentinfo oid:hwm = " << colStructList[i].dataOid << ":" << aExt.hwm << endl; } tableMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); @@ -1387,7 +1381,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, if (it != aColExtsInfo.end()) { hwm = it->hwm; - //cout << "Got from colextinfo hwm for oid " << colStructList[colId].dataOid << " is " << hwm << " and seg is " << colStructList[0].fColSegment << endl; } oldHwm = hwm; //Save this info for rollback @@ -1422,8 +1415,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, curCol, (uint64_t)totalRow, rowIdArray, hwm, newExtent, rowsLeft, newHwm, newFile, newColStructList, newDctnryStructList, dbRootExtentTrackers, insertSelect, true, tableOid, isFirstBatchPm); - //cout << "after allocrowid, total row = " < 0) && (rowIdArray[totalRow - rowsLeft - 1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK)) { - for (unsigned k=0; klength() == 0) { @@ -1560,7 +1551,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, if (rc != NO_ERROR) return rc; - for (uint32_t rows = 0; rows < rowsLeft; rows++) + for (uint32_t rows = 0; rows < rowsLeft; rows++) { if (dctStr_iter->length() == 0) { @@ -1627,13 +1618,11 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, lastRidNew = rowIdArray[totalRow - 1]; } - //cout << "rowid allocated is " << lastRid << endl; //if a new extent is created, all the columns in this table should have their own new extent //First column already processed //@Bug 1701. Close the file (if uncompressed) m_colOp[op(curCol.compressionType)]->clearColumn(curCol); - //cout << "Saving hwm info for new ext batch" << endl; //Update hwm to set them in the end bool succFlag = false; unsigned colWidth = 0; @@ -1663,7 +1652,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, colWidth = colStructList[i].colWidth; succFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); - //cout << "insertcolumnrec oid:rid:fbo:oldhwm = " << colStructList[i].dataOid << ":" << lastRid << ":" << curFbo << ":" << oldHwm << endl; if (succFlag) { if ((HWM)curFbo >= oldHwm) @@ -1677,8 +1665,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, it->current = false; } - //cout << "updated old ext info for oid " << colStructList[i].dataOid << " dbroot:part:seg:hwm:current = " - //<< it->dbRoot<<":"<partNum<<":"<segNum<<":"<hwm<<":"<< it->current<< " and newExtent is " << newExtent << endl; } else return ERR_INVALID_PARAM; @@ -1734,7 +1720,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, colNewValueList.push_back(newColTupleList); newColTupleList.clear(); - //upate the oldvalue list for the old extent + //update the oldvalue list for the old extent for (uint64_t j = 0; j < (totalRow - rowsLeft); j++) { firstPartTupleList.push_back(colTupleList[j]); @@ -1749,7 +1735,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, #ifdef PROFILE timer.start("writeColumnRec"); #endif -//cout << "Writing column record" << endl; if (rc == NO_ERROR) { @@ -3979,7 +3964,7 @@ void WriteEngineWrapper::printInputValue(const ColStructList& colStructList, // WIP replace with a single call char buf[utils::MAXLENGTH16BYTES]; int128_t val = boost::any_cast(curTuple.data); - dataconvert::DataConvert::toString(&val, 0, buf, utils::MAXLENGTH16BYTES); + dataconvert::DataConvert::decimalToString(&val, 0, buf, utils::MAXLENGTH16BYTES, curColStruct.colDataType); curStr.assign(buf); } else if (curTuple.data.type() == typeid(double)) From 62d0c82d75a8b3da7559cf4e48ed56334560c481 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Tue, 10 Mar 2020 19:41:25 -0400 Subject: [PATCH 28/78] MCOL-641 1. Templatized convertValueNum() function. 2. Allocate int128_t buffers in batchprimitiveprocessor if a query involves wide decimal columns. --- dbcon/execplan/calpontsystemcatalog.h | 6 ++ dbcon/joblist/batchprimitiveprocessor-jl.cpp | 21 ++++- dbcon/joblist/batchprimitiveprocessor-jl.h | 5 ++ dbcon/joblist/jlf_common.cpp | 2 +- dbcon/joblist/jlf_execplantojoblist.cpp | 58 +++++++------ dbcon/joblist/lbidlist.cpp | 8 +- dbcon/joblist/pcolstep.cpp | 4 +- dbcon/joblist/primitivemsg.h | 17 ++-- dbcon/joblist/primitivestep.h | 2 +- .../primproc/batchprimitiveprocessor.cpp | 30 +++++-- primitives/primproc/batchprimitiveprocessor.h | 5 +- primitives/primproc/columncommand.cpp | 42 ++++----- primitives/primproc/columncommand.h | 1 + primitives/primproc/filtercommand.cpp | 86 ++++++++++++++++++- primitives/primproc/filtercommand.h | 5 ++ primitives/primproc/passthrucommand.cpp | 15 +--- primitives/primproc/primproc.cpp | 4 +- utils/common/columnwidth.h | 8 ++ utils/joiner/tuplejoiner.cpp | 2 +- utils/messageqcpp/bytestream.cpp | 2 +- utils/messageqcpp/bytestream.h | 8 +- utils/rowgroup/rowgroup.cpp | 2 +- writeengine/wrapper/writeengine.cpp | 7 +- 23 files changed, 235 insertions(+), 105 deletions(-) diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index 4450ce429..3cd285f44 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -44,6 +44,7 @@ #include "bytestream.h" #include "joblisttypes.h" #include "stdexcept" +#include "widedecimalutils.h" #undef min #undef max @@ -1046,6 +1047,11 @@ inline bool isSignedInteger(const execplan::CalpontSystemCatalog::ColDataType ty } } +inline bool isNull(int128_t val, const execplan::CalpontSystemCatalog::ColType& ct) +{ + return utils::isWideDecimalNullValue(val); +} + inline bool isNull(int64_t val, const execplan::CalpontSystemCatalog::ColType& ct) { bool ret = false; diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.cpp b/dbcon/joblist/batchprimitiveprocessor-jl.cpp index 28cd1b66b..234c8d4eb 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.cpp +++ b/dbcon/joblist/batchprimitiveprocessor-jl.cpp @@ -60,6 +60,7 @@ BatchPrimitiveProcessorJL::BatchPrimitiveProcessorJL(const ResourceManager* rm) baseRid(0), ridCount(0), needStrValues(false), + hasWideDecimalType(false), filterCount(0), projectCount(0), needRidsAtDelivery(false), @@ -100,6 +101,8 @@ void BatchPrimitiveProcessorJL::addFilterStep(const pColScanStep& scan, vectorgetWidth())) + hasWideDecimalType = true; idbassert(sessionID == scan.sessionId()); } @@ -114,6 +117,9 @@ void BatchPrimitiveProcessorJL::addFilterStep(const PseudoColStep& pcs) cc->setStepUuid(uuid); filterSteps.push_back(cc); filterCount++; + // TODO MCOL-641 How do we get to this execution path? + //if (utils::isWide(cc->getWidth())) + // hasWideDecimalType = true; idbassert(sessionID == pcs.sessionId()); } @@ -128,6 +134,8 @@ void BatchPrimitiveProcessorJL::addFilterStep(const pColStep& step) cc->setStepUuid(uuid); filterSteps.push_back(cc); filterCount++; + if (utils::isWide(cc->getWidth())) + hasWideDecimalType = true; idbassert(sessionID == step.sessionId()); } @@ -182,6 +190,9 @@ void BatchPrimitiveProcessorJL::addProjectStep(const PseudoColStep& step) colWidths.push_back(cc->getWidth()); tupleLength += cc->getWidth(); projectCount++; + // TODO MCOL-641 How do we get to this execution path? + //if (utils::isWide(cc->getWidth())) + // hasWideDecimalType = true; idbassert(sessionID == step.sessionId()); } @@ -198,6 +209,8 @@ void BatchPrimitiveProcessorJL::addProjectStep(const pColStep& step) colWidths.push_back(cc->getWidth()); tupleLength += cc->getWidth(); projectCount++; + if (utils::isWide(cc->getWidth())) + hasWideDecimalType = true; idbassert(sessionID == step.sessionId()); } @@ -215,6 +228,9 @@ void BatchPrimitiveProcessorJL::addProjectStep(const PassThruStep& step) tupleLength += cc->getWidth(); projectCount++; + if (utils::isWide(cc->getWidth())) + hasWideDecimalType = true; + if (filterCount == 0 && !sendRowGroups) sendValues = true; @@ -958,7 +974,7 @@ void BatchPrimitiveProcessorJL::createBPP(ByteStream& bs) const { ISMPacketHeader ism; uint32_t i; - uint8_t flags = 0; + uint16_t flags = 0; ism.Command = BATCH_PRIMITIVE_CREATE; @@ -994,6 +1010,9 @@ void BatchPrimitiveProcessorJL::createBPP(ByteStream& bs) const if (sendTupleJoinRowGroupData) flags |= JOIN_ROWGROUP_DATA; + if (hasWideDecimalType) + flags |= HAS_WIDE_DECIMAL; + bs << flags; bs << bop; diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.h b/dbcon/joblist/batchprimitiveprocessor-jl.h index b8b7e14c5..1cd07d970 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.h +++ b/dbcon/joblist/batchprimitiveprocessor-jl.h @@ -281,10 +281,15 @@ private: uint16_t relRids[LOGICAL_BLOCK_RIDS]; boost::scoped_array absRids; + // TODO MCOL-641 Do we need uint128_t buffers here? + // When would sendValues=true, in which case values[] + // is sent to primproc? uint64_t values[LOGICAL_BLOCK_RIDS]; uint16_t ridCount; bool needStrValues; + bool hasWideDecimalType; + std::vector filterSteps; std::vector projectSteps; //@bug 1136 diff --git a/dbcon/joblist/jlf_common.cpp b/dbcon/joblist/jlf_common.cpp index d6bd3e5a6..fb4678396 100644 --- a/dbcon/joblist/jlf_common.cpp +++ b/dbcon/joblist/jlf_common.cpp @@ -332,7 +332,7 @@ string extractTableAlias(const SSC& sc) //------------------------------------------------------------------------------ CalpontSystemCatalog::OID isDictCol(const CalpontSystemCatalog::ColType& colType) { - if (colType.colDataType == CalpontSystemCatalog::BINARY) return 0; + if (utils::isWideDecimalType(colType)) return 0; if (colType.colWidth > 8) return colType.ddn.dictOID; diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 8f646de55..70ec8c599 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -88,7 +88,7 @@ using namespace logging; #include "jlf_common.h" #include "jlf_subquery.h" #include "jlf_tuplejoblist.h" - +#include "columnwidth.h" namespace { @@ -312,11 +312,17 @@ int64_t valueNullNum(const CalpontSystemCatalog::ColType& ct, const string& time return n; } -int64_t convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct, bool isNull, uint8_t& rf, const string& timeZone) +template +void convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct, bool isNull, uint8_t& rf, const string& timeZone, T& v) { - if (str.size() == 0 || isNull ) return valueNullNum(ct, timeZone); + if (str.size() == 0 || isNull ) + { + v = valueNullNum(ct, timeZone); + return; + } - int64_t v = 0; + + v = 0; rf = 0; bool pushWarning = false; boost::any anyVal = DataConvert::convertColumnData(ct, str, pushWarning, timeZone, false, true, false); @@ -450,8 +456,10 @@ int64_t convertValueNum(const string& str, const CalpontSystemCatalog::ColType& #else v = boost::any_cast(anyVal); #endif - else + else if (ct.colWidth == execplan::CalpontSystemCatalog::EIGHT_BYTE) v = boost::any_cast(anyVal); + else + v = boost::any_cast(anyVal); break; @@ -485,8 +493,6 @@ int64_t convertValueNum(const string& str, const CalpontSystemCatalog::ColType& rf = (data[0] == '-') ? ROUND_NEG : ROUND_POS; } - - return v; } //TODO: make this totaly case-insensitive @@ -1840,8 +1846,8 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) { // @bug 1151 string longer than colwidth of char/varchar. int64_t value = 0; + int128_t value128 = 0; uint8_t rf = 0; - unsigned __int128 val128 = 0; #ifdef FAILED_ATOI_IS_ZERO //if cvn throws (because there's non-digit data in the string, treat that as zero rather than @@ -1849,7 +1855,7 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) try { bool isNull = ConstantColumn::NULLDATA == cc->type(); - value = convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone); + convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value); if (ct.colDataType == CalpontSystemCatalog::FLOAT && !isNull) { @@ -1887,21 +1893,14 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) } #else + bool isNull = ConstantColumn::NULLDATA == cc->type(); // WIP MCOL-641 width check must be a f() not a literal // make a template from convertValueNum to avoid extra if // this condition doesn't support UDECIMAL - if (ct.colDataType == CalpontSystemCatalog::DECIMAL && - ct.colWidth == 16) - { - bool saturate = false; - val128 = dataconvert::string_to_ll(constval, saturate); - // TODO MCOL-641 check saturate - } + if (utils::isWideDecimalType(ct)) + convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value128); else - { - bool isNull = ConstantColumn::NULLDATA == cc->type(); - value = convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone); - } + convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value); if (ct.colDataType == CalpontSystemCatalog::FLOAT && !isNull) { @@ -1935,10 +1934,8 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) if (sc->isColumnStore()) { - // WIP MCOL-641 - if (ct.colDataType == CalpontSystemCatalog::DECIMAL && - ct.colWidth == 16) - pcs->addFilter(cop, val128, rf); + if (utils::isWideDecimalType(ct)) + pcs->addFilter(cop, value128, rf); else pcs->addFilter(cop, value, rf); } @@ -3008,12 +3005,17 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo) //add each filter to pColStep int8_t cop = op2num(sop); int64_t value = 0; + int128_t value128 = 0; string constval = cc->constval(); // @bug 1151 string longer than colwidth of char/varchar. uint8_t rf = 0; bool isNull = ConstantColumn::NULLDATA == cc->type(); - value = convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone); + + if (utils::isWideDecimalType(ct)) + convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value128); + else + convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value); if (ct.colDataType == CalpontSystemCatalog::FLOAT && !isNull) { @@ -3030,7 +3032,10 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo) if (ConstantColumn::NULLDATA == cc->type() && (opeq == *sop || opne == *sop)) cop = COMPARE_NIL; - pcs->addFilter(cop, value, rf); + if (utils::isWideDecimalType(ct)) + pcs->addFilter(cop, value128, rf); + else + pcs->addFilter(cop, value, rf); } } @@ -3453,7 +3458,6 @@ JLF_ExecPlanToJobList::walkTree(execplan::ParseTree* n, JobInfo& jobInfo) break; case CONSTANTFILTER: - //cout << "ConstantFilter" << endl; jsv = doConstantFilter(dynamic_cast(tn), jobInfo); JLF_ExecPlanToJobList::addJobSteps(jsv, jobInfo, false); break; diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index d0f97f6bf..c34367b8d 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -28,6 +28,7 @@ #include "brm.h" #include "brmtypes.h" #include "dataconvert.h" +#include "columnwidth.h" #define IS_VERBOSE (fDebug >= 4) #define IS_DETAIL (fDebug >= 3) @@ -808,7 +809,12 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, // Should we also check for empty here? // TODO MCOL-641 - if (isNull(value, ct)) // This will work even if the data column is unsigned. + if (utils::isWideDecimalType(ct)) + { + if (isNull(bigValue, ct)) + continue; + } + else if (isNull(value, ct)) // This will work even if the data column is unsigned. { continue; } diff --git a/dbcon/joblist/pcolstep.cpp b/dbcon/joblist/pcolstep.cpp index a0b779a8a..fb8c1c209 100644 --- a/dbcon/joblist/pcolstep.cpp +++ b/dbcon/joblist/pcolstep.cpp @@ -635,13 +635,13 @@ void pColStep::addFilter(int8_t COP, int64_t value, uint8_t roundFlag) } // WIP MCOL-641 -void pColStep::addFilter(int8_t COP, unsigned __int128 value, uint8_t roundFlag) +void pColStep::addFilter(int8_t COP, const int128_t& value, uint8_t roundFlag) { fFilterString << (uint8_t) COP; fFilterString << roundFlag; // bitwise copies into the filter ByteStream - fFilterString << value; + fFilterString << *reinterpret_cast(&value); fFilterCount++; } diff --git a/dbcon/joblist/primitivemsg.h b/dbcon/joblist/primitivemsg.h index a75367978..f82a5eeb7 100644 --- a/dbcon/joblist/primitivemsg.h +++ b/dbcon/joblist/primitivemsg.h @@ -190,14 +190,15 @@ enum ISMPACKETCOMMAND #undef PRIM_DELIVERBASE /* Flags for BPP messages */ -const uint8_t NEED_STR_VALUES = 0x01; //1; -const uint8_t GOT_ABS_RIDS = 0x02; //2; -const uint8_t GOT_VALUES = 0x04; //4; -const uint8_t LBID_TRACE = 0x08; //8; -const uint8_t HAS_JOINER = 0x10; //16; -const uint8_t SEND_RIDS_AT_DELIVERY = 0x20; //32; -const uint8_t HAS_ROWGROUP = 0x40; //64; -const uint8_t JOIN_ROWGROUP_DATA = 0x80; //128 +const uint16_t NEED_STR_VALUES = 0x01; //1; +const uint16_t GOT_ABS_RIDS = 0x02; //2; +const uint16_t GOT_VALUES = 0x04; //4; +const uint16_t LBID_TRACE = 0x08; //8; +const uint16_t HAS_JOINER = 0x10; //16; +const uint16_t SEND_RIDS_AT_DELIVERY = 0x20; //32; +const uint16_t HAS_ROWGROUP = 0x40; //64; +const uint16_t JOIN_ROWGROUP_DATA = 0x80; //128 +const uint16_t HAS_WIDE_DECIMAL = 0x100; //256; //TODO: put this in a namespace to stop global ns pollution enum PrimFlags diff --git a/dbcon/joblist/primitivestep.h b/dbcon/joblist/primitivestep.h index d22ac2821..603bc6ebd 100644 --- a/dbcon/joblist/primitivestep.h +++ b/dbcon/joblist/primitivestep.h @@ -196,7 +196,7 @@ public: void addFilter(int8_t COP, int64_t value, uint8_t roundFlag = 0); void addFilter(int8_t COP, float value); // WIP MCOL-641 - void addFilter(int8_t COP, unsigned __int128 value, uint8_t roundFlag = 0); + void addFilter(int8_t COP, const int128_t& value, uint8_t roundFlag = 0); /** @brief Sets the DataList to get RID values from. * diff --git a/primitives/primproc/batchprimitiveprocessor.cpp b/primitives/primproc/batchprimitiveprocessor.cpp index 99bda331a..4a2021daa 100644 --- a/primitives/primproc/batchprimitiveprocessor.cpp +++ b/primitives/primproc/batchprimitiveprocessor.cpp @@ -100,6 +100,7 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor() : baseRid(0), ridCount(0), needStrValues(false), + hasWideDecimalType(false), filterCount(0), projectCount(0), sendRidsAtDelivery(false), @@ -145,6 +146,7 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor(ByteStream& b, double prefetch, baseRid(0), ridCount(0), needStrValues(false), + hasWideDecimalType(false), filterCount(0), projectCount(0), sendRidsAtDelivery(false), @@ -218,6 +220,7 @@ void BatchPrimitiveProcessor::initBPP(ByteStream& bs) { uint32_t i; uint8_t tmp8; + uint16_t tmp16; Command::CommandType type; bs.advance(sizeof(ISMPacketHeader)); // skip the header @@ -229,15 +232,16 @@ void BatchPrimitiveProcessor::initBPP(ByteStream& bs) bs >> uniqueID; bs >> versionInfo; - bs >> tmp8; - needStrValues = tmp8 & NEED_STR_VALUES; - gotAbsRids = tmp8 & GOT_ABS_RIDS; - gotValues = tmp8 & GOT_VALUES; - LBIDTrace = tmp8 & LBID_TRACE; - sendRidsAtDelivery = tmp8 & SEND_RIDS_AT_DELIVERY; - doJoin = tmp8 & HAS_JOINER; - hasRowGroup = tmp8 & HAS_ROWGROUP; - getTupleJoinRowGroupData = tmp8 & JOIN_ROWGROUP_DATA; + bs >> tmp16; + needStrValues = tmp16 & NEED_STR_VALUES; + gotAbsRids = tmp16 & GOT_ABS_RIDS; + gotValues = tmp16 & GOT_VALUES; + LBIDTrace = tmp16 & LBID_TRACE; + sendRidsAtDelivery = tmp16 & SEND_RIDS_AT_DELIVERY; + doJoin = tmp16 & HAS_JOINER; + hasRowGroup = tmp16 & HAS_ROWGROUP; + getTupleJoinRowGroupData = tmp16 & JOIN_ROWGROUP_DATA; + hasWideDecimalType = tmp16 & HAS_WIDE_DECIMAL; // This used to signify that there was input row data from previous jobsteps, and // it never quite worked right. No need to fix it or update it; all BPP's have started @@ -1019,6 +1023,8 @@ void BatchPrimitiveProcessor::initProcessor() fFiltRidCount[i] = 0; fFiltCmdRids[i].reset(new uint16_t[LOGICAL_BLOCK_RIDS]); fFiltCmdValues[i].reset(new int64_t[LOGICAL_BLOCK_RIDS]); + if (hasWideDecimalType) + fFiltCmdBinaryValues[i].reset(new int128_t[LOGICAL_BLOCK_RIDS]); if (filtOnString) fFiltStrValues[i].reset(new string[LOGICAL_BLOCK_RIDS]); } @@ -1539,6 +1545,11 @@ void BatchPrimitiveProcessor::execute() projectSteps[j]->projectIntoRowGroup(fe1Input, projectForFE1[j]); for (j = 0; j < ridCount; j++, fe1In.nextRow()) + // TODO MCOL-641 + // WHERE clause on a numeric and a non-numeric column + // leads to this execution path: + // SELECT a, b from t1 where a!=b + // Here, a is e.g., decimal(38), b is varchar(15) if (fe1->evaluate(&fe1In)) { applyMapping(fe1ToProjection, fe1In, &fe1Out); @@ -2339,6 +2350,7 @@ SBPP BatchPrimitiveProcessor::duplicate() bpp->stepID = stepID; bpp->uniqueID = uniqueID; bpp->needStrValues = needStrValues; + bpp->hasWideDecimalType = hasWideDecimalType; bpp->gotAbsRids = gotAbsRids; bpp->gotValues = gotValues; bpp->LBIDTrace = LBIDTrace; diff --git a/primitives/primproc/batchprimitiveprocessor.h b/primitives/primproc/batchprimitiveprocessor.h index e8a9b9384..0ffec0044 100644 --- a/primitives/primproc/batchprimitiveprocessor.h +++ b/primitives/primproc/batchprimitiveprocessor.h @@ -204,11 +204,13 @@ private: uint64_t baseRid; // first rid of the logical block uint16_t relRids[LOGICAL_BLOCK_RIDS]; - int64_t values[LOGICAL_BLOCK_RIDS]; + int64_t values[LOGICAL_BLOCK_RIDS]; + int128_t binaryValues[LOGICAL_BLOCK_RIDS]; boost::scoped_array absRids; boost::scoped_array strValues; uint16_t ridCount; bool needStrValues; + bool hasWideDecimalType; /* Common space for primitive data */ static const uint32_t BUFFER_SIZE = 131072; @@ -274,6 +276,7 @@ private: bool filtOnString; boost::scoped_array fFiltCmdRids[2]; boost::scoped_array fFiltCmdValues[2]; + boost::scoped_array fFiltCmdBinaryValues[2]; boost::scoped_array fFiltStrValues[2]; uint64_t fFiltRidCount[2]; diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index 3dbd2fa61..39009bfbf 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -71,7 +71,6 @@ ColumnCommand::~ColumnCommand() { } void ColumnCommand::_execute() { -// cout << "CC: executing" << endl; if (_isScan) makeScanMsg(); else if (bpp->ridCount == 0) // this would cause a scan @@ -93,11 +92,20 @@ void ColumnCommand::_execute() void ColumnCommand::execute() { if (fFilterFeeder == LEFT_FEEDER) + { values = bpp->fFiltCmdValues[0].get(); + binaryValues = bpp->fFiltCmdBinaryValues[0].get(); + } else if (fFilterFeeder == RIGHT_FEEDER) + { values = bpp->fFiltCmdValues[1].get(); + binaryValues = bpp->fFiltCmdBinaryValues[1].get(); + } else + { values = bpp->values; + binaryValues = bpp->binaryValues; + } _execute(); } @@ -258,7 +266,6 @@ void ColumnCommand::issuePrimitive() loadData(); -// cout << "issuing primitive for LBID " << primMsg->LBID << endl; if (!suppressFilter) bpp->pp.setParsedColumnFilter(parsedColumnFilter); else @@ -295,7 +302,6 @@ void ColumnCommand::process_OT_BOTH() bpp->ridCount = outMsg->NVALS; bpp->ridMap = outMsg->RidFlags; -// cout << "rid Count is " << bpp->ridCount << endl; /* this is verbose and repetative to minimize the work per row */ switch (colType.colWidth) @@ -308,24 +314,12 @@ void ColumnCommand::process_OT_BOTH() bpp->relRids[i] = *((uint16_t*) &bpp->outputMsg[pos]); pos += 2; - // WIP - // values[i] is 8 Bytes wide so coping the pointer to bpp->outputMsg[pos] and crossing fingers - // I dont know the liveness of bpp->outputMsg but also I dont know if there is other memory area I can use - values[i] = (int64_t) &bpp->outputMsg[pos]; - -// cout<< "CC: BIN16 " << i << " " -// << hex -// << *((int64_t*)values[i]) -// << " " -// << *(((int64_t*)values[i]) +1) -// << endl; + binaryValues[i] = *((int128_t*) &bpp->outputMsg[pos]); pos += 16; } break; - - - + case 8: for (i = 0, pos = sizeof(NewColResultHeader); i < outMsg->NVALS; ++i) { @@ -389,28 +383,24 @@ void ColumnCommand::process_OT_RID() memcpy(bpp->relRids, outMsg + 1, outMsg->NVALS << 1); bpp->ridCount = outMsg->NVALS; bpp->ridMap = outMsg->RidFlags; -// cout << "rid Count is " << bpp->ridCount << endl; } void ColumnCommand::process_OT_DATAVALUE() { bpp->ridCount = outMsg->NVALS; -// cout << "rid Count is " << bpp->ridCount << endl; switch (colType.colWidth) { case 16: - { - memcpy(values, outMsg + 1, outMsg->NVALS << 3); + { + memcpy(binaryValues, outMsg + 1, outMsg->NVALS << 4); cout << " CC: first value is " << values[0] << endl; break; - } + } - case 8: { memcpy(values, outMsg + 1, outMsg->NVALS << 3); -// cout << " CC: first value is " << values[0] << endl; break; } @@ -488,8 +478,6 @@ void ColumnCommand::processResult() for (uint64_t i = 0; i < bpp->ridCount; i++) bpp->fFiltCmdRids[1][i] = bpp->relRids[i]; } - -// cout << "processed " << outMsg->NVALS << " rows" << endl; } void ColumnCommand::createCommand(ByteStream& bs) @@ -823,7 +811,7 @@ void ColumnCommand::projectResultRG(RowGroup& rg, uint32_t pos) cout << __FILE__<< ":" <<__LINE__ << " ColumnCommand::projectResultRG " << endl; for (i = 0; i < outMsg->NVALS; ++i, msg8 += gapSize) { - r.setBinaryField_offset(msg8, colType.colWidth, offset); + r.setBinaryField_offset((int128_t*)msg8, colType.colWidth, offset); r.nextRow(rowSize); } break; diff --git a/primitives/primproc/columncommand.h b/primitives/primproc/columncommand.h index c1db001b0..1c1a29cb9 100644 --- a/primitives/primproc/columncommand.h +++ b/primitives/primproc/columncommand.h @@ -147,6 +147,7 @@ private: uint16_t filterCount; bool makeAbsRids; int64_t* values; // this is usually bpp->values; RTSCommand needs to use a different container + int128_t* binaryValues; uint8_t mask, shift; // vars for the selective block loader diff --git a/primitives/primproc/filtercommand.cpp b/primitives/primproc/filtercommand.cpp index 99059f88f..65034cb7a 100644 --- a/primitives/primproc/filtercommand.cpp +++ b/primitives/primproc/filtercommand.cpp @@ -174,7 +174,8 @@ Command* FilterCommand::makeFilterCommand(ByteStream& bs, vector& cmds } -FilterCommand::FilterCommand() : Command(FILTER_COMMAND), fBOP(0) +FilterCommand::FilterCommand() : Command(FILTER_COMMAND), fBOP(0), + hasWideDecimalType(false) { } @@ -247,6 +248,9 @@ void FilterCommand::setColTypes(const execplan::CalpontSystemCatalog::ColType& l { leftColType = left; rightColType = right; + + if (utils::isWideDecimalType(left) || utils::isWideDecimalType(right)) + hasWideDecimalType = true; } @@ -255,6 +259,13 @@ void FilterCommand::doFilter() bpp->ridMap = 0; bpp->ridCount = 0; + bool (FilterCommand::*compareFunc)(uint64_t, uint64_t); + + if (hasWideDecimalType) + compareFunc = &FilterCommand::binaryCompare; + else + compareFunc = &FilterCommand::compare; + // rids in [0] is used for scan [1], so [1] is a subset of [0], and same order. // -- see makeFilterCommand() above. for (uint64_t i = 0, j = 0; j < bpp->fFiltRidCount[1]; ) @@ -265,10 +276,15 @@ void FilterCommand::doFilter() } else { - if (compare(i, j) == true) + if ((this->*compareFunc)(i, j) == true) { bpp->relRids[bpp->ridCount] = bpp->fFiltCmdRids[0][i]; - bpp->values[bpp->ridCount] = bpp->fFiltCmdValues[0][i]; + // WIP MCOL-641 How is bpp->(binary)values used given that + // we are setting the relRids? + if (utils::isWideDecimalType(leftColType)) + bpp->binaryValues[bpp->ridCount] = bpp->fFiltCmdBinaryValues[0][i]; + else + bpp->values[bpp->ridCount] = bpp->fFiltCmdValues[0][i]; bpp->ridMap |= 1 << (bpp->relRids[bpp->ridCount] >> 10); bpp->ridCount++; } @@ -321,6 +337,70 @@ bool FilterCommand::compare(uint64_t i, uint64_t j) } } +bool FilterCommand::binaryCompare(uint64_t i, uint64_t j) +{ + // We type-promote to int128_t if either of the columns are + // not int128_t + int128_t leftVal, rightVal; + + if (utils::isWideDecimalType(leftColType)) + { + if (execplan::isNull(bpp->fFiltCmdBinaryValues[0][i], leftColType)) + return false; + leftVal = bpp->fFiltCmdBinaryValues[0][i]; + } + else + { + if (execplan::isNull(bpp->fFiltCmdValues[0][i], leftColType)) + return false; + leftVal = bpp->fFiltCmdValues[0][i]; + } + + if (utils::isWideDecimalType(rightColType)) + { + if (execplan::isNull(bpp->fFiltCmdBinaryValues[1][j], rightColType)) + return false; + rightVal = bpp->fFiltCmdBinaryValues[1][j]; + } + else + { + if (execplan::isNull(bpp->fFiltCmdValues[1][j], rightColType)) + return false; + rightVal = bpp->fFiltCmdValues[1][j]; + } + + switch (fBOP) + { + case COMPARE_GT: + return leftVal > rightVal; + break; + + case COMPARE_LT: + return leftVal < rightVal; + break; + + case COMPARE_EQ: + return leftVal == rightVal; + break; + + case COMPARE_GE: + return leftVal >= rightVal; + break; + + case COMPARE_LE: + return leftVal <= rightVal; + break; + + case COMPARE_NE: + return leftVal != rightVal; + break; + + default: + return false; + break; + } +} + bool FilterCommand::operator==(const FilterCommand& c) const { diff --git a/primitives/primproc/filtercommand.h b/primitives/primproc/filtercommand.h index be4a7931f..4d91c925d 100644 --- a/primitives/primproc/filtercommand.h +++ b/primitives/primproc/filtercommand.h @@ -76,9 +76,14 @@ protected: // compare method, take the indices to the values array virtual bool compare(uint64_t, uint64_t); + // compare method, take the indices to the values array + virtual bool binaryCompare(uint64_t, uint64_t); + // binary operator uint8_t fBOP; + bool hasWideDecimalType; + // column type for null check execplan::CalpontSystemCatalog::ColType leftColType; execplan::CalpontSystemCatalog::ColType rightColType; diff --git a/primitives/primproc/passthrucommand.cpp b/primitives/primproc/passthrucommand.cpp index 6c7a0336f..707b11b62 100644 --- a/primitives/primproc/passthrucommand.cpp +++ b/primitives/primproc/passthrucommand.cpp @@ -79,6 +79,8 @@ void PassThruCommand::project() { case 16: cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + bpp->serialized->append((uint8_t*) bpp->binaryValues, bpp->ridCount << 4); + break; case 8: bpp->serialized->append((uint8_t*) bpp->values, bpp->ridCount << 3); @@ -121,7 +123,6 @@ void PassThruCommand::projectIntoRowGroup(RowGroup& rg, uint32_t col) case 1: for (i = 0; i < bpp->ridCount; i++) { -// cout << "PTC: " << bpp->values[i] << endl; r.setUintField_offset<1>(bpp->values[i], offset); r.nextRow(rowSize); } @@ -131,7 +132,6 @@ void PassThruCommand::projectIntoRowGroup(RowGroup& rg, uint32_t col) case 2: for (i = 0; i < bpp->ridCount; i++) { -// cout << "PTC: " << bpp->values[i] << endl; r.setUintField_offset<2>(bpp->values[i], offset); r.nextRow(rowSize); } @@ -150,7 +150,6 @@ void PassThruCommand::projectIntoRowGroup(RowGroup& rg, uint32_t col) case 8: for (i = 0; i < bpp->ridCount; i++) { -// cout << "PTC: " << bpp->values[i] << endl; r.setUintField_offset<8>(bpp->values[i], offset); r.nextRow(rowSize); } @@ -160,15 +159,7 @@ void PassThruCommand::projectIntoRowGroup(RowGroup& rg, uint32_t col) cout << __FILE__ << ":" << __LINE__ << " PassThruCommand::projectIntoRowGroup" << " Addition for 16 Bytes" << endl; for (i = 0; i < bpp->ridCount; i++) { - cout << "PTC: " << "BIN16 " << i << " " - << hex - << *((int64_t*) bpp->values[i]) - << " " - << *(((int64_t*) bpp->values[i]) +1) - << endl; - // values[i] is 8 bytes so it contains the pointer to bpp->outputMsg set by ColumnCommand::process_OT_BOTH() - r.setBinaryField_offset((uint128_t*)bpp->values[i], 16, offset); - + r.setBinaryField_offset(&bpp->binaryValues[i], 16, offset); r.nextRow(rowSize); } } diff --git a/primitives/primproc/primproc.cpp b/primitives/primproc/primproc.cpp index f282f8a7a..a898e8d97 100644 --- a/primitives/primproc/primproc.cpp +++ b/primitives/primproc/primproc.cpp @@ -513,7 +513,9 @@ int ServicePrimProc::Child() // do not allow to read beyond the end of an extent const int MaxReadAheadSz = (extentRows) / BLOCK_SIZE; //defaultBufferSize = 512 * 1024; // @bug 2627 - changed default dict buffer from 256K to 512K, allows for cols w/ length of 61. - defaultBufferSize = 100 * 1024; // 1/17/12 - made the dict buffer dynamic, max size for a numeric col is 80k + ovrhd + // WIP MCOL-641 Check with Patrick on this. Changed it from 100*1024 to 128*1024 + // to match with BatchPrimitiveProcessor::BUFFER_SIZE + defaultBufferSize = 128 * 1024; // 1/17/12 - made the dict buffer dynamic, max size for a numeric col is 80k + ovrhd // This parm controls whether we rotate through the output sockets diff --git a/utils/common/columnwidth.h b/utils/common/columnwidth.h index 9d02a7312..26b2a2a15 100644 --- a/utils/common/columnwidth.h +++ b/utils/common/columnwidth.h @@ -18,6 +18,7 @@ #ifndef UTILS_COLWIDTH_H #define UTILS_COLWIDTH_H +#include "calpontsystemcatalog.h" #include "branchpred.h" namespace utils @@ -35,6 +36,13 @@ namespace utils return width <= MAXLEGACYWIDTH; } + inline bool isWideDecimalType(const execplan::CalpontSystemCatalog::ColType& ct) + { + return ((ct.colDataType == execplan::CalpontSystemCatalog::DECIMAL || + ct.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && + ct.colWidth == MAXCOLUMNWIDTH); + } + /** @brief Map a DECIMAL precision to data width in bytes */ inline uint8_t widthByPrecision(unsigned p) { diff --git a/utils/joiner/tuplejoiner.cpp b/utils/joiner/tuplejoiner.cpp index 9dd6626c2..6297699a6 100644 --- a/utils/joiner/tuplejoiner.cpp +++ b/utils/joiner/tuplejoiner.cpp @@ -1147,7 +1147,7 @@ void TupleJoiner::updateCPData(const Row& r) } } } - else if (utils::isWide(r.getColumnWidth(colIdx)) + else if (r.getColumnWidth(colIdx) == utils::MAXCOLUMNWIDTH && (r.getColType(colIdx) == CalpontSystemCatalog::DECIMAL || r.getColType(colIdx) == CalpontSystemCatalog::UDECIMAL)) { diff --git a/utils/messageqcpp/bytestream.cpp b/utils/messageqcpp/bytestream.cpp index 73ca1487c..edfa9f90b 100644 --- a/utils/messageqcpp/bytestream.cpp +++ b/utils/messageqcpp/bytestream.cpp @@ -236,7 +236,7 @@ ByteStream& ByteStream::operator<<(const uint64_t o) } // WIP MCOL-641 -ByteStream& ByteStream::operator<<(const uint128_t o) +ByteStream& ByteStream::operator<<(const uint128_t& o) { if (fBuf == 0 || (fCurInPtr - fBuf + 16U > fMaxLen + ISSOverhead)) growBuf(fMaxLen + BlockSize); diff --git a/utils/messageqcpp/bytestream.h b/utils/messageqcpp/bytestream.h index 9f5339243..aea3c60b1 100644 --- a/utils/messageqcpp/bytestream.h +++ b/utils/messageqcpp/bytestream.h @@ -148,9 +148,9 @@ public: EXPORT ByteStream& operator<<(const uint64_t o); // WIP MCOL-641 /** - * push an unsigned __int128 onto the end of the stream. The byte order is whatever the native byte order is. + * push an uint128_t onto the end of the stream. The byte order is whatever the native byte order is. */ - EXPORT ByteStream& operator<<(const uint128_t o); + EXPORT ByteStream& operator<<(const uint128_t& o); /** * push a float onto the end of the stream. The byte order is * whatever the native byte order is. @@ -217,7 +217,7 @@ public: EXPORT ByteStream& operator>>(uint64_t& o); // WIP MCOL-641 /** - * extract an unsigned __int128 from the front of the stream. The byte order is whatever the native byte order is. + * extract an uint128_t from the front of the stream. The byte order is whatever the native byte order is. */ EXPORT ByteStream& operator>>(uint128_t& o); /** @@ -292,7 +292,7 @@ public: EXPORT void peek(uint64_t& o) const; // WIP MCOL-641 /** - * Peek at an unsigned __int128 from the front of the stream. The byte order is whatever the native byte order is. + * Peek at an uint128_t from the front of the stream. The byte order is whatever the native byte order is. */ EXPORT void peek(uint128_t& o) const; /** diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 9ee1b5fbe..5a11c92b1 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -639,7 +639,7 @@ string Row::toString() const break; case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - if (utils::isWide(colWidths[i])) + if (colWidths[i] == utils::MAXCOLUMNWIDTH) { unsigned int buflen = precision[i] + 3; char *buf = (char*)alloca(buflen); diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index f4ba6c314..a89b48199 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -70,6 +70,9 @@ namespace WriteEngine { StopWatch timer; +using dataconvert::int128_t; +using dataconvert::uint128_t; + /**@brief WriteEngineWrapper Constructor */ WriteEngineWrapper::WriteEngineWrapper() : m_opType(NOOP) @@ -219,10 +222,6 @@ void WriteEngineWrapper::findSmallestColumn(uint32_t& colId, ColStructList colSt } } -// MCOL-641 WIP -using int128_t = __int128; -using uint128_t = unsigned __int128; - /*@convertValArray - Convert interface values to internal values */ /*********************************************************** From b09f3088ca97fdd9389ff9ccc992543f86940a77 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 12 Mar 2020 19:39:10 +0000 Subject: [PATCH 29/78] MCOL-641 Initial version of Math operations for wide decimal. --- CMakeLists.txt | 3 +- datatypes/CMakeLists.txt | 2 +- datatypes/csdecimal.cpp | 30 - datatypes/csdecimal.h | 39 -- datatypes/mcs_decimal.cpp | 261 +++++++++ datatypes/mcs_decimal.h | 224 ++++++++ dbcon/execplan/CMakeLists.txt | 3 +- dbcon/execplan/arithmeticoperator.h | 64 +-- dbcon/execplan/treenode.h | 4 +- dbcon/joblist/jlf_common.cpp | 3 +- dbcon/joblist/jlf_execplantojoblist.cpp | 10 +- dbcon/joblist/lbidlist.cpp | 4 +- primitives/primproc/filtercommand.cpp | 9 +- tests/CMakeLists.txt | 6 + tests/csdecimal.cpp | 23 - tests/mcs_decimal-tests.cpp | 731 ++++++++++++++++++++++++ utils/common/columnwidth.h | 8 - utils/common/widedecimalutils.h | 4 +- utils/dataconvert/dataconvert.cpp | 74 +-- utils/funcexp/funcexp.cpp | 23 +- utils/loggingcpp/exceptclasses.h | 10 + utils/rowgroup/rowgroup.cpp | 1 - 22 files changed, 1323 insertions(+), 213 deletions(-) delete mode 100644 datatypes/csdecimal.cpp delete mode 100644 datatypes/csdecimal.h create mode 100644 datatypes/mcs_decimal.cpp create mode 100644 datatypes/mcs_decimal.h delete mode 100644 tests/csdecimal.cpp create mode 100644 tests/mcs_decimal-tests.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index abd646c2f..e0a3fd636 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -200,7 +200,8 @@ ENDIF() SET (ENGINE_LDFLAGS "-Wl,--no-as-needed -Wl,--add-needed") -SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client datatypes) +SET (ENGINE_DT_LIB datatypes) +SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client ${ENGINE_DT_LIB}) SET (ENGINE_OAM_LIBS oamcpp alarmmanager) SET (ENGINE_BRM_LIBS brm idbdatafile cacheutils rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS}) SET (ENGINE_EXEC_LIBS joblist execplan windowfunction joiner rowgroup funcexp udfsdk regr dataconvert common compress querystats querytele thrift threadpool ${ENGINE_BRM_LIBS}) diff --git a/datatypes/CMakeLists.txt b/datatypes/CMakeLists.txt index c9616b1e6..660ddb15e 100644 --- a/datatypes/CMakeLists.txt +++ b/datatypes/CMakeLists.txt @@ -1,7 +1,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} ) set(datatypes_LIB_SRCS - csdecimal.cpp) + mcs_decimal.cpp) add_library(datatypes SHARED ${datatypes_LIB_SRCS}) diff --git a/datatypes/csdecimal.cpp b/datatypes/csdecimal.cpp deleted file mode 100644 index 7f961f4ce..000000000 --- a/datatypes/csdecimal.cpp +++ /dev/null @@ -1,30 +0,0 @@ -/* Copyright (C) 2020 MariaDB Corporation - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#include "csdecimal.h" - -const datatypes::Decimal someDecimal; - -namespace datatypes -{ - - int Decimal::compare(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r) - { - return 0; - } - -} // end of namespace diff --git a/datatypes/csdecimal.h b/datatypes/csdecimal.h deleted file mode 100644 index a567ce252..000000000 --- a/datatypes/csdecimal.h +++ /dev/null @@ -1,39 +0,0 @@ -/* Copyright (C) 2020 MariaDB Corporation - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#ifndef H_DECIMALDATATYPE -#define H_DECIMALDATATYPE - -#include - -namespace execplan -{ - struct IDB_Decimal; -} - -namespace datatypes -{ -class Decimal -{ - public: - Decimal() { }; - ~Decimal() { }; - static int compare(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r); -}; - -} //end of namespace -#endif diff --git a/datatypes/mcs_decimal.cpp b/datatypes/mcs_decimal.cpp new file mode 100644 index 000000000..2f9272491 --- /dev/null +++ b/datatypes/mcs_decimal.cpp @@ -0,0 +1,261 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include +#include + +#include "mcs_decimal.h" +#include "treenode.h" +#include "exceptclasses.h" + +namespace datatypes +{ + template + void execute(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, + execplan::IDB_Decimal& result, + BinaryOperation op, + OpOverflowCheck opOverflowCheck, + MultiplicationOverflowCheck mulOverflowCheck) + { + int128_t lValue = Decimal::isWideDecimalType(l.precision) + ? l.s128Value : l.value; + int128_t rValue = Decimal::isWideDecimalType(l.precision) + ? r.s128Value : r.value; + + if (result.scale == l.scale && result.scale == r.scale) + { + opOverflowCheck(lValue, rValue); + result.s128Value = op(lValue, rValue); + return; + } + + if (result.scale > l.scale) + { + int128_t scaleMultiplier; + getScaleDivisor(scaleMultiplier, result.scale - l.scale); + mulOverflowCheck(lValue, scaleMultiplier); + lValue *= scaleMultiplier; + } + else if (result.scale < l.scale) + { + int128_t scaleMultiplier; + getScaleDivisor(scaleMultiplier, l.scale - result.scale); + lValue /= scaleMultiplier; + } + + if (result.scale > r.scale) + { + int128_t scaleMultiplier; + getScaleDivisor(scaleMultiplier, result.scale - r.scale); + mulOverflowCheck(rValue, scaleMultiplier); + rValue *= scaleMultiplier; + } + else if (result.scale < r.scale) + { + int128_t scaleMultiplier; + getScaleDivisor(scaleMultiplier, r.scale - result.scale); + mulOverflowCheck(rValue, scaleMultiplier); + rValue /= scaleMultiplier; + } + + // We assume there is no way that lValue or rValue calculations + // give an overflow and this is an incorrect assumption. + opOverflowCheck(lValue, rValue); + + result.s128Value = op(lValue, rValue); + } + + // This is wide Decimal version only ATM + std::string Decimal::toString(execplan::IDB_Decimal& value) + { + char buf[utils::MAXLENGTH16BYTES]; + dataconvert::DataConvert::decimalToString(&value.s128Value, + value.scale, buf, sizeof(buf), + execplan::CalpontSystemCatalog::DECIMAL); + return std::string(buf); + } + + int Decimal::compare(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r) + { + int128_t divL, divR; + getScaleDivisor(divL, l.scale); + getScaleDivisor(divR, r.scale); + int128_t quotinentL, quotinentR, remainderL, remainderR; + quotinentL = l.s128Value/divL; + remainderL = l.s128Value%divL; + quotinentR = r.s128Value/divR; + remainderR = r.s128Value%divR; + + int ret = 0; + + if (quotinentL > quotinentR) + { + ret = 1; + } + else if (quotinentL < quotinentR) + { + ret = -1; + } + else + { + // rem carries the value's sign, but needs to be normalized. + int32_t s = l.scale - r.scale; + + if (s < 0) + { + if ((remainderL * mcs_pow_10[-s]) > remainderR) + ret = 1; + else if ((remainderL * mcs_pow_10[-s]) < remainderR) + ret = -1; + } + else + { + if (remainderL > (remainderR * mcs_pow_10[s])) + ret = 1; + else if (remainderL < (remainderR * mcs_pow_10[s])) + ret = -1; + } + } + + return ret; + } + + // no overflow check + template<> + void Decimal::addition(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + { + std::plus add; + NoOverflowCheck noOverflowCheck; + execute(l, r, result, add, noOverflowCheck, noOverflowCheck); + } + + template + void Decimal::addition(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result); + + // with overflow check + template<> + void Decimal::addition(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + { + std::plus add; + AdditionOverflowCheck overflowCheck; + MultiplicationOverflowCheck mulOverflowCheck; + execute(l, r, result, add, overflowCheck, mulOverflowCheck); + } + + template + void Decimal::addition(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result); + + template<> + void Decimal::addition(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + { + if (result.scale == l.scale && result.scale == r.scale) + { + result.value = l.value + r.value; + return; + } + + int64_t lValue = 0, rValue = 0; + + if (result.scale >= l.scale) + lValue = l.value * mcs_pow_10[result.scale - l.scale]; + else + lValue = (int64_t)(l.value > 0 ? + (double)l.value / mcs_pow_10[l.scale - result.scale] + 0.5 : + (double)l.value / mcs_pow_10[l.scale - result.scale] - 0.5); + + if (result.scale >= r.scale) + rValue = r.value * mcs_pow_10[result.scale - r.scale]; + else + rValue = (int64_t)(r.value > 0 ? + (double)r.value / mcs_pow_10[r.scale - result.scale] + 0.5 : + (double)r.value / mcs_pow_10[r.scale - result.scale] - 0.5); + + result.value = lValue + rValue; + } + template + void Decimal::addition(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result); + + template<> + void Decimal::addition(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + { + throw logging::NotImplementedExcept("Decimal::addition"); + } + + template<> + void Decimal::division(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + { + std::divides division; + NoOverflowCheck noOverflowCheck; + execute(l, r, result, division, noOverflowCheck, noOverflowCheck); + } + + template + void Decimal::division(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result); + + // With overflow check + template<> + void Decimal::division(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + { + std::divides division; + DivisionOverflowCheck overflowCheck; + MultiplicationOverflowCheck mulOverflowCheck; + execute(l, r, result, division, overflowCheck, mulOverflowCheck); + } + + // We rely on the zero check from ArithmeticOperator::execute + template<> + void Decimal::division(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + { + if (result.scale >= l.scale - r.scale) + result.value = (int64_t)(( (l.value > 0 && r.value > 0) + || (l.value < 0 && r.value < 0) ? + (long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)] + 0.5 : + (long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)] - 0.5)); + else + result.value = (int64_t)(( (l.value > 0 && r.value > 0) + || (l.value < 0 && r.value < 0) ? + (long double)l.value / r.value / mcs_pow_10[l.scale - r.scale - result.scale] + 0.5 : + (long double)l.value / r.value / mcs_pow_10[l.scale - r.scale - result.scale] - 0.5)); + + } + + template + void Decimal::division(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result); + + template<> + void Decimal::division(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + { + throw logging::NotImplementedExcept("Decimal::division"); + } + +} // end of namespace diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h new file mode 100644 index 000000000..aab919e92 --- /dev/null +++ b/datatypes/mcs_decimal.h @@ -0,0 +1,224 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#ifndef H_DECIMALDATATYPE +#define H_DECIMALDATATYPE + +#include + +#include "calpontsystemcatalog.h" + +using int128_t = __int128; + +namespace execplan +{ + struct IDB_Decimal; +} + +namespace datatypes +{ + +constexpr uint32_t MAXDECIMALWIDTH = 16U; +constexpr uint8_t INT64MAXPRECISION = 18U; +constexpr uint8_t INT128MAXPRECISION = 38U; + +const uint64_t mcs_pow_10[20] = +{ + 1ULL, + 10ULL, + 100ULL, + 1000ULL, + 10000ULL, + 100000ULL, + 1000000ULL, + 10000000ULL, + 100000000ULL, + 1000000000ULL, + 10000000000ULL, + 100000000000ULL, + 1000000000000ULL, + 10000000000000ULL, + 100000000000000ULL, + 1000000000000000ULL, + 10000000000000000ULL, + 100000000000000000ULL, + 1000000000000000000ULL, + 10000000000000000000ULL +}; + +constexpr uint32_t maxPowOf10 = sizeof(mcs_pow_10)/sizeof(mcs_pow_10[0])-1; +constexpr int128_t Decimal128Null = int128_t(0x8000000000000000LL) << 64; + +/** + @brief The function to produce scale multiplier/divisor for + wide decimals. +*/ +inline void getScaleDivisor(int128_t& divisor, const int8_t scale) +{ + divisor = 1; + switch (scale/maxPowOf10) + { + case 2: + divisor *= mcs_pow_10[maxPowOf10]; + divisor *= mcs_pow_10[maxPowOf10]; + break; + case 1: + divisor *= mcs_pow_10[maxPowOf10]; + case 0: + divisor *= mcs_pow_10[scale%maxPowOf10]; + default: + break; + } +} + +/** + @brief The template to generalise common math operation + execution path using struct from . +*/ +template +void execute(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, + execplan::IDB_Decimal& result, + BinaryOperation op, + OverflowCheck overflowCheck); + +/** + @brief Contains subset of decimal related operations. + + Most of the methods are static to allow to call them from + the existing code. + The main purpose of the class is to collect methods for a + base Datatype class. +*/ +class Decimal +{ + public: + Decimal() { }; + ~Decimal() { }; + + static constexpr int128_t minInt128 = int128_t(0x8000000000000000LL) << 64; + static constexpr int128_t maxInt128 = (int128_t(0x7FFFFFFFFFFFFFFFLL) << 64) + 0xFFFFFFFFFFFFFFFFLL; + + /** + @brief Compares two IDB_Decimal taking scale into account. + */ + static int compare(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r); + /** + @brief Addition template that supports overflow check and + two internal representations of decimal. + */ + template + static void addition(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, + execplan::IDB_Decimal& result); + + /** + @brief Division template that supports overflow check and + two internal representations of decimal. + */ + template + static void division(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, + execplan::IDB_Decimal& result); + + /** + @brief Convinience method to put decimal into a std:;string. + */ + static std::string toString(execplan::IDB_Decimal& value); + + /** + @brief The method detects whether decimal type is wide + using csc data type. + */ + static constexpr inline bool isWideDecimalType(const execplan::CalpontSystemCatalog::ColType& ct) + { + return ((ct.colDataType == execplan::CalpontSystemCatalog::DECIMAL || + ct.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && + ct.colWidth == MAXDECIMALWIDTH); + } + + /** + @brief The method detects whether decimal type is wide + using precision. + */ + static constexpr inline bool isWideDecimalType(const int32_t precision) + { + return precision > INT64MAXPRECISION + && precision <= INT128MAXPRECISION; + } + +}; + +/** + @brief The structure contains an overflow check for int128 + division. +*/ +struct DivisionOverflowCheck { + void operator()(const int128_t& x, const int128_t& y) + { + if (x == Decimal::maxInt128 && y == -1) + { + throw logging::OperationOverflowExcept( + "Decimal::division produces an overflow."); + } + } +}; + +/** + @brief The structure contains an overflow check for int128 + addition. +*/ +struct MultiplicationOverflowCheck { + void operator()(const int128_t& x, const int128_t& y) + { + if (x * y / x != y) + { + throw logging::OperationOverflowExcept( + "Decimal::multiplication or scale multiplicationproduces an overflow."); + } + } +}; + +/** + @brief The structure contains an overflow check for int128 + addition. +*/ +struct AdditionOverflowCheck { + void operator()(const int128_t& x, const int128_t& y) + { + if ((y > 0 && x > Decimal::maxInt128 - y) + || (y < 0 && x < Decimal::minInt128 - y)) + { + throw logging::OperationOverflowExcept( + "Decimal::addition produces an overflow."); + } + } +}; + +/** + @brief The strucuture runs an empty overflow check for int128 + operation. +*/ +struct NoOverflowCheck { + void operator()(const int128_t& x, const int128_t& y) + { + return; + } +}; + +} //end of namespace +#endif diff --git a/dbcon/execplan/CMakeLists.txt b/dbcon/execplan/CMakeLists.txt index 2a1c759a2..798290c67 100755 --- a/dbcon/execplan/CMakeLists.txt +++ b/dbcon/execplan/CMakeLists.txt @@ -48,7 +48,6 @@ add_library(execplan SHARED ${execplan_LIB_SRCS}) add_dependencies(execplan loggingcpp) -target_link_libraries(execplan ${NETSNMP_LIBRARIES} ${MARIADB_STRING_LIBS}) +target_link_libraries(execplan ${NETSNMP_LIBRARIES} ${MARIADB_STRING_LIBS} ${ENGINE_DT_LIB}) install(TARGETS execplan DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) - diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index 479b9e645..bc6f02a01 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -290,31 +290,19 @@ inline void ArithmeticOperator::execute(IDB_Decimal& result, IDB_Decimal op1, ID case OP_ADD: if (resultCscType.colWidth == 16) { - result.s128Value = op1.s128Value + op2.s128Value; - break; + datatypes::Decimal::addition( + op1, op2, result); } - - if (result.scale == op1.scale && result.scale == op2.scale) + else if (resultCscType.colWidth == 8) { - result.value = op1.value + op2.value; - break; + datatypes::Decimal::addition( + op1, op2, result); } - - if (result.scale >= op1.scale) - op1.value *= IDB_pow[result.scale - op1.scale]; else - op1.value = (int64_t)(op1.value > 0 ? - (double)op1.value / IDB_pow[op1.scale - result.scale] + 0.5 : - (double)op1.value / IDB_pow[op1.scale - result.scale] - 0.5); - - if (result.scale >= op2.scale) - op2.value *= IDB_pow[result.scale - op2.scale]; - else - op2.value = (int64_t)(op2.value > 0 ? - (double)op2.value / IDB_pow[op2.scale - result.scale] + 0.5 : - (double)op2.value / IDB_pow[op2.scale - result.scale] - 0.5); - - result.value = op1.value + op2.value; + { + throw logging::InvalidArgumentExcept( + "Unexpected result width"); + } break; case OP_SUB: @@ -352,21 +340,33 @@ inline void ArithmeticOperator::execute(IDB_Decimal& result, IDB_Decimal op1, ID break; case OP_DIV: - if (op2.value == 0) + if (resultCscType.colWidth == 16) { - isNull = true; - break; + if (op2.s128Value == 0) + { + isNull = true; + break; + } + + datatypes::Decimal::division( + op1, op2, result); } + else if (resultCscType.colWidth == 8) + { + if (op2.value == 0) + { + isNull = true; + break; + } - if (result.scale >= op1.scale - op2.scale) - result.value = (int64_t)(( (op1.value > 0 && op2.value > 0) || (op1.value < 0 && op2.value < 0) ? - (long double)op1.value / op2.value * IDB_pow[result.scale - (op1.scale - op2.scale)] + 0.5 : - (long double)op1.value / op2.value * IDB_pow[result.scale - (op1.scale - op2.scale)] - 0.5)); + datatypes::Decimal::division( + op1, op2, result); + } else - result.value = (int64_t)(( (op1.value > 0 && op2.value > 0) || (op1.value < 0 && op2.value < 0) ? - (long double)op1.value / op2.value / IDB_pow[op1.scale - op2.scale - result.scale] + 0.5 : - (long double)op1.value / op2.value / IDB_pow[op1.scale - op2.scale - result.scale] - 0.5)); - + { + throw logging::InvalidArgumentExcept( + "Unexpected result width"); + } break; default: diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 1c2f6b05a..99f5084a6 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -36,9 +36,7 @@ #include "exceptclasses.h" #include "dataconvert.h" #include "columnwidth.h" -#include "csdecimal.h" - -using int128_t = __int128; +#include "mcs_decimal.h" namespace messageqcpp { diff --git a/dbcon/joblist/jlf_common.cpp b/dbcon/joblist/jlf_common.cpp index fb4678396..ef5964866 100644 --- a/dbcon/joblist/jlf_common.cpp +++ b/dbcon/joblist/jlf_common.cpp @@ -38,6 +38,7 @@ using namespace BRM; #include "jlf_common.h" using namespace joblist; +#include "mcs_decimal.h" namespace { @@ -332,7 +333,7 @@ string extractTableAlias(const SSC& sc) //------------------------------------------------------------------------------ CalpontSystemCatalog::OID isDictCol(const CalpontSystemCatalog::ColType& colType) { - if (utils::isWideDecimalType(colType)) return 0; + if (datatypes::Decimal::isWideDecimalType(colType)) return 0; if (colType.colWidth > 8) return colType.ddn.dictOID; diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 70ec8c599..8b1211513 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -88,7 +88,7 @@ using namespace logging; #include "jlf_common.h" #include "jlf_subquery.h" #include "jlf_tuplejoblist.h" -#include "columnwidth.h" +#include "mcs_decimal.h" namespace { @@ -1897,7 +1897,7 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) // WIP MCOL-641 width check must be a f() not a literal // make a template from convertValueNum to avoid extra if // this condition doesn't support UDECIMAL - if (utils::isWideDecimalType(ct)) + if (datatypes::Decimal::isWideDecimalType(ct)) convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value128); else convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value); @@ -1934,7 +1934,7 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) if (sc->isColumnStore()) { - if (utils::isWideDecimalType(ct)) + if (datatypes::Decimal::isWideDecimalType(ct)) pcs->addFilter(cop, value128, rf); else pcs->addFilter(cop, value, rf); @@ -3012,7 +3012,7 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo) uint8_t rf = 0; bool isNull = ConstantColumn::NULLDATA == cc->type(); - if (utils::isWideDecimalType(ct)) + if (datatypes::Decimal::isWideDecimalType(ct)) convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value128); else convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value); @@ -3032,7 +3032,7 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo) if (ConstantColumn::NULLDATA == cc->type() && (opeq == *sop || opne == *sop)) cop = COMPARE_NIL; - if (utils::isWideDecimalType(ct)) + if (datatypes::Decimal::isWideDecimalType(ct)) pcs->addFilter(cop, value128, rf); else pcs->addFilter(cop, value, rf); diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index c34367b8d..90697bbfc 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -28,7 +28,7 @@ #include "brm.h" #include "brmtypes.h" #include "dataconvert.h" -#include "columnwidth.h" +#include "mcs_decimal.h" #define IS_VERBOSE (fDebug >= 4) #define IS_DETAIL (fDebug >= 3) @@ -809,7 +809,7 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, // Should we also check for empty here? // TODO MCOL-641 - if (utils::isWideDecimalType(ct)) + if (datatypes::Decimal::isWideDecimalType(ct)) { if (isNull(bigValue, ct)) continue; diff --git a/primitives/primproc/filtercommand.cpp b/primitives/primproc/filtercommand.cpp index 65034cb7a..e90239d05 100644 --- a/primitives/primproc/filtercommand.cpp +++ b/primitives/primproc/filtercommand.cpp @@ -27,6 +27,7 @@ #include "dictstep.h" #include "filtercommand.h" #include "dataconvert.h" +#include "mcs_decimal.h" using namespace std; using namespace messageqcpp; @@ -249,7 +250,7 @@ void FilterCommand::setColTypes(const execplan::CalpontSystemCatalog::ColType& l leftColType = left; rightColType = right; - if (utils::isWideDecimalType(left) || utils::isWideDecimalType(right)) + if (datatypes::Decimal::isWideDecimalType(left) || datatypes::Decimal::isWideDecimalType(right)) hasWideDecimalType = true; } @@ -281,7 +282,7 @@ void FilterCommand::doFilter() bpp->relRids[bpp->ridCount] = bpp->fFiltCmdRids[0][i]; // WIP MCOL-641 How is bpp->(binary)values used given that // we are setting the relRids? - if (utils::isWideDecimalType(leftColType)) + if (datatypes::Decimal::isWideDecimalType(leftColType)) bpp->binaryValues[bpp->ridCount] = bpp->fFiltCmdBinaryValues[0][i]; else bpp->values[bpp->ridCount] = bpp->fFiltCmdValues[0][i]; @@ -343,7 +344,7 @@ bool FilterCommand::binaryCompare(uint64_t i, uint64_t j) // not int128_t int128_t leftVal, rightVal; - if (utils::isWideDecimalType(leftColType)) + if (datatypes::Decimal::isWideDecimalType(leftColType)) { if (execplan::isNull(bpp->fFiltCmdBinaryValues[0][i], leftColType)) return false; @@ -356,7 +357,7 @@ bool FilterCommand::binaryCompare(uint64_t i, uint64_t j) leftVal = bpp->fFiltCmdValues[0][i]; } - if (utils::isWideDecimalType(rightColType)) + if (datatypes::Decimal::isWideDecimalType(rightColType)) { if (execplan::isNull(bpp->fFiltCmdBinaryValues[1][j], rightColType)) return false; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5bc8f600a..31e15e1c8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -12,6 +12,12 @@ if (WITH_ARITHMETICOPERATOR_UT) install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) endif() +if (WITH_CSDECIMAL_UT) + add_executable(mcs_decimal_tests mcs_decimal-tests.cpp) + target_link_libraries(mcs_decimal_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) + install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) +endif() + if (WITH_DATACONVERT_UT) add_executable(dataconvert_tests dataconvert-tests.cpp) target_link_libraries(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) diff --git a/tests/csdecimal.cpp b/tests/csdecimal.cpp deleted file mode 100644 index 91bddcd42..000000000 --- a/tests/csdecimal.cpp +++ /dev/null @@ -1,23 +0,0 @@ -/* Copyright (C) 2020 MariaDB Corporation - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; version 2 of - the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ - -#include "gtest/gtest.h" - -TEST(DataConvertTest, Strtoll128) -{ - EXPECT_TRUE(true); -} diff --git a/tests/mcs_decimal-tests.cpp b/tests/mcs_decimal-tests.cpp new file mode 100644 index 000000000..b41ec8751 --- /dev/null +++ b/tests/mcs_decimal-tests.cpp @@ -0,0 +1,731 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include "gtest/gtest.h" + +#include "treenode.h" +#include "mcs_decimal.h" +#include "widedecimalutils.h" + +TEST(Decimal, compareCheck) +{ + // L values = R value, L scale < R scale + execplan::IDB_Decimal l, r; + l.scale = 20; + l.precision = 38; + l.s128Value = 42; + + r.scale = 21; + l.precision = 38; + r.s128Value = 420; + EXPECT_EQ(0, datatypes::Decimal::compare(l, r)); + // L values = R value, L scale > R scale + l.scale = 21; + l.precision = 38; + l.s128Value = 420; + + r.scale = 20; + l.precision = 38; + r.s128Value = 42; + EXPECT_EQ(0, datatypes::Decimal::compare(l, r)); + // L values > R value, L scale < R scale + l.scale = 20; + l.precision = 38; + l.s128Value = 999999; + + r.scale = 21; + l.precision = 38; + r.s128Value = 420; + EXPECT_EQ(1, datatypes::Decimal::compare(l, r)); + // L values > R value, L scale > R scale + l.scale = 21; + l.precision = 38; + l.s128Value = 99999999; + + r.scale = 20; + l.precision = 38; + r.s128Value = 420; + EXPECT_EQ(1, datatypes::Decimal::compare(l, r)); + // L values < R value, L scale < R scale + l.scale = 20; + l.precision = 38; + l.s128Value = 99; + + r.scale = 21; + l.precision = 38; + r.s128Value = 42000; + EXPECT_EQ(-1, datatypes::Decimal::compare(l, r)); + // L values < R value, L scale > R scale + l.scale = 21; + l.precision = 38; + l.s128Value = 99; + + r.scale = 20; + l.precision = 38; + r.s128Value = 420; + EXPECT_EQ(-1, datatypes::Decimal::compare(l, r)); +} + +TEST(Decimal, additionNoOverflowCheck) +{ + // Addition w/o overflow check + execplan::IDB_Decimal l, r, result; + // same precision, same scale, both positive values + l.scale = 38; + l.precision = 38; + l.s128Value = 42; + + r.scale = 38; + r.precision = 38; + r.s128Value = 420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + EXPECT_EQ(462, result.s128Value); + // same precision, same scale, both negative values + l.scale = 38; + l.precision = 38; + l.s128Value = -42; + + r.scale = 38; + r.precision = 38; + r.s128Value = -420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + EXPECT_EQ(-462, result.s128Value); + // same precision, same scale, +- values + l.scale = 38; + l.precision = 38; + l.s128Value = 42; + + r.scale = 38; + r.precision = 38; + r.s128Value = -420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + EXPECT_EQ(-378, result.s128Value); + // same precision, same scale, both 0 + l.scale = 38; + l.precision = 38; + l.s128Value = 0; + + r.scale = 38; + r.precision = 38; + r.s128Value = 0; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + EXPECT_EQ(0, result.s128Value); + // diff scale + // same precision, L scale > R scale, both positive values + l.scale = 38; + l.precision = 38; + l.s128Value = 42; + + r.scale = 15; + r.precision = 38; + r.s128Value = 420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + int128_t s128ScaleMultiplier1 = + static_cast(10000000000000)*10000000000; + int128_t s128Result = r.s128Value*s128ScaleMultiplier1+l.s128Value; + EXPECT_EQ(s128Result, result.s128Value); + // same precision, L scale > R scale, both negative values + l.scale = 38; + l.precision = 38; + l.s128Value = -42; + + r.scale = 15; + r.precision = 38; + r.s128Value = -420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + s128Result = r.s128Value*s128ScaleMultiplier1+l.s128Value; + EXPECT_EQ(s128Result, result.s128Value); + // same precision, L scale > R scale, +- values + l.scale = 38; + l.precision = 38; + l.s128Value = 42; + + r.scale = 15; + r.precision = 38; + r.s128Value = -420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + s128Result = r.s128Value*s128ScaleMultiplier1+l.s128Value; + EXPECT_EQ(s128Result, result.s128Value); + // same precision, L scale > R scale, both 0 + l.scale = 38; + l.precision = 38; + l.s128Value = 0; + + r.scale = 15; + r.precision = 38; + r.s128Value = 0; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + EXPECT_EQ(0, result.s128Value); + // same precision, L scale < R scale, both positive values + l.scale = 15; + l.precision = 38; + l.s128Value = 42; + + r.scale = 38; + r.precision = 38; + r.s128Value = 420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + s128Result = l.s128Value*s128ScaleMultiplier1+r.s128Value; + EXPECT_EQ(s128Result, result.s128Value); + // same precision, L scale < R scale, both negative values + l.scale = 15; + l.precision = 38; + l.s128Value = -42; + + r.scale = 38; + r.precision = 38; + r.s128Value = -420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + s128Result = l.s128Value*s128ScaleMultiplier1+r.s128Value; + EXPECT_EQ(s128Result, result.s128Value); + // same precision, L scale < R scale, +- values + l.scale = 15; + l.precision = 38; + l.s128Value = 42; + + r.scale = 38; + r.precision = 38; + r.s128Value = -420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + s128Result = l.s128Value*s128ScaleMultiplier1+r.s128Value; + EXPECT_EQ(s128Result, result.s128Value); + // same precision, L scale < R scale, both 0 + l.scale = 15; + l.precision = 38; + l.s128Value = 0; + + r.scale = 38; + r.precision = 38; + r.s128Value = 0; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::addition(l, r, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + s128Result = l.s128Value*s128ScaleMultiplier1+r.s128Value; + EXPECT_EQ(s128Result, result.s128Value); +} + +TEST(Decimal, divisionNoOverflowCheck) +{ + // DIVISION + // same precision, same scale, both positive values + std::string decimalStr; + execplan::IDB_Decimal l, r, result; + l.scale = 38; + l.precision = 38; + l.s128Value = 43; + + r.scale = 38; + r.precision = 38; + r.s128Value = 420; + + result.scale = r.scale; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + + EXPECT_EQ(r.scale, result.scale); + EXPECT_EQ(result.precision, result.precision); + EXPECT_EQ(r.s128Value/l.s128Value, result.s128Value); + // same precision, same scale, both negative values + l.scale = 38; + l.precision = 38; + l.s128Value = -42; + + r.scale = 38; + r.precision = 38; + r.s128Value = -420; + + result.scale = r.scale; + result.precision = r.precision; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ(r.scale, result.scale); + EXPECT_EQ(r.precision, result.precision); + EXPECT_EQ(r.s128Value/l.s128Value, result.s128Value); + // same precision, same scale, +- values + l.scale = 38; + l.precision = 38; + l.s128Value = 42; + + r.scale = 38; + r.precision = 38; + r.s128Value = -420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::division(l, r, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + EXPECT_EQ(l.s128Value/r.s128Value, result.s128Value); + // same precision, same scale, l = 0 + l.scale = 38; + l.precision = 38; + l.s128Value = 0; + + r.scale = 38; + r.precision = 38; + r.s128Value = 42424242; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::division(l, r, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + EXPECT_EQ(0, result.s128Value); + // diff scale + // same precision, L scale > R scale, both positive values + l.scale = 38; + l.precision = 38; + l.s128Value = 42; + + r.scale = 15; + r.precision = 38; + r.s128Value = 420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + int128_t s128ScaleMultiplier1 = + static_cast(10000000000000)*10000000000; + int128_t s128Result = r.s128Value*s128ScaleMultiplier1/l.s128Value; + EXPECT_EQ(s128Result, result.s128Value); + // same precision, L scale > R scale, both negative values + l.scale = 38; + l.precision = 38; + l.s128Value = -42; + + r.scale = 15; + r.precision = 38; + r.s128Value = -420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + s128Result = r.s128Value*s128ScaleMultiplier1/l.s128Value; + EXPECT_EQ(s128Result, result.s128Value); + // same precision, L scale > R scale, +- values + l.scale = 38; + l.precision = 38; + l.s128Value = 42; + + r.scale = 15; + r.precision = 38; + r.s128Value = -420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + s128Result = r.s128Value*s128ScaleMultiplier1/l.s128Value; + EXPECT_EQ(s128Result, result.s128Value); + // same precision, L scale > R scale, L = 0 + l.scale = 38; + l.precision = 38; + l.s128Value = 0; + + r.scale = 15; + r.precision = 38; + r.s128Value = 424242; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::division(l, r, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + EXPECT_EQ(0, result.s128Value); + // same precision, L scale > R scale, both MAX positive values + // WIP Investigate the next two + l.scale = 38; + l.precision = 38; + l.s128Value = 0; utils::int128Max(l.s128Value); + + r.scale = 15; + r.precision = 38; + r.s128Value = 0; utils::int128Max(r.s128Value); + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::division(l, r, result); + // Use as an examplar + utils::int128Max(r.s128Value); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + s128Result = r.s128Value*s128ScaleMultiplier1/l.s128Value; + // WIP + //EXPECT_EQ(s128Result, result.s128Value); + // same precision, L scale > R scale, both MIN negative values + l.scale = 38; + l.precision = 38; + l.s128Value = 0; utils::int128Min(l.s128Value); + + r.scale = 15; + r.precision = 38; + r.s128Value = 0; utils::int128Min(l.s128Value); + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + //datatypes::Decimal::division(l, r, result); + // Use as an examplar + utils::int128Min(r.s128Value); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + s128Result = r.s128Value*s128ScaleMultiplier1/l.s128Value; + //EXPECT_EQ(s128Result, result.s128Value); + // WIP + //EXPECT_EQ(r.s128Value, result.s128Value); + + // same precision, L scale < R scale, both positive values + l.scale = 15; + l.precision = 38; + l.s128Value = 42; + + r.scale = 38; + r.precision = 38; + r.s128Value = 420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::division(l, r, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + s128Result = l.s128Value*s128ScaleMultiplier1/r.s128Value; + EXPECT_EQ(s128Result, result.s128Value); + // same precision, L scale < R scale, both negative values + l.scale = 15; + l.precision = 38; + l.s128Value = -42; + + r.scale = 38; + r.precision = 38; + r.s128Value = -420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::division(l, r, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + s128Result = l.s128Value*s128ScaleMultiplier1/r.s128Value; + EXPECT_EQ(s128Result, result.s128Value); + // same precision, L scale < R scale, +- values + l.scale = 15; + l.precision = 38; + l.s128Value = 42; + + r.scale = 38; + r.precision = 38; + r.s128Value = -420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::division(l, r, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + s128Result = l.s128Value*s128ScaleMultiplier1/r.s128Value; + EXPECT_EQ(s128Result, result.s128Value); + // same precision, L scale < R scale, L = 0 + l.scale = 15; + l.precision = 38; + l.s128Value = 0; + + r.scale = 38; + r.precision = 38; + r.s128Value = 42; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::division(l, r, result); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + s128Result = l.s128Value*s128ScaleMultiplier1/r.s128Value; + EXPECT_EQ(s128Result, result.s128Value); + // same precision, L scale < R scale, both MAX positive values + // WIP Investigate the next two + l.scale = 15; + l.precision = 38; + l.s128Value = 0; utils::int128Max(l.s128Value); + + r.scale = 38; + r.precision = 38; + r.s128Value = 0; utils::int128Max(r.s128Value); + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::division(l, r, result); + // Use as an examplar + utils::int128Max(r.s128Value); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + s128Result = l.s128Value*s128ScaleMultiplier1/r.s128Value; + // WIP + //EXPECT_EQ(s128Result, result.s128Value); + //EXPECT_EQ(r.s128Value, result.s128Value); + // same precision, L scale < R scale, both MIN negative values + l.scale = 15; + l.precision = 38; + l.s128Value = 0; utils::int128Min(l.s128Value); + + r.scale = 38; + r.precision = 38; + r.s128Value = 0; utils::int128Min(l.s128Value); + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + //datatypes::Decimal::division(l, r, result); + // Use as an examplar + utils::int128Min(r.s128Value); + EXPECT_EQ(38, result.scale); + EXPECT_EQ(38, result.precision); + s128Result = l.s128Value*s128ScaleMultiplier1/r.s128Value; + // WIP + // EXPECT_EQ(s128Result, result.s128Value); + //EXPECT_EQ(r.s128Value, result.s128Value); +} + +void doDiv(execplan::IDB_Decimal& l, + execplan::IDB_Decimal& r, + execplan::IDB_Decimal& result) +{ + datatypes::Decimal::division(l, r, result); +} + +TEST(Decimal, divisionWithOverflowCheck) +{ + + // Divide max int128 by -1 + execplan::IDB_Decimal l, r, result; + l.scale = 0; + l.precision = 38; + l.s128Value = datatypes::Decimal::maxInt128; + + r.scale = 0; + r.precision = 38; + r.s128Value = -1; + + result.scale = 0; + result.precision = 38; + result.s128Value = 42; + EXPECT_THROW(doDiv(l, r, result), logging::OperationOverflowExcept); + // Divide two ints one of which overflows after the scaling. + l.scale = 0; + l.precision = 38; + l.s128Value = datatypes::Decimal::maxInt128; + + r.scale = 1; + r.precision = 38; + r.s128Value = 42; + + result.scale = 1; + result.precision = 38; + result.s128Value = 42; + EXPECT_THROW(doDiv(l, r, result), logging::OperationOverflowExcept); + // Normal execution w/o overflow + l.scale = 0; + l.precision = 38; + l.s128Value = datatypes::Decimal::maxInt128-1; + + r.scale = 0; + r.precision = 38; + r.s128Value = 0xFFFFFFFFFFFFFFFF; + + result.scale = 0; + result.precision = 38; + result.s128Value = 0; + + EXPECT_NO_THROW(doDiv(l, r, result)); + + l.s128Value /= r.s128Value; + + EXPECT_EQ(0, result.scale); + EXPECT_EQ(38, result.precision); + EXPECT_EQ(l.s128Value, result.s128Value); +} + +void doAdd(execplan::IDB_Decimal& l, + execplan::IDB_Decimal& r, + execplan::IDB_Decimal& result) +{ + datatypes::Decimal::addition(l, r, result); +} + +TEST(Decimal, additionWithOverflowCheck) +{ + // Add two max ints + execplan::IDB_Decimal l, r, result; + l.scale = 0; + l.precision = 38; + l.s128Value = datatypes::Decimal::maxInt128-1; + + r.scale = 0; + r.precision = 38; + r.s128Value = datatypes::Decimal::maxInt128-1; + + result.scale = 0; + result.precision = 38; + result.s128Value = 42; + EXPECT_THROW(doAdd(l, r, result), logging::OperationOverflowExcept); + // Add two ints one of which overflows after the scaling. + l.scale = 0; + l.precision = 38; + l.s128Value = datatypes::Decimal::maxInt128-1; + + r.scale = 1; + r.precision = 38; + r.s128Value = 0xFFFFFFFFFFFFFFFF; + + result.scale = 1; + result.precision = 38; + result.s128Value = 0; + + EXPECT_THROW(doAdd(l, r, result), logging::OperationOverflowExcept); + // Normal execution w/o overflow + l.scale = 0; + l.precision = 38; + l.s128Value = datatypes::Decimal::maxInt128-1; + + r.scale = 0; + r.precision = 38; + r.s128Value = 0xFFFFFFFFFFFFFFFF; + + result.scale = 0; + result.precision = 38; + result.s128Value = 0; + + EXPECT_NO_THROW(doDiv(l, r, result)); + + l.s128Value /= r.s128Value; + + EXPECT_EQ(0, result.scale); + EXPECT_EQ(38, result.precision); + EXPECT_EQ(l.s128Value, result.s128Value); +} diff --git a/utils/common/columnwidth.h b/utils/common/columnwidth.h index 26b2a2a15..9d02a7312 100644 --- a/utils/common/columnwidth.h +++ b/utils/common/columnwidth.h @@ -18,7 +18,6 @@ #ifndef UTILS_COLWIDTH_H #define UTILS_COLWIDTH_H -#include "calpontsystemcatalog.h" #include "branchpred.h" namespace utils @@ -36,13 +35,6 @@ namespace utils return width <= MAXLEGACYWIDTH; } - inline bool isWideDecimalType(const execplan::CalpontSystemCatalog::ColType& ct) - { - return ((ct.colDataType == execplan::CalpontSystemCatalog::DECIMAL || - ct.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && - ct.colWidth == MAXCOLUMNWIDTH); - } - /** @brief Map a DECIMAL precision to data width in bytes */ inline uint8_t widthByPrecision(unsigned p) { diff --git a/utils/common/widedecimalutils.h b/utils/common/widedecimalutils.h index 7ae3d0ebd..a27859828 100644 --- a/utils/common/widedecimalutils.h +++ b/utils/common/widedecimalutils.h @@ -80,9 +80,7 @@ namespace utils inline void int128Min(int128_t& val) { - uint64_t* ptr = reinterpret_cast(&val); - ptr[0] = 0; - ptr[1] = 0x8000000000000000; + val = int128_t(0x8000000000000000LL) << 64; } inline void uint128Max(uint128_t& val) diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 4bffee3d5..1a2086188 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -102,30 +102,6 @@ const string columnstore_big_precision[20] = "99999999999999999999999999999999999999" }; -const uint64_t columnstore_pow_10[20] = -{ - 1ULL, - 10ULL, - 100ULL, - 1000ULL, - 10000ULL, - 100000ULL, - 1000000ULL, - 10000000ULL, - 100000000ULL, - 1000000000ULL, - 10000000000ULL, - 100000000000ULL, - 1000000000000ULL, - 10000000000000ULL, - 100000000000000ULL, - 1000000000000000ULL, - 10000000000000000ULL, - 100000000000000000ULL, - 1000000000000000000ULL, - 10000000000000000000ULL -}; - template bool from_string(T& t, const std::string& s, std::ios_base & (*f)(std::ios_base&)) { @@ -1257,33 +1233,30 @@ size_t DataConvert::writeIntPart(int128_t* dec, { int128_t intPart = *dec; int128_t high = 0, mid = 0, low = 0; - uint64_t div = 10000000000000000000ULL; + uint64_t maxUint64divisor = 10000000000000000000ULL; if (scale) { - const uint8_t maxPowOf10 = - (sizeof(columnstore_pow_10) / sizeof(columnstore_pow_10[0])) - 1; - // Assuming scale = [0, 56] - switch (scale / maxPowOf10) + switch (scale / datatypes::maxPowOf10) { case 2: // scale = [38, 56] - intPart /= columnstore_pow_10[maxPowOf10]; - intPart /= columnstore_pow_10[maxPowOf10]; + intPart /= datatypes::mcs_pow_10[datatypes::maxPowOf10]; + intPart /= datatypes::mcs_pow_10[datatypes::maxPowOf10]; low = intPart; break; case 1: // scale = [19, 37] - intPart /= columnstore_pow_10[maxPowOf10]; - intPart /= columnstore_pow_10[scale % maxPowOf10]; - low = intPart % div; - mid = intPart / div; + intPart /= datatypes::mcs_pow_10[datatypes::maxPowOf10]; + intPart /= datatypes::mcs_pow_10[scale % datatypes::maxPowOf10]; + low = intPart % maxUint64divisor; + mid = intPart / maxUint64divisor; break; case 0: // scale = [0, 18] - intPart /= columnstore_pow_10[scale % maxPowOf10]; - low = intPart % div; - intPart /= div; - mid = intPart % div; - high = intPart / div; + intPart /= datatypes::mcs_pow_10[scale % datatypes::maxPowOf10]; + low = intPart % maxUint64divisor; + intPart /= maxUint64divisor; + mid = intPart % maxUint64divisor; + high = intPart / maxUint64divisor; break; default: throw QueryDataExcept("writeIntPart() bad scale", formatErr); @@ -1291,10 +1264,10 @@ size_t DataConvert::writeIntPart(int128_t* dec, } else { - low = intPart % div; - intPart /= div; - mid = intPart % div; - high = intPart / div; + low = intPart % maxUint64divisor; + intPart /= maxUint64divisor; + mid = intPart % maxUint64divisor; + high = intPart / maxUint64divisor; } // pod[0] is low 8 bytes, pod[1] is high 8 bytes @@ -1337,19 +1310,16 @@ size_t DataConvert::writeFractionalPart(int128_t* dec, { int128_t scaleDivisor = 1; - const uint8_t maxPowOf10 = - (sizeof(columnstore_pow_10) / sizeof(columnstore_pow_10[0])) - 1; - - switch (scale / maxPowOf10) + switch (scale / datatypes::maxPowOf10) { case 2: - scaleDivisor *= columnstore_pow_10[maxPowOf10]; - scaleDivisor *= columnstore_pow_10[maxPowOf10]; + scaleDivisor *= datatypes::mcs_pow_10[datatypes::maxPowOf10]; + scaleDivisor *= datatypes::mcs_pow_10[datatypes::maxPowOf10]; break; case 1: - scaleDivisor *= columnstore_pow_10[maxPowOf10]; + scaleDivisor *= datatypes::mcs_pow_10[datatypes::maxPowOf10]; case 0: - scaleDivisor *= columnstore_pow_10[scale % maxPowOf10]; + scaleDivisor *= datatypes::mcs_pow_10[scale % datatypes::maxPowOf10]; } int128_t fractionalPart = *dec % scaleDivisor; diff --git a/utils/funcexp/funcexp.cpp b/utils/funcexp/funcexp.cpp index 1499c9744..d9dc24180 100644 --- a/utils/funcexp/funcexp.cpp +++ b/utils/funcexp/funcexp.cpp @@ -42,6 +42,8 @@ using namespace joblist; #include "../udfsdk/udfsdk.h" #endif +#include "mcs_decimal.h" + namespace funcexp { @@ -471,13 +473,22 @@ void FuncExp::evaluate(rowgroup::Row& row, std::vector& expressi case CalpontSystemCatalog::UDECIMAL: { IDB_Decimal val = expression[i]->getDecimalVal(row, isNull); - - // WIP check for null and overflow here. - if (expression[i]->resultType().colWidth == 16) + if (expression[i]->resultType().colWidth + == datatypes::MAXDECIMALWIDTH) { - row.setBinaryField_offset(&val.s128Value, - expression[i]->resultType().colWidth, - row.getOffset(expression[i]->outputIndex())); + if (isNull) + { + row.setBinaryField_offset( + const_cast(&datatypes::Decimal128Null), + expression[i]->resultType().colWidth, + row.getOffset(expression[i]->outputIndex())); + } + else + { + row.setBinaryField_offset(&val.s128Value, + expression[i]->resultType().colWidth, + row.getOffset(expression[i]->outputIndex())); + } } else { diff --git a/utils/loggingcpp/exceptclasses.h b/utils/loggingcpp/exceptclasses.h index 805a2f778..3fd77dc8a 100644 --- a/utils/loggingcpp/exceptclasses.h +++ b/utils/loggingcpp/exceptclasses.h @@ -187,6 +187,16 @@ public: std::runtime_error(msg) { } }; +/** @brief Exception for F&E framework to throw on op overflow + * Invalid Operation Exception + */ +class OperationOverflowExcept : public std::runtime_error +{ +public: + /** Takes a character string describing the error. */ + OperationOverflowExcept(const std::string& msg) : + std::runtime_error(msg) { } +}; /** @brief specific error exception class for getSysData in Calpontsystemcatalog. * @bug 2574 * diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 5a11c92b1..3bb0f95ea 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -1629,7 +1629,6 @@ void applyMapping(const int* mapping, const Row& in, Row* out) out->setVarBinaryField(in.getVarBinaryField(i), in.getVarBinaryLength(i), mapping[i]); else if (UNLIKELY(in.isLongString(i))) out->setStringField(in.getStringPointer(i), in.getStringLength(i), mapping[i]); - //out->setStringField(in.getStringField(i), mapping[i]); else if (UNLIKELY(in.isShortString(i))) out->setUintField(in.getUintField(i), mapping[i]); else if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::LONGDOUBLE)) From 0bd172cd6e6d6a5f96ba595c3e7b6e0834f1b32d Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 17 Mar 2020 09:01:20 +0000 Subject: [PATCH 30/78] MCOL-641 The fix to support recent changes in 10.5.1. --- dbcon/mysql/ha_mcs_impl.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index e798ead3f..1de0870f9 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -1494,8 +1494,8 @@ uint32_t doUpdateDelete(THD* thd, gp_walk_info& gwi, const std::vector& c // Minor optimization: // do not perform updates of the form "update t1 set a = a;" if (!strcmp(item->name.str, setIt->name.str) - && item->table_name && setIt->table_name && !strcmp(item->table_name, setIt->table_name) - && item->db_name && setIt->db_name && !strcmp(item->db_name, setIt->db_name)) + && item->table_name.str && setIt->table_name.str && !strcmp(item->table_name.str, setIt->table_name.str) + && item->db_name.str && setIt->db_name.str && !strcmp(item->db_name.str, setIt->db_name.str)) { delete columnAssignmentPtr; continue; From 74b64eb4f1bf544eb2d427372587680e12e48d85 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Fri, 13 Mar 2020 15:42:25 -0400 Subject: [PATCH 31/78] MCOL-641 1. Add support for int128_t in ParsedColumnFilter. 2. Set Decimal precision in SimpleColumn::evaluate(). 3. Add support for int128_t in ConstantColumn. 4. Set IDB_Decimal::s128Value in buildDecimalColumn(). 5. Use width 16 as first if predicate for branching based on decimal width. --- datatypes/mcs_decimal.cpp | 51 +++++-- dbcon/execplan/constantcolumn.cpp | 4 + dbcon/execplan/simplecolumn.cpp | 8 +- dbcon/execplan/treenode.h | 6 + dbcon/joblist/jlf_common.cpp | 3 +- dbcon/joblist/jlf_execplantojoblist.cpp | 41 +++--- dbcon/joblist/tupleconstantstep.cpp | 2 +- dbcon/mysql/ha_mcs_execplan.cpp | 9 +- dbcon/mysql/ha_mcs_impl.cpp | 2 +- primitives/linux-port/column.cpp | 120 +++++++-------- primitives/linux-port/primitiveprocessor.h | 21 +++ tests/CMakeLists.txt | 4 +- tests/dataconvert-tests.cpp | 64 +++++++- utils/common/columnwidth.h | 1 - utils/dataconvert/dataconvert.cpp | 164 ++++++++++----------- utils/joiner/tuplejoiner.cpp | 2 +- utils/rowgroup/rowgroup.cpp | 4 +- utils/rowgroup/rowgroup.h | 2 +- writeengine/bulk/we_bulkloadbuffer.cpp | 15 +- writeengine/server/we_dmlcommandproc.cpp | 41 ++++-- writeengine/wrapper/writeengine.cpp | 22 +-- 21 files changed, 348 insertions(+), 238 deletions(-) diff --git a/datatypes/mcs_decimal.cpp b/datatypes/mcs_decimal.cpp index 2f9272491..007ac9c81 100644 --- a/datatypes/mcs_decimal.cpp +++ b/datatypes/mcs_decimal.cpp @@ -24,6 +24,27 @@ namespace datatypes { + + struct lldiv_t_128 + { + int128_t quot; + int128_t rem; + lldiv_t_128() : quot(0), rem(0) {} + }; + + inline lldiv_t_128 lldiv128(const int128_t& dividend, const int128_t& divisor) + { + lldiv_t_128 res; + + if (UNLIKELY(divisor == 0) || UNLIKELY(dividend == 0)) + return res; + + res.quot = dividend / divisor; + res.rem = dividend % divisor; + + return res; + } + template @@ -94,42 +115,42 @@ namespace datatypes int Decimal::compare(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r) { - int128_t divL, divR; - getScaleDivisor(divL, l.scale); - getScaleDivisor(divR, r.scale); - int128_t quotinentL, quotinentR, remainderL, remainderR; - quotinentL = l.s128Value/divL; - remainderL = l.s128Value%divL; - quotinentR = r.s128Value/divR; - remainderR = r.s128Value%divR; + int128_t divisorL, divisorR; + getScaleDivisor(divisorL, l.scale); + getScaleDivisor(divisorR, r.scale); + + lldiv_t_128 d1 = lldiv128(l.s128Value, divisorL); + lldiv_t_128 d2 = lldiv128(r.s128Value, divisorR); int ret = 0; - if (quotinentL > quotinentR) + if (d1.quot > d2.quot) { ret = 1; } - else if (quotinentL < quotinentR) + else if (d1.quot < d2.quot) { ret = -1; } else { // rem carries the value's sign, but needs to be normalized. - int32_t s = l.scale - r.scale; + int64_t s = l.scale - r.scale; + int128_t divisor; + getScaleDivisor(divisor, abs(s)); if (s < 0) { - if ((remainderL * mcs_pow_10[-s]) > remainderR) + if ((d1.rem * divisor) > d2.rem) ret = 1; - else if ((remainderL * mcs_pow_10[-s]) < remainderR) + else if ((d1.rem * divisor) < d2.rem) ret = -1; } else { - if (remainderL > (remainderR * mcs_pow_10[s])) + if (d1.rem > (d2.rem * divisor)) ret = 1; - else if (remainderL < (remainderR * mcs_pow_10[s])) + else if (d1.rem < (d2.rem * divisor)) ret = -1; } } diff --git a/dbcon/execplan/constantcolumn.cpp b/dbcon/execplan/constantcolumn.cpp index 28a7611d4..775ad88ed 100644 --- a/dbcon/execplan/constantcolumn.cpp +++ b/dbcon/execplan/constantcolumn.cpp @@ -230,6 +230,7 @@ ConstantColumn::ConstantColumn(const int64_t val, TYPE type) : fResult.doubleVal = (double)fResult.intVal; fResult.longDoubleVal = (long double)fResult.intVal; fResult.decimalVal.value = fResult.intVal; + fResult.decimalVal.s128Value = fResult.intVal; fResult.decimalVal.scale = 0; fResultType.colDataType = CalpontSystemCatalog::BIGINT; fResultType.colWidth = 8; @@ -250,6 +251,7 @@ ConstantColumn::ConstantColumn(const uint64_t val, TYPE type) : fResult.doubleVal = (double)fResult.uintVal; fResult.longDoubleVal = (long double)fResult.uintVal; fResult.decimalVal.value = fResult.uintVal; + fResult.decimalVal.s128Value = fResult.uintVal; fResult.decimalVal.scale = 0; fResultType.colDataType = CalpontSystemCatalog::UBIGINT; fResultType.colWidth = 8; @@ -317,6 +319,7 @@ void ConstantColumn::serialize(messageqcpp::ByteStream& b) const b << (uint8_t)fResult.boolVal; b << fResult.strVal; b << (uint64_t)fResult.decimalVal.value; + b << (uint128_t)fResult.decimalVal.s128Value; b << (uint8_t)fResult.decimalVal.scale; b << (uint8_t)fResult.decimalVal.precision; } @@ -341,6 +344,7 @@ void ConstantColumn::unserialize(messageqcpp::ByteStream& b) b >> (uint8_t&)fResult.boolVal; b >> fResult.strVal; b >> (uint64_t&)fResult.decimalVal.value; + b >> (uint128_t&)fResult.decimalVal.s128Value; b >> (uint8_t&)fResult.decimalVal.scale; b >> (uint8_t&)fResult.decimalVal.precision; } diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index 34e02a4df..09328ae9f 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -637,38 +637,36 @@ void SimpleColumn::evaluate(Row& row, bool& isNull) { fResult.decimalVal.s128Value = *row.getBinaryField_offset(fInputOffset); - fResult.decimalVal.scale = (unsigned)fResultType.scale; break; } case 1: { fResult.decimalVal.value = row.getIntField<1>(fInputIndex); - fResult.decimalVal.scale = (unsigned)fResultType.scale; break; } case 2: { fResult.decimalVal.value = row.getIntField<2>(fInputIndex); - fResult.decimalVal.scale = (unsigned)fResultType.scale; break; } case 4: { fResult.decimalVal.value = row.getIntField<4>(fInputIndex); - fResult.decimalVal.scale = (unsigned)fResultType.scale; break; } default: { fResult.decimalVal.value = (int64_t)row.getUintField<8>(fInputIndex); - fResult.decimalVal.scale = (unsigned)fResultType.scale; break; } } + fResult.decimalVal.scale = (unsigned)fResultType.scale; + fResult.decimalVal.precision = (unsigned)fResultType.precision; + break; } diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 99f5084a6..be105cf92 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -67,6 +67,7 @@ struct IDB_Decimal { s128Value = 0; } + IDB_Decimal(int64_t val, int8_t s, uint8_t p) : value (val), scale(s), @@ -131,6 +132,7 @@ struct IDB_Decimal return (decimalComp(rhs) == 0); } } + bool operator>(const IDB_Decimal& rhs) const { if (utils::widthByPrecision(precision) == 16) @@ -148,6 +150,7 @@ struct IDB_Decimal return (decimalComp(rhs) > 0); } } + bool operator<(const IDB_Decimal& rhs) const { if (utils::widthByPrecision(precision) == 16) @@ -165,6 +168,7 @@ struct IDB_Decimal return (decimalComp(rhs) < 0); } } + bool operator>=(const IDB_Decimal& rhs) const { if (utils::widthByPrecision(precision) == 16) @@ -182,6 +186,7 @@ struct IDB_Decimal return (decimalComp(rhs) >= 0); } } + bool operator<=(const IDB_Decimal& rhs) const { if (utils::widthByPrecision(precision) == 16) @@ -199,6 +204,7 @@ struct IDB_Decimal return (decimalComp(rhs) <= 0); } } + bool operator!=(const IDB_Decimal& rhs) const { if (utils::widthByPrecision(precision) == 16) diff --git a/dbcon/joblist/jlf_common.cpp b/dbcon/joblist/jlf_common.cpp index ef5964866..c07ab056e 100644 --- a/dbcon/joblist/jlf_common.cpp +++ b/dbcon/joblist/jlf_common.cpp @@ -333,7 +333,8 @@ string extractTableAlias(const SSC& sc) //------------------------------------------------------------------------------ CalpontSystemCatalog::OID isDictCol(const CalpontSystemCatalog::ColType& colType) { - if (datatypes::Decimal::isWideDecimalType(colType)) return 0; + if (datatypes::Decimal::isWideDecimalType(colType) || + colType.colDataType == CalpontSystemCatalog::BINARY) return 0; if (colType.colWidth > 8) return colType.ddn.dictOID; diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 8b1211513..a184fbe24 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -130,9 +130,10 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo); /* This looks like an inefficient way to get NULL values. Much easier ways to do it. */ -int64_t valueNullNum(const CalpontSystemCatalog::ColType& ct, const string& timeZone) +template +void valueNullNum(const CalpontSystemCatalog::ColType& ct, const string& timeZone, T& val) { - int64_t n = 0; + T& n = val; bool pushWarning = false; boost::any anyVal = DataConvert::convertColumnData(ct, "", pushWarning, timeZone, true, false, false); @@ -292,24 +293,22 @@ int64_t valueNullNum(const CalpontSystemCatalog::ColType& ct, const string& time case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - n = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) - n = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::FOUR_BYTE) - n = boost::any_cast(anyVal); + if (LIKELY(ct.colWidth == datatypes::MAXDECIMALWIDTH)) + n = boost::any_cast(anyVal); else if (ct.colWidth == execplan::CalpontSystemCatalog::EIGHT_BYTE) n = boost::any_cast(anyVal); - else - n = 0xfffffffffffffffeLL; + else if (ct.colWidth == execplan::CalpontSystemCatalog::FOUR_BYTE) + n = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) + n = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) + n = boost::any_cast(anyVal); break; default: break; } - - return n; } template @@ -317,7 +316,7 @@ void convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct, { if (str.size() == 0 || isNull ) { - v = valueNullNum(ct, timeZone); + valueNullNum(ct, timeZone, v); return; } @@ -445,10 +444,10 @@ void convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct, case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - v = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) - v = boost::any_cast(anyVal); + if (LIKELY(ct.colWidth == datatypes::MAXDECIMALWIDTH)) + v = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::EIGHT_BYTE) + v = boost::any_cast(anyVal); else if (ct.colWidth == execplan::CalpontSystemCatalog::FOUR_BYTE) #ifdef _MSC_VER v = boost::any_cast(anyVal); @@ -456,10 +455,10 @@ void convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct, #else v = boost::any_cast(anyVal); #endif - else if (ct.colWidth == execplan::CalpontSystemCatalog::EIGHT_BYTE) - v = boost::any_cast(anyVal); - else - v = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) + v = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) + v = boost::any_cast(anyVal); break; diff --git a/dbcon/joblist/tupleconstantstep.cpp b/dbcon/joblist/tupleconstantstep.cpp index f6601cb1e..5b8f53810 100644 --- a/dbcon/joblist/tupleconstantstep.cpp +++ b/dbcon/joblist/tupleconstantstep.cpp @@ -529,7 +529,7 @@ void TupleConstantStep::fillInConstants(const rowgroup::Row& rowIn, rowgroup::Ro //fRowConst.copyField(rowOut.getData()+2, 0); // hardcoded 2 for rid length for (uint32_t i = 1; i < rowOut.getColumnCount(); i++) // WIP MCOL-641 implement copyBinaryField inside copyField - if (UNLIKELY(rowIn.getColumnWidth(i - 1) == 16)) + if (UNLIKELY(utils::isWide(rowIn.getColumnWidth(i - 1)))) rowIn.copyBinaryField(rowOut, i, i - 1); else rowIn.copyField(rowOut, i, i - 1); diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index bba566d7b..043da5168 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -4365,8 +4365,15 @@ ConstantColumn* buildDecimalColumn(Item* item, gp_walk_info& gwi) columnstore_decimal_val << str->ptr()[i]; } - columnstore_decimal.value = strtoll(columnstore_decimal_val.str().c_str(), 0, 10); + if (idp->decimal_precision() <= 18) + columnstore_decimal.value = strtoll(columnstore_decimal_val.str().c_str(), 0, 10); + else + { + bool dummy = false; + columnstore_decimal.s128Value = dataconvert::strtoll128(columnstore_decimal_val.str().c_str(), dummy, 0); + } + // TODO MCOL-641 Add support here if (gwi.internalDecimalScale >= 0 && idp->decimals > (uint)gwi.internalDecimalScale) { columnstore_decimal.scale = gwi.internalDecimalScale; diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 1de0870f9..f80f9738b 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -805,7 +805,7 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - if (colType.colWidth == 16) + if (LIKELY(colType.colWidth == datatypes::MAXDECIMALWIDTH)) { // unset null bit first // Might be redundant diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 1161f2cda..d9724d727 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -41,6 +41,7 @@ using namespace boost; #include "primproc.h" #include "dataconvert.h" #include "widedecimalutils.h" +#include "mcs_decimal.h" using namespace logging; using namespace dbbc; using namespace primitives; @@ -639,8 +640,6 @@ inline string fixChar(int64_t intval) inline bool colCompare(int64_t val1, int64_t val2, uint8_t COP, uint8_t rf, int type, uint8_t width, const idb_regex_t& regex, bool isNull = false) { -// cout << "comparing " << hex << val1 << " to " << val2 << endl; - if (COMPARE_NIL == COP) return false; //@bug 425 added isNull condition @@ -689,6 +688,19 @@ inline bool colCompare(int64_t val1, int64_t val2, uint8_t COP, uint8_t rf, int } } +inline bool colCompare(int128_t val1, int128_t val2, uint8_t COP, uint8_t rf, int type, uint8_t width, bool isNull = false) +{ + if (COMPARE_NIL == COP) return false; + + /* isNullVal should work on the normalized value on little endian machines */ + bool val2Null = isNullVal(width, type, (uint8_t*) &val2); + + if (isNull == val2Null || (val2Null && COP == COMPARE_NE)) + return colCompare_(val1, val2, COP, rf); + else + return false; +} + inline bool colCompareUnsigned(uint64_t val1, uint64_t val2, uint8_t COP, uint8_t rf, int type, uint8_t width, const idb_regex_t& regex, bool isNull = false) { // cout << "comparing unsigned" << hex << val1 << " to " << val2 << endl; @@ -1602,7 +1614,7 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, int nextRidIndex = 0, argIndex = 0; bool done = false, cmp = false, isNull = false, isEmpty = false; uint16_t rid = 0; - prestored_set_t::const_iterator it; + prestored_set_t_128::const_iterator it; binWtype* argVals = (binWtype*)alloca(in->NOPS * W); uint8_t* std_cops = (uint8_t*)alloca(in->NOPS * sizeof(uint8_t)); @@ -1635,7 +1647,7 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, // we have a pre-parsed filter, and it's in the form of op and value arrays else if (parsedColumnFilter->columnFilterMode == TWO_ARRAYS) { - argVals = (binWtype*) parsedColumnFilter->prestored_argVals.get(); + argVals = (binWtype*) parsedColumnFilter->prestored_argVals128.get(); cops = parsedColumnFilter->prestored_cops.get(); rfs = parsedColumnFilter->prestored_rfs.get(); regex = parsedColumnFilter->prestored_regex.get(); @@ -1646,14 +1658,11 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, bval = (binWtype*)nextBinColValue(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull, &isEmpty, &rid, in->OutputType, reinterpret_cast(block), itemsPerBlk); - uint128_t val; - dataconvert::Int128Pod_t *valPod; - valPod = reinterpret_cast(&val); + int128_t val; while (!done) { - valPod->lo = *reinterpret_cast(*bval); - valPod->hi = *(reinterpret_cast(*bval) + 1); + val = *reinterpret_cast(bval); if (cops == NULL) // implies parsedColumnFilter && columnFilterMode == SET { @@ -1661,13 +1670,13 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, if (!(isNull && in->BOP == BOP_AND)) { - it = parsedColumnFilter->prestored_set->find(val); + it = parsedColumnFilter->prestored_set_128->find(val); if (in->BOP == BOP_OR) { // assume COP == COMPARE_EQ - if (it != parsedColumnFilter->prestored_set->end()) + if (it != parsedColumnFilter->prestored_set_128->end()) { store(in, out, outSize, written, rid, reinterpret_cast(block)); } @@ -1675,7 +1684,7 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, else if (in->BOP == BOP_AND) { // assume COP == COMPARE_NE - if (it == parsedColumnFilter->prestored_set->end()) + if (it == parsedColumnFilter->prestored_set_128->end()) { store(in, out, outSize, written, rid, reinterpret_cast(block)); } @@ -1686,45 +1695,11 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, { for (argIndex = 0; argIndex < in->NOPS; argIndex++) { - -// if((*((uint64_t *) (uval))) != 0) cout << "comparing " << dec << (*((uint64_t *) (uval))) << " to " << (*((uint64_t *) (argVals[argIndex]))) << endl; - // WIP MCOL-641 - uint128_t filterVal; - dataconvert::Int128Pod_t *filterValPod; - filterValPod = reinterpret_cast(&filterVal); + int128_t filterVal = *reinterpret_cast(argVals[argIndex]); - filterValPod->lo = *reinterpret_cast(argVals[argIndex]); - filterValPod->hi = *(reinterpret_cast(argVals[argIndex]) + 1); - - switch (cops[argIndex]) { - case COMPARE_NIL: - cmp = false; - break; - case COMPARE_LT: - cmp = val < filterVal; - break; - case COMPARE_EQ: - cmp = val == filterVal; - break; - case COMPARE_LE: - cmp = val <= filterVal; - break; - case COMPARE_GT: - cmp = val > filterVal; - break; - case COMPARE_NE: - cmp = val != filterVal; - break; - case COMPARE_GE: - cmp = val >= filterVal; - break; - default: - logIt(34, cops[argIndex], "colCompare"); - cmp = false; // throw an exception here? - } - -// cout << cmp << endl; + cmp = colCompare(val, filterVal, cops[argIndex], + rfs[argIndex], in->DataType, W, isNull); if (in->NOPS == 1) { @@ -1732,7 +1707,6 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, { store(in, out, outSize, written, rid, reinterpret_cast(block)); } - break; } else if (in->BOP == BOP_AND && cmp == false) @@ -1765,19 +1739,19 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, } else if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType)) { - if (static_cast(out->Min) > val) - out->Min = static_cast(val); + if (static_cast(out->Min) > static_cast(val)) + out->Min = val; - if (static_cast(out->Max) < val) - out->Max = static_cast(val);; + if (static_cast(out->Max) < static_cast(val)) + out->Max = val; } else { - if (out->Min > (__int128) val) - out->Min = (__int128) val; + if (out->Min > val) + out->Min = val; - if (out->Max < (__int128) val) - out->Max = (__int128) val; + if (out->Max < val) + out->Max = val; } } @@ -1897,9 +1871,8 @@ boost::shared_ptr parseColumnFilter ret.reset(new ParsedColumnFilter()); ret->columnFilterMode = TWO_ARRAYS; - // WIP MCOL-641 - if (colWidth == 16) - ret->prestored_argVals.reset(new int64_t[filterCount * 2]); + if (colWidth == datatypes::MAXDECIMALWIDTH) + ret->prestored_argVals128.reset(new int128_t[filterCount]); else ret->prestored_argVals.reset(new int64_t[filterCount]); ret->prestored_cops.reset(new uint8_t[filterCount]); @@ -1997,9 +1970,8 @@ boost::shared_ptr parseColumnFilter ret->prestored_argVals[argIndex] = *reinterpret_cast(args->val); break; - // WIP MCOL-641 case 16: - memcpy(ret->prestored_argVals.get() + (argIndex * 2), args->val, 16); + ret->prestored_argVals128[argIndex] = *reinterpret_cast(args->val); } } @@ -2028,12 +2000,24 @@ boost::shared_ptr parseColumnFilter if (convertToSet) { ret->columnFilterMode = UNORDERED_SET; - ret->prestored_set.reset(new prestored_set_t()); + if (colWidth == datatypes::MAXDECIMALWIDTH) + { + ret->prestored_set_128.reset(new prestored_set_t_128()); - // @bug 2584, use COMPARE_NIL for "= null" to allow "is null" in OR expression - for (argIndex = 0; argIndex < filterCount; argIndex++) - if (ret->prestored_rfs[argIndex] == 0) - ret->prestored_set->insert(ret->prestored_argVals[argIndex]); + // @bug 2584, use COMPARE_NIL for "= null" to allow "is null" in OR expression + for (argIndex = 0; argIndex < filterCount; argIndex++) + if (ret->prestored_rfs[argIndex] == 0) + ret->prestored_set_128->insert(ret->prestored_argVals128[argIndex]); + } + else + { + ret->prestored_set.reset(new prestored_set_t()); + + // @bug 2584, use COMPARE_NIL for "= null" to allow "is null" in OR expression + for (argIndex = 0; argIndex < filterCount; argIndex++) + if (ret->prestored_rfs[argIndex] == 0) + ret->prestored_set->insert(ret->prestored_argVals[argIndex]); + } } return ret; diff --git a/primitives/linux-port/primitiveprocessor.h b/primitives/linux-port/primitiveprocessor.h index 0e8a93396..af0ea1909 100644 --- a/primitives/linux-port/primitiveprocessor.h +++ b/primitives/linux-port/primitiveprocessor.h @@ -82,7 +82,26 @@ public: } }; +class pcfHasher128 +{ +public: + inline size_t operator()(const int128_t i) const + { + return *reinterpret_cast(&i); + } +}; + +class pcfEqual128 +{ +public: + inline bool operator()(const int128_t f1, const int128_t f2) const + { + return f1 == f2; + } +}; + typedef std::tr1::unordered_set prestored_set_t; +typedef std::tr1::unordered_set prestored_set_t_128; typedef std::tr1::unordered_set DictEqualityFilter; struct idb_regex_t @@ -109,9 +128,11 @@ struct ParsedColumnFilter { ColumnFilterMode columnFilterMode; boost::shared_array prestored_argVals; + boost::shared_array prestored_argVals128; boost::shared_array prestored_cops; boost::shared_array prestored_rfs; boost::shared_ptr prestored_set; + boost::shared_ptr prestored_set_128; boost::shared_array prestored_regex; uint8_t likeOps; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 31e15e1c8..d0b88ffd1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -9,13 +9,13 @@ endif() if (WITH_ARITHMETICOPERATOR_UT) add_executable(arithmeticoperator_tests arithmeticoperator-tests.cpp) target_link_libraries(arithmeticoperator_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) - install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) + install(TARGETS arithmeticoperator_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) endif() if (WITH_CSDECIMAL_UT) add_executable(mcs_decimal_tests mcs_decimal-tests.cpp) target_link_libraries(mcs_decimal_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) - install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) + install(TARGETS mcs_decimal_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) endif() if (WITH_DATACONVERT_UT) diff --git a/tests/dataconvert-tests.cpp b/tests/dataconvert-tests.cpp index 3e7f0d987..07b191b1a 100644 --- a/tests/dataconvert-tests.cpp +++ b/tests/dataconvert-tests.cpp @@ -729,5 +729,67 @@ TEST(DataConvertTest, DecimalToStringCheckScale38) EXPECT_EQ(string(buf), expected); } -TEST(DataConvertTest, ConvertColumnData) { +TEST(DataConvertTest, DecimalToStringCheckScale37) +{ + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::DECIMAL; + char buf[42]; + string expected; + ct.precision = 38; + ct.scale = 37; + int128_t res; + + // test simple values + res = 0; + expected = "0.0000000000000000000000000000000000000"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = 2; + expected = "0.0000000000000000000000000000000000002"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -2; + expected = "-0.0000000000000000000000000000000000002"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = 123; + expected = "0.0000000000000000000000000000000000123"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -123; + expected = "-0.0000000000000000000000000000000000123"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = ((((((int128_t)1234567890 * 10000000000) + 1234567890) * 10000000000) + 1234567890) * 100000000 ) + 12345678; + expected = "1.2345678901234567890123456789012345678"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -res; + expected = "-1.2345678901234567890123456789012345678"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + + // test max/min decimal (i.e. 38 9's) + res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + expected = "9.9999999999999999999999999999999999999"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -res; + expected = "-9.9999999999999999999999999999999999999"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + + // test trailing zeros + res = 123000; + expected = "0.0000000000000000000000000000000123000"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -res; + expected = "-0.0000000000000000000000000000000123000"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); +} + +TEST(DataConvertTest, ConvertColumnData) +{ } diff --git a/utils/common/columnwidth.h b/utils/common/columnwidth.h index 9d02a7312..d21e15230 100644 --- a/utils/common/columnwidth.h +++ b/utils/common/columnwidth.h @@ -23,7 +23,6 @@ namespace utils { const uint8_t MAXLEGACYWIDTH = 8ULL; - const uint8_t MAXCOLUMNWIDTH = 16ULL; inline bool isWide(uint8_t width) { diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 1a2086188..546c24057 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -391,29 +391,21 @@ void number_int_value(const string& data, case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - if (ct.colWidth == 1) + if (LIKELY(ct.colWidth == 16)) { - if (intVal < MIN_TINYINT) + int128_t tmp; + utils::int128Min(tmp); + if (intVal < tmp + 2) // + 2 for NULL and EMPTY values { - intVal = MIN_TINYINT; - pushwarning = true; - } - else if (intVal > MAX_TINYINT) - { - intVal = MAX_TINYINT; + intVal = tmp + 2; pushwarning = true; } } - else if (ct.colWidth == 2) + else if (ct.colWidth == 8) { - if (intVal < MIN_SMALLINT) + if (intVal < MIN_BIGINT) { - intVal = MIN_SMALLINT; - pushwarning = true; - } - else if (intVal > MAX_SMALLINT) - { - intVal = MAX_SMALLINT; + intVal = MIN_BIGINT; pushwarning = true; } } @@ -430,21 +422,29 @@ void number_int_value(const string& data, pushwarning = true; } } - else if (ct.colWidth == 8) + else if (ct.colWidth == 2) { - if (intVal < MIN_BIGINT) + if (intVal < MIN_SMALLINT) { - intVal = MIN_BIGINT; + intVal = MIN_SMALLINT; + pushwarning = true; + } + else if (intVal > MAX_SMALLINT) + { + intVal = MAX_SMALLINT; pushwarning = true; } } - else if (ct.colWidth == 16) + else if (ct.colWidth == 1) { - int128_t tmp; - utils::int128Min(tmp); - if (intVal < tmp + 2) // + 2 for NULL and EMPTY values + if (intVal < MIN_TINYINT) { - intVal = tmp + 2; + intVal = MIN_TINYINT; + pushwarning = true; + } + else if (intVal > MAX_TINYINT) + { + intVal = MAX_TINYINT; pushwarning = true; } } @@ -1469,31 +1469,31 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, break; case CalpontSystemCatalog::DECIMAL: - if (colType.colWidth == 1) + if (LIKELY(colType.colWidth == 16)) { - number_int_value(data, colType, pushWarning, noRoundup, val64); - value = (char) val64; - } - else if (colType.colWidth == 2) - { - number_int_value(data, colType, pushWarning, noRoundup, val64); - value = (short) val64; - } - else if (colType.colWidth == 4) - { - number_int_value(data, colType, pushWarning, noRoundup, val64); - value = (int) val64; + int128_t val128; + number_int_value(data, colType, pushWarning, noRoundup, val128); + value = (int128_t) val128; } else if (colType.colWidth == 8) { number_int_value(data, colType, pushWarning, noRoundup, val64); value = (long long) val64; } - else if (colType.colWidth == 16) + else if (colType.colWidth == 4) { - int128_t val128; - number_int_value(data, colType, pushWarning, noRoundup, val128); - value = (int128_t) val128; + number_int_value(data, colType, pushWarning, noRoundup, val64); + value = (int) val64; + } + else if (colType.colWidth == 2) + { + number_int_value(data, colType, pushWarning, noRoundup, val64); + value = (short) val64; + } + else if (colType.colWidth == 1) + { + number_int_value(data, colType, pushWarning, noRoundup, val64); + value = (char) val64; } //else if (colType.colWidth == 32) // value = data; @@ -1503,29 +1503,29 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, case CalpontSystemCatalog::UDECIMAL: // UDECIMAL numbers may not be negative - if (colType.colWidth == 1) + if (LIKELY(colType.colWidth == 16)) { - number_int_value(data, colType, pushWarning, noRoundup, val64); - char ival = (char) val64; + int128_t val128; + number_int_value(data, colType, pushWarning, noRoundup, val128); - if (ival < 0 && - ival != static_cast(joblist::TINYINTEMPTYROW) && - ival != static_cast(joblist::TINYINTNULL)) + if (val128 < 0 && + !utils::isWideDecimalNullValue(val128) && + !utils::isWideDecimalEmptyValue(val128)) { - ival = 0; + val128 = 0; pushWarning = true; } - value = ival; + value = val128; } - else if (colType.colWidth == 2) + else if (colType.colWidth == 8) { number_int_value(data, colType, pushWarning, noRoundup, val64); - short ival = (short) val64; + long long ival = static_cast(val64); if (ival < 0 && - ival != static_cast(joblist::SMALLINTEMPTYROW) && - ival != static_cast(joblist::SMALLINTNULL)) + ival != static_cast(joblist::BIGINTEMPTYROW) && + ival != static_cast(joblist::BIGINTNULL)) { ival = 0; pushWarning = true; @@ -1548,14 +1548,14 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, value = ival; } - else if (colType.colWidth == 8) + else if (colType.colWidth == 2) { number_int_value(data, colType, pushWarning, noRoundup, val64); - long long ival = static_cast(val64); + short ival = (short) val64; if (ival < 0 && - ival != static_cast(joblist::BIGINTEMPTYROW) && - ival != static_cast(joblist::BIGINTNULL)) + ival != static_cast(joblist::SMALLINTEMPTYROW) && + ival != static_cast(joblist::SMALLINTNULL)) { ival = 0; pushWarning = true; @@ -1563,20 +1563,20 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, value = ival; } - else if (colType.colWidth == 16) + else if (colType.colWidth == 1) { - int128_t val128; - number_int_value(data, colType, pushWarning, noRoundup, val128); + number_int_value(data, colType, pushWarning, noRoundup, val64); + char ival = (char) val64; - if (val128 < 0 && - !utils::isWideDecimalNullValue(val128) && - !utils::isWideDecimalEmptyValue(val128)) + if (ival < 0 && + ival != static_cast(joblist::TINYINTEMPTYROW) && + ival != static_cast(joblist::TINYINTNULL)) { - val128 = 0; + ival = 0; pushWarning = true; } - value = val128; + value = ival; } break; @@ -1865,31 +1865,31 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - if (colType.colWidth == CalpontSystemCatalog::ONE_BYTE) + if (LIKELY(colType.colWidth == 16)) { - char tinyintvalue = joblist::TINYINTNULL; - value = tinyintvalue; - } - else if (colType.colWidth == CalpontSystemCatalog::TWO_BYTE) - { - short smallintvalue = joblist::SMALLINTNULL; - value = smallintvalue; - } - else if (colType.colWidth == CalpontSystemCatalog::FOUR_BYTE) - { - int intvalue = joblist::INTNULL; - value = intvalue; + int128_t val; + utils::setWideDecimalNullValue(val); + value = val; } else if (colType.colWidth == CalpontSystemCatalog::EIGHT_BYTE) { long long eightbyte = joblist::BIGINTNULL; value = eightbyte; } - else if (colType.colWidth == 16) + else if (colType.colWidth == CalpontSystemCatalog::FOUR_BYTE) { - int128_t val; - utils::setWideDecimalNullValue(val); - value = val; + int intvalue = joblist::INTNULL; + value = intvalue; + } + else if (colType.colWidth == CalpontSystemCatalog::TWO_BYTE) + { + short smallintvalue = joblist::SMALLINTNULL; + value = smallintvalue; + } + else if (colType.colWidth == CalpontSystemCatalog::ONE_BYTE) + { + char tinyintvalue = joblist::TINYINTNULL; + value = tinyintvalue; } else { diff --git a/utils/joiner/tuplejoiner.cpp b/utils/joiner/tuplejoiner.cpp index 6297699a6..9282e7253 100644 --- a/utils/joiner/tuplejoiner.cpp +++ b/utils/joiner/tuplejoiner.cpp @@ -1147,7 +1147,7 @@ void TupleJoiner::updateCPData(const Row& r) } } } - else if (r.getColumnWidth(colIdx) == utils::MAXCOLUMNWIDTH + else if (r.getColumnWidth(colIdx) == datatypes::MAXDECIMALWIDTH && (r.getColType(colIdx) == CalpontSystemCatalog::DECIMAL || r.getColType(colIdx) == CalpontSystemCatalog::UDECIMAL)) { diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 3bb0f95ea..e8dc38f50 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -639,9 +639,9 @@ string Row::toString() const break; case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - if (colWidths[i] == utils::MAXCOLUMNWIDTH) + if (colWidths[i] == datatypes::MAXDECIMALWIDTH) { - unsigned int buflen = precision[i] + 3; + unsigned int buflen = utils::MAXLENGTH16BYTES; char *buf = (char*)alloca(buflen); // empty the buffer dataconvert::DataConvert::decimalToString(getBinaryField(i), diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index f3c7fec39..edd552c93 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -1235,7 +1235,7 @@ inline void Row::copyField(Row& out, uint32_t destIndex, uint32_t srcIndex) cons // WIP MCOL-641 inline void Row::copyBinaryField(Row& out, uint32_t destIndex, uint32_t srcIndex) const { - out.setBinaryField(getBinaryField(srcIndex), 16, destIndex); + out.setBinaryField(getBinaryField(srcIndex), 16, destIndex); } inline void Row::setRid(uint64_t rid) diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index b899dcd8b..79d6cf275 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -39,6 +39,7 @@ #include "brmtypes.h" #include "dataconvert.h" #include "exceptclasses.h" +#include "mcs_decimal.h" #include "joblisttypes.h" @@ -986,18 +987,18 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, if ( (column.dataType == CalpontSystemCatalog::DECIMAL) || (column.dataType == CalpontSystemCatalog::UDECIMAL)) { - if (width <= 8) - { - // errno is initialized and set in convertDecimalString - llVal = Convertor::convertDecimalString( - field, fieldLength, column.scale ); - } - else + if (LIKELY(width == datatypes::MAXDECIMALWIDTH)) { bool saturate = false; bigllVal = dataconvert::string_to_ll(string(field), saturate); // TODO MCOL-641 check saturate } + else if (width <= 8) + { + // errno is initialized and set in convertDecimalString + llVal = Convertor::convertDecimalString( + field, fieldLength, column.scale ); + } } else { diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 3f3f732a2..dbbc5162c 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -2622,7 +2622,6 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, uint64_t& blocksChanged) { uint8_t rc = 0; - //cout << " In processUpdate" << endl; uint32_t tmp32, sessionID; TxnID txnId; bs >> PMId; @@ -2639,7 +2638,6 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, uint8_t pkgType; bs >> pkgType; cpackages[txnId].read(bs); - //cout << "Processed meta data in update" << endl; rc = fWEWrapper.startTransaction(txnId); @@ -2811,9 +2809,20 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, rid = relativeRID; convertToRelativeRid (rid, extentNum, blockNum); rowIDLists.push_back(rid); - uint32_t colWidth = ((colTypes[j].colWidth > 8 && - !(colTypes[j].colDataType == CalpontSystemCatalog::DECIMAL || - colTypes[j].colDataType == CalpontSystemCatalog::UDECIMAL)) ? 8 : colTypes[j].colWidth); + + uint32_t colWidth = colTypes[j].colWidth; + + if (colWidth > 8 && + !(colTypes[j].colDataType == CalpontSystemCatalog::DECIMAL || + colTypes[j].colDataType == CalpontSystemCatalog::UDECIMAL)) + { + colWidth = 8; + } + else if (colWidth >= datatypes::MAXDECIMALWIDTH) + { + colWidth = datatypes::MAXDECIMALWIDTH; + } + int rrid = (int) relativeRID / (BYTE_PER_BLOCK / colWidth); // populate stats.blocksChanged if (rrid > preBlkNums[j]) @@ -2984,16 +2993,18 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, case CalpontSystemCatalog::UDECIMAL: { // WIP MCOL-641 - if (fetchColColwidths[fetchColPos] == 16) + if (fetchColColwidths[fetchColPos] == datatypes::MAXDECIMALWIDTH) { int128_t* dec; char buf[utils::MAXLENGTH16BYTES]; dec = row.getBinaryField(fetchColPos); + dataconvert::DataConvert::decimalToString(dec, (unsigned)fetchColScales[fetchColPos], buf, sizeof(buf), fetchColTypes[fetchColPos]); value.assign(buf); + break; } @@ -3347,17 +3358,19 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - if (fetchColColwidths[fetchColPos] == 16) + if (fetchColColwidths[fetchColPos] == datatypes::MAXDECIMALWIDTH) { // WIP MCOL-641 int128_t* dec; char buf[utils::MAXLENGTH16BYTES]; dec = row.getBinaryField(fetchColPos); + dataconvert::DataConvert::decimalToString(dec, (unsigned)fetchColScales[fetchColPos], buf, sizeof(buf), fetchColTypes[fetchColPos]); value = buf; + break; } @@ -3994,7 +4007,6 @@ uint8_t WE_DMLCommandProc::processDelete(messageqcpp::ByteStream& bs, uint64_t& blocksChanged) { uint8_t rc = 0; - //cout << " In processDelete" << endl; uint32_t tmp32, sessionID; TxnID txnId; bs >> PMId; @@ -4069,7 +4081,18 @@ uint8_t WE_DMLCommandProc::processDelete(messageqcpp::ByteStream& bs, for (uint32_t j = 0; j < row.getColumnCount(); j++) { preBlkNums[j] = -1; - colWidth[j] = (row.getColumnWidth(j) >= 16 ? 16 : row.getColumnWidth(j)); + colWidth[j] = row.getColumnWidth(j); + execplan::CalpontSystemCatalog::ColDataType colDataType = row.getColType(j); + if (colWidth[j] >= 8 && + !(colDataType == execplan::CalpontSystemCatalog::DECIMAL || + colDataType == execplan::CalpontSystemCatalog::UDECIMAL)) + { + colWidth[j] = 8; + } + else if (colWidth[j] >= datatypes::MAXDECIMALWIDTH) + { + colWidth[j] = datatypes::MAXDECIMALWIDTH; + } } //Get the file information from rowgroup diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index a89b48199..bb7489a48 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -70,9 +70,6 @@ namespace WriteEngine { StopWatch timer; -using dataconvert::int128_t; -using dataconvert::uint128_t; - /**@brief WriteEngineWrapper Constructor */ WriteEngineWrapper::WriteEngineWrapper() : m_opType(NOOP) @@ -803,21 +800,9 @@ int WriteEngineWrapper::deleteRow(const TxnID& txnid, const vector cscColType = cscColTypeList[i]; Convertor::convertColType(&curColStruct); - /*if (curColStruct.colType == WriteEngine::WR_BINARY) - { - uint128_t bigEmptyVal; - emptyVal = m_colOp[op(curColStruct.fCompressionType)]-> - getEmptyRowValue(curColStruct.colDataType, curColStruct.colWidth); - *(reinterpret_cast(&bigEmptyVal)) = emptyVal; - *(reinterpret_cast(&bigEmptyVal) + 1) = emptyVal; - curTuple.data = bigEmptyVal; - } - else - {*/ - m_colOp[op(curColStruct.fCompressionType)]-> - getEmptyRowValue(curColStruct.colDataType, curColStruct.colWidth, (uint8_t*)&emptyVal); - curTuple.data = emptyVal; - //} + m_colOp[op(curColStruct.fCompressionType)]-> + getEmptyRowValue(curColStruct.colDataType, curColStruct.colWidth, (uint8_t*)&emptyVal); + curTuple.data = emptyVal; curTupleList.push_back(curTuple); colValueList.push_back(curTupleList); @@ -4371,7 +4356,6 @@ int WriteEngineWrapper::updateColumnRecs(const TxnID& txnid, ColumnOp* colOp = NULL; bool successFlag = true; unsigned width = 0; - \ int curFbo = 0, curBio, lastFbo = -1; RID aRid = ridLists[0]; int rc = 0; From b5534eb8477b53dfc43711ed5d5eb9b54ab852d1 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 24 Mar 2020 13:41:28 +0000 Subject: [PATCH 32/78] MCOL-641 Refactored MultiplicationOverflowCheck but it still has flaws. Introduced fDecimalOverflowCheck to enable/disable overflow check. Add support into a FunctionColumn. Low level scanning crashes on medium sized data sets. --- datatypes/mcs_decimal.cpp | 2 - datatypes/mcs_decimal.h | 16 +- dbcon/execplan/arithmeticcolumn.cpp | 5 +- dbcon/execplan/arithmeticoperator.cpp | 15 +- dbcon/execplan/arithmeticoperator.h | 15 +- dbcon/execplan/functioncolumn.h | 33 +- dbcon/execplan/parsetree.h | 1 + dbcon/execplan/predicateoperator.h | 430 ++++++++++++++++++++++++++ dbcon/execplan/simplecolumn_int.h | 7 +- dbcon/execplan/simplecolumn_uint.h | 4 +- dbcon/mysql/ha_mcs_execplan.cpp | 19 +- primitives/primproc/columncommand.cpp | 5 +- tests/mcs_decimal-tests.cpp | 12 + 13 files changed, 528 insertions(+), 36 deletions(-) diff --git a/datatypes/mcs_decimal.cpp b/datatypes/mcs_decimal.cpp index 007ac9c81..cc227f215 100644 --- a/datatypes/mcs_decimal.cpp +++ b/datatypes/mcs_decimal.cpp @@ -92,7 +92,6 @@ namespace datatypes { int128_t scaleMultiplier; getScaleDivisor(scaleMultiplier, r.scale - result.scale); - mulOverflowCheck(rValue, scaleMultiplier); rValue /= scaleMultiplier; } @@ -103,7 +102,6 @@ namespace datatypes result.s128Value = op(lValue, rValue); } - // This is wide Decimal version only ATM std::string Decimal::toString(execplan::IDB_Decimal& value) { char buf[utils::MAXLENGTH16BYTES]; diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index aab919e92..93b827207 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -185,12 +185,24 @@ struct DivisionOverflowCheck { struct MultiplicationOverflowCheck { void operator()(const int128_t& x, const int128_t& y) { - if (x * y / x != y) + if (x * y / y != x) { throw logging::OperationOverflowExcept( - "Decimal::multiplication or scale multiplicationproduces an overflow."); + "Decimal::multiplication or scale multiplication \ + produces an overflow."); } } + bool operator()(const int128_t& x, const int128_t& y, int128_t& r) + { + if ((r = x * y) / y != x) + { + throw logging::OperationOverflowExcept( + "Decimal::multiplication or scale multiplication \ + produces an overflow."); + } + return true; + } + }; /** diff --git a/dbcon/execplan/arithmeticcolumn.cpp b/dbcon/execplan/arithmeticcolumn.cpp index 2128a47ed..e88adf899 100644 --- a/dbcon/execplan/arithmeticcolumn.cpp +++ b/dbcon/execplan/arithmeticcolumn.cpp @@ -62,7 +62,7 @@ namespace execplan */ ArithmeticColumn::ArithmeticColumn(): ReturnedColumn(), - fExpression (0) + fExpression(0) {} ArithmeticColumn::ArithmeticColumn(const string& sql, const uint32_t sessionID): @@ -314,8 +314,7 @@ const string ArithmeticColumn::toString() const oss << "expressionId=" << fExpressionId << endl; oss << "joinInfo=" << fJoinInfo << " returnAll=" << fReturnAll << " sequence#=" << fSequence << endl; - oss << "resultType=" << colDataTypeToString(fResultType.colDataType) << "|" << fResultType.colWidth << - endl; + oss << "resultType=" << colDataTypeToString(fResultType.colDataType) << "|" << fResultType.colWidth << endl; return oss.str(); } diff --git a/dbcon/execplan/arithmeticoperator.cpp b/dbcon/execplan/arithmeticoperator.cpp index d968ceb5b..610bf2ad7 100644 --- a/dbcon/execplan/arithmeticoperator.cpp +++ b/dbcon/execplan/arithmeticoperator.cpp @@ -34,17 +34,20 @@ namespace execplan /** * Constructors/Destructors */ -ArithmeticOperator::ArithmeticOperator() : Operator() +ArithmeticOperator::ArithmeticOperator() : Operator(), + fDecimalOverflowCheck(true) { } -ArithmeticOperator::ArithmeticOperator(const string& operatorName): Operator(operatorName) +ArithmeticOperator::ArithmeticOperator(const string& operatorName): Operator(operatorName), + fDecimalOverflowCheck(true) { } ArithmeticOperator::ArithmeticOperator(const ArithmeticOperator& rhs): Operator(rhs), - fTimeZone(rhs.timeZone()) + fTimeZone(rhs.timeZone()), + fDecimalOverflowCheck(true) { } @@ -63,6 +66,7 @@ ostream& operator<<(ostream& output, const ArithmeticOperator& rhs) { output << rhs.toString(); output << "opType=" << rhs.operationType().colDataType << endl; + output << "decimalOverflowCheck=" << rhs.getOverflowCheck() << endl; return output; } @@ -73,6 +77,8 @@ void ArithmeticOperator::serialize(messageqcpp::ByteStream& b) const { b << (ObjectReader::id_t) ObjectReader::ARITHMETICOPERATOR; b << fTimeZone; + const messageqcpp::ByteStream::byte tmp = fDecimalOverflowCheck; + b << tmp; Operator::serialize(b); } @@ -80,6 +86,9 @@ void ArithmeticOperator::unserialize(messageqcpp::ByteStream& b) { ObjectReader::checkType(b, ObjectReader::ARITHMETICOPERATOR); b >> fTimeZone; + messageqcpp::ByteStream::byte tmp; + b >> tmp; + fDecimalOverflowCheck = tmp; Operator::unserialize(b); } diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index bc6f02a01..88c4c1f11 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -196,6 +196,14 @@ public: return TreeNode::getBoolVal(); } void adjustResultType(const CalpontSystemCatalog::ColType& m); + constexpr inline bool getOverflowCheck() + { + return fDecimalOverflowCheck; + } + inline void setOverflowCheck(bool check) + { + fDecimalOverflowCheck = check; + } private: template @@ -203,6 +211,7 @@ private: inline void execute(IDB_Decimal& result, IDB_Decimal op1, IDB_Decimal op2, bool& isNull); inline void execute(IDB_Decimal& result, IDB_Decimal op1, IDB_Decimal op2, bool& isNull, cscType& resultCscType); std::string fTimeZone; + bool fDecimalOverflowCheck; }; #include "parsetree.h" @@ -288,12 +297,12 @@ inline void ArithmeticOperator::execute(IDB_Decimal& result, IDB_Decimal op1, ID switch (fOp) { case OP_ADD: - if (resultCscType.colWidth == 16) + if (resultCscType.colWidth == datatypes::MAXDECIMALWIDTH) { - datatypes::Decimal::addition( + datatypes::Decimal::addition( op1, op2, result); } - else if (resultCscType.colWidth == 8) + else if (resultCscType.colWidth == utils::MAXLEGACYWIDTH) { datatypes::Decimal::addition( op1, op2, result); diff --git a/dbcon/execplan/functioncolumn.h b/dbcon/execplan/functioncolumn.h index 2be25897f..9952910ad 100644 --- a/dbcon/execplan/functioncolumn.h +++ b/dbcon/execplan/functioncolumn.h @@ -246,15 +246,34 @@ public: { IDB_Decimal decimal = fFunctor->getDecimalVal(row, fFunctionParms, isNull, fOperationType); - if (fResultType.scale == decimal.scale) + if (UNLIKELY(fResultType.colWidth == utils::MAXLEGACYWIDTH + && fResultType.scale == decimal.scale)) return decimal; - if (fResultType.scale > decimal.scale) - decimal.value *= IDB_pow[fResultType.scale - decimal.scale]; - else - decimal.value = (int64_t)(decimal.value > 0 ? - (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] + 0.5 : - (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] - 0.5); + if (LIKELY(fResultType.colWidth == datatypes::MAXDECIMALWIDTH)) + { + decimal.s128Value = + (datatypes::Decimal::isWideDecimalType(decimal.precision)) ? + decimal.s128Value : decimal.value; + + int128_t scaleMultiplier, result; + int32_t scaleDiff = fResultType.scale - decimal.scale; + datatypes::getScaleDivisor(scaleMultiplier, abs(scaleDiff)); + // WIP MCOL-641 Unconditionall overflow check + datatypes::MultiplicationOverflowCheck mul; + decimal.s128Value = (scaleDiff > 0 + && mul(decimal.s128Value, scaleMultiplier, result)) + ? result : decimal.s128Value / scaleMultiplier; + } + else if (fResultType.colWidth == utils::MAXLEGACYWIDTH) + { + if (fResultType.scale > decimal.scale) + decimal.value *= IDB_pow[fResultType.scale - decimal.scale]; + else + decimal.value = (int64_t)(decimal.value > 0 ? + (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] + 0.5 : + (double)decimal.value / IDB_pow[decimal.scale - fResultType.scale] - 0.5); + } decimal.scale = fResultType.scale; decimal.precision = fResultType.precision; diff --git a/dbcon/execplan/parsetree.h b/dbcon/execplan/parsetree.h index f31a2a013..34131615d 100644 --- a/dbcon/execplan/parsetree.h +++ b/dbcon/execplan/parsetree.h @@ -30,6 +30,7 @@ #include #include "treenode.h" #include "operator.h" +#include "mcs_decimal.h" namespace rowgroup { diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index 4cd5d37f0..66120eb9c 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -120,6 +120,7 @@ public: void setOpType(Type& l, Type& r); private: + inline bool numericCompare(IDB_Decimal& op1, IDB_Decimal& op2); template inline bool numericCompare(result_t op1, result_t op2); inline bool strTrimCompare(const std::string& op1, const std::string& op2); @@ -127,6 +128,435 @@ private: const CHARSET_INFO* cs; }; +inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, ReturnedColumn* lop, ReturnedColumn* rop) +{ + // like operator. both sides are string. + if (fOp == OP_LIKE || fOp == OP_NOTLIKE) + { + SP_CNX_Regex regex = rop->regex(); + + // Ugh. The strings returned by getStrVal have null padding out to the col width. boost::regex + // considers these nulls significant, but they're not in the pattern, so we need to strip + // them off... + const std::string& v = lop->getStrVal(row, isNull); +// char* c = (char*)alloca(v.length() + 1); +// memcpy(c, v.c_str(), v.length()); +// c[v.length()] = 0; +// std::string vv(c); + + if (regex) + { +#ifdef POSIX_REGEX + bool ret = regexec(regex.get(), v.c_str(), 0, NULL, 0) == 0; +#else + bool ret = boost::regex_match(v.c_str(), *regex); +#endif + return (((fOp == OP_LIKE) ? ret : !ret) && !isNull); + } + else + { +#ifdef POSIX_REGEX + regex_t regex; + std::string str = dataconvert::DataConvert::constructRegexp(rop->getStrVal(row, isNull)); + regcomp(®ex, str.c_str(), REG_NOSUB | REG_EXTENDED); + bool ret = regexec(®ex, v.c_str(), 0, NULL, 0) == 0; + regfree(®ex); +#else + boost::regex regex(dataconvert::DataConvert::constructRegexp(rop->getStrVal(row, isNull))); + bool ret = boost::regex_match(v.c_str(), regex); +#endif + return (((fOp == OP_LIKE) ? ret : !ret) && !isNull); + } + } + + // fOpType should have already been set on the connector during parsing + switch (fOperationType.colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + { + if (fOp == OP_ISNULL) + { + lop->getIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + int64_t val1 = lop->getIntVal(row, isNull); + + if (isNull) + return false; + + return numericCompare(val1, rop->getIntVal(row, isNull)) && !isNull; + } + + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + if (fOp == OP_ISNULL) + { + lop->getUintVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getUintVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + uint64_t val1 = lop->getUintVal(row, isNull); + + if (isNull) + return false; + + return numericCompare(val1, rop->getUintVal(row, isNull)) && !isNull; + } + + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + { + if (fOp == OP_ISNULL) + { + lop->getDoubleVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getDoubleVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + double val1 = lop->getDoubleVal(row, isNull); + + if (isNull) + return false; + + return numericCompare(val1, rop->getDoubleVal(row, isNull)) && !isNull; + } + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + if (fOp == OP_ISNULL) + { + lop->getLongDoubleVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getLongDoubleVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + long double val1 = lop->getLongDoubleVal(row, isNull); + if (isNull) + return false; + + long double val2 = rop->getLongDoubleVal(row, isNull); + if (isNull) + return false; + + // In many case, rounding error will prevent an eq compare to work + // In these cases, use the largest scale of the two items. + if (fOp == execplan::OP_EQ) + { + // In case a val is a representation of a very large integer, + // we won't want to just multiply by scale, as it may move + // significant digits out of scope. So we break them apart + // and compare each separately + int64_t scale = std::max(lop->resultType().scale, rop->resultType().scale); + if (scale) + { + long double intpart1; + long double fract1 = modfl(val1, &intpart1); + long double intpart2; + long double fract2 = modfl(val2, &intpart2); + if (numericCompare(intpart1, intpart2)) + { + double factor = pow(10.0, (double)scale); + fract1 = roundl(fract1 * factor); + fract2 = roundl(fract2 * factor); + return numericCompare(fract1, fract2); + } + else + { + return false; + } + } + } + return numericCompare(val1, val2); + } + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (fOp == OP_ISNULL) + { + lop->getDecimalVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getDecimalVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + IDB_Decimal val1 = lop->getDecimalVal(row, isNull); + + if (isNull) + return false; + + return numericCompare(val1, rop->getDecimalVal(row, isNull)) && !isNull; + } + + case execplan::CalpontSystemCatalog::DATE: + { + if (fOp == OP_ISNULL) + { + lop->getDateIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getDateIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + int64_t val1 = lop->getDateIntVal(row, isNull); + + if (isNull) + return false; + + return numericCompare(val1, (int64_t)rop->getDateIntVal(row, isNull)) && !isNull; + } + + case execplan::CalpontSystemCatalog::DATETIME: + { + if (fOp == OP_ISNULL) + { + lop->getDatetimeIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getDatetimeIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + int64_t val1 = lop->getDatetimeIntVal(row, isNull); + + if (isNull) + return false; + + return numericCompare(val1, rop->getDatetimeIntVal(row, isNull)) && !isNull; + } + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + if (fOp == OP_ISNULL) + { + lop->getTimestampIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getTimestampIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + int64_t val1 = lop->getTimestampIntVal(row, isNull); + + if (isNull) + return false; + + return numericCompare(val1, rop->getTimestampIntVal(row, isNull)) && !isNull; + } + + case execplan::CalpontSystemCatalog::TIME: + { + if (fOp == OP_ISNULL) + { + lop->getTimeIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getTimeIntVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + int64_t val1 = lop->getTimeIntVal(row, isNull); + + if (isNull) + return false; + + return numericCompare(val1, rop->getTimeIntVal(row, isNull)) && !isNull; + } + + + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + if (fOp == OP_ISNULL) + { + lop->getStrVal(row, isNull); + bool ret = isNull; + isNull = false; + return ret; + } + + if (fOp == OP_ISNOTNULL) + { + lop->getStrVal(row, isNull); + bool ret = isNull; + isNull = false; + return !ret; + } + + if (isNull) + return false; + + const std::string& val1 = lop->getStrVal(row, isNull); + if (isNull) + return false; + + return strTrimCompare(val1, rop->getStrVal(row, isNull)) && !isNull; +// return strCompare(val1, rop->getStrVal(row, isNull)) && !isNull; + + } + + // MCOL-641 WIP This is an incorrect assumption. + case execplan::CalpontSystemCatalog::VARBINARY: + case execplan::CalpontSystemCatalog::BLOB: + return false; + break; + + default: + { + std::ostringstream oss; + oss << "invalid predicate operation type: " << fOperationType.colDataType; + throw logging::InvalidOperationExcept(oss.str()); + } + } + + return false; +} + +inline bool PredicateOperator::numericCompare(IDB_Decimal& op1, IDB_Decimal& op2) +{ + switch (fOp) + { + case OP_EQ: + return op1 == op2; + + case OP_NE: + return op1 != op2; + + case OP_GT: + return op1 > op2; + + case OP_GE: + return op1 >= op2; + + case OP_LT: + return op1 < op2; + + case OP_LE: + return op1 <= op2; + + default: + { + std::ostringstream oss; + oss << "invalid predicate operation: " << fOp; + throw logging::InvalidOperationExcept(oss.str()); + } + } +} template inline bool PredicateOperator::numericCompare(result_t op1, result_t op2) diff --git a/dbcon/execplan/simplecolumn_int.h b/dbcon/execplan/simplecolumn_int.h index 4a770ec89..7e00207ca 100644 --- a/dbcon/execplan/simplecolumn_int.h +++ b/dbcon/execplan/simplecolumn_int.h @@ -32,6 +32,7 @@ #include "objectreader.h" #include "joblisttypes.h" #include "rowgroup.h" +#include "mcs_decimal.h" /** * Namespace @@ -217,7 +218,7 @@ inline IDB_Decimal SimpleColumn_INT::getDecimalVal(rowgroup::Row& row, bool isNull = true; fResult.decimalVal.value = (int64_t)row.getIntField(fInputIndex); - fResult.decimalVal.precision = 65; + fResult.decimalVal.precision = datatypes::INT64MAXPRECISION; fResult.decimalVal.scale = 0; return fResult.decimalVal; } @@ -242,8 +243,6 @@ void SimpleColumn_INT::serialize(messageqcpp::ByteStream& b) const case 8: b << (ObjectReader::id_t) ObjectReader::SIMPLECOLUMN_INT8; break; - case 16: - std::cout << __FILE__<< ":" << __LINE__ << " Fix for 16 Bytes ?" << std::endl; } SimpleColumn::serialize(b); @@ -269,8 +268,6 @@ void SimpleColumn_INT::unserialize(messageqcpp::ByteStream& b) case 8: ObjectReader::checkType(b, ObjectReader::SIMPLECOLUMN_INT8); break; - case 16: - std::cout << __FILE__<< ":" << __LINE__ << " Fix for 16 Bytes ?" << std::endl; } SimpleColumn::unserialize(b); diff --git a/dbcon/execplan/simplecolumn_uint.h b/dbcon/execplan/simplecolumn_uint.h index 289a62463..7a6da72e4 100644 --- a/dbcon/execplan/simplecolumn_uint.h +++ b/dbcon/execplan/simplecolumn_uint.h @@ -32,6 +32,7 @@ #include "objectreader.h" #include "joblisttypes.h" #include "rowgroup.h" +#include "mcs_decimal.h" /** * Namespace @@ -218,7 +219,8 @@ inline IDB_Decimal SimpleColumn_UINT::getDecimalVal(rowgroup::Row& row, boo isNull = true; fResult.decimalVal.value = (uint64_t)row.getUintField(fInputIndex); - fResult.decimalVal.precision = 65; + // WIP MCOL-641 + fResult.decimalVal.precision = datatypes::INT64MAXPRECISION+1; fResult.decimalVal.scale = 0; return fResult.decimalVal; } diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 043da5168..5a3260b65 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -3106,14 +3106,16 @@ CalpontSystemCatalog::ColType colType_MysqlToIDB (const Item* item) { Item_decimal* idp = (Item_decimal*)item; ct.colDataType = CalpontSystemCatalog::DECIMAL; - // MCOL-641 WIP Make this dynamic - ct.colWidth = (idp->max_length >= 18) ? 16 : 8; + ct.colWidth = (idp->max_length >= datatypes::INT64MAXPRECISION) + ? datatypes::MAXDECIMALWIDTH : utils::MAXLEGACYWIDTH; ct.scale = idp->decimals; - + // WIP MCOL-641 if (ct.scale == 0) - ct.precision = (idp->max_length > 38) ? 38 : idp->max_length - 1; + ct.precision = (idp->max_length > datatypes::INT128MAXPRECISION) + ? datatypes::INT128MAXPRECISION : idp->max_length - 1; else - ct.precision = (idp->max_length > 38) ? 38 : idp->max_length - idp->decimals; + ct.precision = (idp->max_length > datatypes::INT128MAXPRECISION ) + ? datatypes::INT128MAXPRECISION : idp->max_length - idp->decimals; break; } @@ -4365,12 +4367,13 @@ ConstantColumn* buildDecimalColumn(Item* item, gp_walk_info& gwi) columnstore_decimal_val << str->ptr()[i]; } - if (idp->decimal_precision() <= 18) + if (idp->decimal_precision() <= datatypes::INT64MAXPRECISION) columnstore_decimal.value = strtoll(columnstore_decimal_val.str().c_str(), 0, 10); - else + else if (idp->decimal_precision() <= datatypes::INT128MAXPRECISION) { bool dummy = false; - columnstore_decimal.s128Value = dataconvert::strtoll128(columnstore_decimal_val.str().c_str(), dummy, 0); + columnstore_decimal.s128Value = + dataconvert::strtoll128(columnstore_decimal_val.str().c_str(), dummy, 0); } // TODO MCOL-641 Add support here diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index 39009bfbf..6ab2929d6 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -153,8 +153,9 @@ void ColumnCommand::loadData() bool lastBlockReached = false; oidLastLbid = getLastLbid(); uint32_t blocksToLoad = 0; - BRM::LBID_t* lbids = (BRM::LBID_t*) alloca(8 * sizeof(BRM::LBID_t)); - uint8_t** blockPtrs = (uint8_t**) alloca(8 * sizeof(uint8_t*)); + // WIP MCOL-641 + BRM::LBID_t* lbids = (BRM::LBID_t*) alloca(16 * sizeof(BRM::LBID_t)); + uint8_t** blockPtrs = (uint8_t**) alloca(16 * sizeof(uint8_t*)); int i; diff --git a/tests/mcs_decimal-tests.cpp b/tests/mcs_decimal-tests.cpp index b41ec8751..b3a041867 100644 --- a/tests/mcs_decimal-tests.cpp +++ b/tests/mcs_decimal-tests.cpp @@ -729,3 +729,15 @@ TEST(Decimal, additionWithOverflowCheck) EXPECT_EQ(38, result.precision); EXPECT_EQ(l.s128Value, result.s128Value); } + +TEST(Decimal, multiplicationWithOverflowCheck) +{ + datatypes::MultiplicationOverflowCheck mul; + int128_t x = 42, y = 42, r = 0; + execplan::IDB_Decimal d; + EXPECT_NO_THROW(mul(x, y, r)); + EXPECT_EQ(x*y, r); + + x = datatypes::Decimal::maxInt128, y = 42, r = 0; + EXPECT_THROW(mul(x, y, r), logging::OperationOverflowExcept); +} From 554c6da8e87b8658698f7ea9cdeeea2c7c1ebff5 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Fri, 3 Apr 2020 13:30:10 -0400 Subject: [PATCH 33/78] MCOL-641 Implement int128_t versions of arithmetic operations and add unit test cases. --- datatypes/mcs_decimal.cpp | 359 ++++++++-- datatypes/mcs_decimal.h | 162 ++++- dbcon/execplan/arithmeticoperator.cpp | 6 +- dbcon/execplan/arithmeticoperator.h | 148 ++-- dbcon/mysql/ha_mcs_execplan.cpp | 73 +- dbcon/mysql/ha_mcs_sysvars.cpp | 19 + dbcon/mysql/ha_mcs_sysvars.h | 3 + tests/mcs_decimal-tests.cpp | 986 ++++++++++++++++---------- 8 files changed, 1254 insertions(+), 502 deletions(-) diff --git a/datatypes/mcs_decimal.cpp b/datatypes/mcs_decimal.cpp index cc227f215..325f67721 100644 --- a/datatypes/mcs_decimal.cpp +++ b/datatypes/mcs_decimal.cpp @@ -48,7 +48,7 @@ namespace datatypes template - void execute(const execplan::IDB_Decimal& l, + void addSubtractExecute(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result, BinaryOperation op, @@ -57,7 +57,7 @@ namespace datatypes { int128_t lValue = Decimal::isWideDecimalType(l.precision) ? l.s128Value : l.value; - int128_t rValue = Decimal::isWideDecimalType(l.precision) + int128_t rValue = Decimal::isWideDecimalType(r.precision) ? r.s128Value : r.value; if (result.scale == l.scale && result.scale == r.scale) @@ -78,7 +78,9 @@ namespace datatypes { int128_t scaleMultiplier; getScaleDivisor(scaleMultiplier, l.scale - result.scale); - lValue /= scaleMultiplier; + lValue = (int128_t) (lValue > 0 ? + (__float128)lValue / scaleMultiplier + 0.5 : + (__float128)lValue / scaleMultiplier - 0.5); } if (result.scale > r.scale) @@ -92,7 +94,9 @@ namespace datatypes { int128_t scaleMultiplier; getScaleDivisor(scaleMultiplier, r.scale - result.scale); - rValue /= scaleMultiplier; + rValue = (int128_t) (rValue > 0 ? + (__float128)rValue / scaleMultiplier + 0.5 : + (__float128)rValue / scaleMultiplier - 0.5); } // We assume there is no way that lValue or rValue calculations @@ -102,6 +106,94 @@ namespace datatypes result.s128Value = op(lValue, rValue); } + template + void divisionExecute(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, + execplan::IDB_Decimal& result, + OpOverflowCheck opOverflowCheck, + MultiplicationOverflowCheck mulOverflowCheck) + { + int128_t lValue = Decimal::isWideDecimalType(l.precision) + ? l.s128Value : l.value; + int128_t rValue = Decimal::isWideDecimalType(r.precision) + ? r.s128Value : r.value; + + opOverflowCheck(lValue, rValue); + + if (result.scale >= l.scale - r.scale) + { + int128_t scaleMultiplier; + + getScaleDivisor(scaleMultiplier, result.scale - (l.scale - r.scale)); + + // TODO How do we check overflow of (int128_t)((__float128)lValue / rValue * scaleMultiplier) ? + + result.s128Value = (int128_t)(( (lValue > 0 && rValue > 0) || (lValue < 0 && rValue < 0) ? + (__float128)lValue / rValue * scaleMultiplier + 0.5 : + (__float128)lValue / rValue * scaleMultiplier - 0.5)); + } + else + { + int128_t scaleMultiplier; + + getScaleDivisor(scaleMultiplier, (l.scale - r.scale) - result.scale); + + result.s128Value = (int128_t)(( (lValue > 0 && rValue > 0) || (lValue < 0 && rValue < 0) ? + (__float128)lValue / rValue / scaleMultiplier + 0.5 : + (__float128)lValue / rValue / scaleMultiplier - 0.5)); + } + } + + template + void multiplicationExecute(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, + execplan::IDB_Decimal& result, + OpOverflowCheck opOverflowCheck, + MultiplicationOverflowCheck mulOverflowCheck) + { + int128_t lValue = Decimal::isWideDecimalType(l.precision) + ? l.s128Value : l.value; + int128_t rValue = Decimal::isWideDecimalType(r.precision) + ? r.s128Value : r.value; + + if (lValue == 0 || rValue == 0) + { + result.s128Value = 0; + return; + } + + if (result.scale >= l.scale + r.scale) + { + int128_t scaleMultiplier; + + getScaleDivisor(scaleMultiplier, result.scale - (l.scale + r.scale)); + + opOverflowCheck(lValue, rValue, result.s128Value); + opOverflowCheck(result.s128Value, scaleMultiplier, result.s128Value); + } + else + { + unsigned int diff = l.scale + r.scale - result.scale; + + int128_t scaleMultiplierL, scaleMultiplierR; + + getScaleDivisor(scaleMultiplierL, diff / 2); + getScaleDivisor(scaleMultiplierR, diff - (diff / 2)); + + lValue = (int128_t)(( (lValue > 0) ? + (__float128)lValue / scaleMultiplierL + 0.5 : + (__float128)lValue / scaleMultiplierL - 0.5)); + + rValue = (int128_t)(( (rValue > 0) ? + (__float128)rValue / scaleMultiplierR + 0.5 : + (__float128)rValue / scaleMultiplierR - 0.5)); + + opOverflowCheck(lValue, rValue, result.s128Value);; + } + } + std::string Decimal::toString(execplan::IDB_Decimal& value) { char buf[utils::MAXLENGTH16BYTES]; @@ -163,13 +255,9 @@ namespace datatypes { std::plus add; NoOverflowCheck noOverflowCheck; - execute(l, r, result, add, noOverflowCheck, noOverflowCheck); + addSubtractExecute(l, r, result, add, noOverflowCheck, noOverflowCheck); } - template - void Decimal::addition(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result); - // with overflow check template<> void Decimal::addition(const execplan::IDB_Decimal& l, @@ -178,13 +266,10 @@ namespace datatypes std::plus add; AdditionOverflowCheck overflowCheck; MultiplicationOverflowCheck mulOverflowCheck; - execute(l, r, result, add, overflowCheck, mulOverflowCheck); + addSubtractExecute(l, r, result, add, overflowCheck, mulOverflowCheck); } - template - void Decimal::addition(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result); - + // no overflow check template<> void Decimal::addition(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) @@ -195,86 +280,252 @@ namespace datatypes return; } - int64_t lValue = 0, rValue = 0; + int64_t lValue = l.value, rValue = r.value; - if (result.scale >= l.scale) - lValue = l.value * mcs_pow_10[result.scale - l.scale]; - else - lValue = (int64_t)(l.value > 0 ? - (double)l.value / mcs_pow_10[l.scale - result.scale] + 0.5 : - (double)l.value / mcs_pow_10[l.scale - result.scale] - 0.5); + if (result.scale > l.scale) + lValue *= mcs_pow_10[result.scale - l.scale]; + else if (result.scale < l.scale) + lValue = (int64_t)(lValue > 0 ? + (double)lValue / mcs_pow_10[l.scale - result.scale] + 0.5 : + (double)lValue / mcs_pow_10[l.scale - result.scale] - 0.5); - if (result.scale >= r.scale) - rValue = r.value * mcs_pow_10[result.scale - r.scale]; - else - rValue = (int64_t)(r.value > 0 ? - (double)r.value / mcs_pow_10[r.scale - result.scale] + 0.5 : - (double)r.value / mcs_pow_10[r.scale - result.scale] - 0.5); + if (result.scale > r.scale) + rValue *= mcs_pow_10[result.scale - r.scale]; + else if (result.scale < r.scale) + rValue = (int64_t)(rValue > 0 ? + (double)rValue / mcs_pow_10[r.scale - result.scale] + 0.5 : + (double)rValue / mcs_pow_10[r.scale - result.scale] - 0.5); result.value = lValue + rValue; } - template - void Decimal::addition(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result); + // with overflow check template<> void Decimal::addition(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) { - throw logging::NotImplementedExcept("Decimal::addition"); + AdditionOverflowCheck additionOverflowCheck; + MultiplicationOverflowCheck mulOverflowCheck; + + if (result.scale == l.scale && result.scale == r.scale) + { + additionOverflowCheck(l.value, r.value); + result.value = l.value + r.value; + return; + } + + int64_t lValue = l.value, rValue = r.value; + + if (result.scale > l.scale) + mulOverflowCheck(lValue, mcs_pow_10[result.scale - l.scale], lValue); + else if (result.scale < l.scale) + lValue = (int64_t)(lValue > 0 ? + (double)lValue / mcs_pow_10[l.scale - result.scale] + 0.5 : + (double)lValue / mcs_pow_10[l.scale - result.scale] - 0.5); + + if (result.scale > r.scale) + mulOverflowCheck(rValue, mcs_pow_10[result.scale - r.scale], rValue); + else if (result.scale < r.scale) + rValue = (int64_t)(rValue > 0 ? + (double)rValue / mcs_pow_10[r.scale - result.scale] + 0.5 : + (double)rValue / mcs_pow_10[r.scale - result.scale] - 0.5); + + additionOverflowCheck(lValue, rValue); + result.value = lValue + rValue; } + // no overflow check + template<> + void Decimal::subtraction(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + { + std::minus subtract; + NoOverflowCheck noOverflowCheck; + addSubtractExecute(l, r, result, subtract, noOverflowCheck, noOverflowCheck); + } + + // with overflow check + template<> + void Decimal::subtraction(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + { + std::minus subtract; + SubtractionOverflowCheck overflowCheck; + MultiplicationOverflowCheck mulOverflowCheck; + addSubtractExecute(l, r, result, subtract, overflowCheck, mulOverflowCheck); + } + + // no overflow check + template<> + void Decimal::subtraction(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + { + if (result.scale == l.scale && result.scale == r.scale) + { + result.value = l.value - r.value; + return; + } + + int64_t lValue = l.value, rValue = r.value; + + if (result.scale > l.scale) + lValue *= mcs_pow_10[result.scale - l.scale]; + else if (result.scale < l.scale) + lValue = (int64_t)(lValue > 0 ? + (double)lValue / mcs_pow_10[l.scale - result.scale] + 0.5 : + (double)lValue / mcs_pow_10[l.scale - result.scale] - 0.5); + + if (result.scale > r.scale) + rValue *= mcs_pow_10[result.scale - r.scale]; + else if (result.scale < r.scale) + rValue = (int64_t)(rValue > 0 ? + (double)rValue / mcs_pow_10[r.scale - result.scale] + 0.5 : + (double)rValue / mcs_pow_10[r.scale - result.scale] - 0.5); + + result.value = lValue - rValue; + } + + // with overflow check + template<> + void Decimal::subtraction(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + { + SubtractionOverflowCheck subtractionOverflowCheck; + MultiplicationOverflowCheck mulOverflowCheck; + + if (result.scale == l.scale && result.scale == r.scale) + { + subtractionOverflowCheck(l.value, r.value); + result.value = l.value - r.value; + return; + } + + int64_t lValue = l.value, rValue = r.value; + + if (result.scale > l.scale) + mulOverflowCheck(lValue, mcs_pow_10[result.scale - l.scale], lValue); + else if (result.scale < l.scale) + lValue = (int64_t)(lValue > 0 ? + (double)lValue / mcs_pow_10[l.scale - result.scale] + 0.5 : + (double)lValue / mcs_pow_10[l.scale - result.scale] - 0.5); + + if (result.scale > r.scale) + mulOverflowCheck(rValue, mcs_pow_10[result.scale - r.scale], rValue); + else if (result.scale < r.scale) + rValue = (int64_t)(rValue > 0 ? + (double)rValue / mcs_pow_10[r.scale - result.scale] + 0.5 : + (double)rValue / mcs_pow_10[r.scale - result.scale] - 0.5); + + subtractionOverflowCheck(lValue, rValue); + result.value = lValue - rValue; + } + + // no overflow check template<> void Decimal::division(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) { - std::divides division; NoOverflowCheck noOverflowCheck; - execute(l, r, result, division, noOverflowCheck, noOverflowCheck); + divisionExecute(l, r, result, noOverflowCheck, noOverflowCheck); } - template - void Decimal::division(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result); - // With overflow check template<> void Decimal::division(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) { - std::divides division; DivisionOverflowCheck overflowCheck; MultiplicationOverflowCheck mulOverflowCheck; - execute(l, r, result, division, overflowCheck, mulOverflowCheck); + divisionExecute(l, r, result, overflowCheck, mulOverflowCheck); } + // no overflow check // We rely on the zero check from ArithmeticOperator::execute template<> void Decimal::division(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) { if (result.scale >= l.scale - r.scale) - result.value = (int64_t)(( (l.value > 0 && r.value > 0) - || (l.value < 0 && r.value < 0) ? - (long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)] + 0.5 : - (long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)] - 0.5)); - else - result.value = (int64_t)(( (l.value > 0 && r.value > 0) - || (l.value < 0 && r.value < 0) ? - (long double)l.value / r.value / mcs_pow_10[l.scale - r.scale - result.scale] + 0.5 : - (long double)l.value / r.value / mcs_pow_10[l.scale - r.scale - result.scale] - 0.5)); - + result.value = (int64_t)(( (l.value > 0 && r.value > 0) || (l.value < 0 && r.value < 0) ? + (long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)] + 0.5 : + (long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)] - 0.5)); + else + result.value = (int64_t)(( (l.value > 0 && r.value > 0) || (l.value < 0 && r.value < 0) ? + (long double)l.value / r.value / mcs_pow_10[l.scale - r.scale - result.scale] + 0.5 : + (long double)l.value / r.value / mcs_pow_10[l.scale - r.scale - result.scale] - 0.5)); } - template - void Decimal::division(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result); - + // With overflow check template<> void Decimal::division(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) { - throw logging::NotImplementedExcept("Decimal::division"); + DivisionOverflowCheck divisionOverflowCheck; + + divisionOverflowCheck(l.value, r.value); + + if (result.scale >= l.scale - r.scale) + // TODO How do we check overflow of (int64_t)((long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)]) ? + result.value = (int64_t)(( (l.value > 0 && r.value > 0) || (l.value < 0 && r.value < 0) ? + (long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)] + 0.5 : + (long double)l.value / r.value * mcs_pow_10[result.scale - (l.scale - r.scale)] - 0.5)); + else + result.value = (int64_t)(( (l.value > 0 && r.value > 0) || (l.value < 0 && r.value < 0) ? + (long double)l.value / r.value / mcs_pow_10[l.scale - r.scale - result.scale] + 0.5 : + (long double)l.value / r.value / mcs_pow_10[l.scale - r.scale - result.scale] - 0.5)); + } + + // no overflow check + template<> + void Decimal::multiplication(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + { + MultiplicationNoOverflowCheck noOverflowCheck; + multiplicationExecute(l, r, result, noOverflowCheck, noOverflowCheck); + } + + // With overflow check + template<> + void Decimal::multiplication(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + { + MultiplicationOverflowCheck mulOverflowCheck; + multiplicationExecute(l, r, result, mulOverflowCheck, mulOverflowCheck); + } + + // no overflow check + template<> + void Decimal::multiplication(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + { + if (result.scale >= l.scale + r.scale) + result.value = l.value * r.value * mcs_pow_10[result.scale - (l.scale + r.scale)]; + else + result.value = (int64_t)(( (l.value > 0 && r.value > 0) || (l.value < 0 && r.value < 0) ? + (double)l.value * r.value / mcs_pow_10[l.scale + r.scale - result.scale] + 0.5 : + (double)l.value * r.value / mcs_pow_10[l.scale + r.scale - result.scale] - 0.5)); + } + + // With overflow check + template<> + void Decimal::multiplication(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + { + MultiplicationOverflowCheck mulOverflowCheck; + + if (result.scale >= l.scale + r.scale) + { + mulOverflowCheck(l.value, r.value, result.value); + mulOverflowCheck(result.value, mcs_pow_10[result.scale - (l.scale + r.scale)], result.value); + } + else + { + mulOverflowCheck(l.value, r.value, result.value); + + result.value = (int64_t)(( (result.value > 0) ? + (double)result.value / mcs_pow_10[l.scale + r.scale - result.scale] + 0.5 : + (double)result.value / mcs_pow_10[l.scale + r.scale - result.scale] - 0.5)); + } } } // end of namespace diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index 93b827207..54071c03b 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -35,6 +35,7 @@ namespace datatypes constexpr uint32_t MAXDECIMALWIDTH = 16U; constexpr uint8_t INT64MAXPRECISION = 18U; constexpr uint8_t INT128MAXPRECISION = 38U; +constexpr uint8_t MAXLEGACYWIDTH = 8U; const uint64_t mcs_pow_10[20] = { @@ -126,6 +127,15 @@ class Decimal const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result); + /** + @brief Subtraction template that supports overflow check and + two internal representations of decimal. + */ + template + static void subtraction(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, + execplan::IDB_Decimal& result); + /** @brief Division template that supports overflow check and two internal representations of decimal. @@ -136,7 +146,16 @@ class Decimal execplan::IDB_Decimal& result); /** - @brief Convinience method to put decimal into a std:;string. + @brief Multiplication template that supports overflow check and + two internal representations of decimal. + */ + template + static void multiplication(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, + execplan::IDB_Decimal& result); + + /** + @brief Convenience method to put decimal into a std::string. */ static std::string toString(execplan::IDB_Decimal& value); @@ -161,6 +180,66 @@ class Decimal && precision <= INT128MAXPRECISION; } + /** + @brief The method sets the legacy scale and precision of a wide decimal + column which is the result of an arithmetic operation. + */ + static inline void setDecimalScalePrecisionLegacy(execplan::CalpontSystemCatalog::ColType& ct, + unsigned int precision, unsigned int scale) + { + ct.scale = scale; + + if (ct.scale == 0) + ct.precision = precision - 1; + else + ct.precision = precision - scale; + } + + /** + @brief The method sets the scale and precision of a wide decimal + column which is the result of an arithmetic operation. + */ + static inline void setDecimalScalePrecision(execplan::CalpontSystemCatalog::ColType& ct, + unsigned int precision, unsigned int scale) + { + ct.colWidth = (precision > INT64MAXPRECISION) + ? MAXDECIMALWIDTH : MAXLEGACYWIDTH; + + ct.precision = (precision > INT128MAXPRECISION) + ? INT128MAXPRECISION : precision; + + ct.scale = scale; + } + + /** + @brief The method sets the scale and precision of a wide decimal + column which is the result of an arithmetic operation, based on a heuristic. + */ + static inline void setDecimalScalePrecisionHeuristic(execplan::CalpontSystemCatalog::ColType& ct, + unsigned int precision, unsigned int scale) + { + unsigned int diff = 0; + + if (precision > INT128MAXPRECISION) + { + ct.precision = INT128MAXPRECISION; + diff = precision - INT128MAXPRECISION; + } + else + { + ct.precision = precision; + } + + ct.scale = scale; + + if (diff != 0) + { + ct.scale = scale - (int)(diff * (38.0/65.0)); + + if (ct.scale < 0) + ct.scale = 0; + } + } }; /** @@ -170,17 +249,25 @@ class Decimal struct DivisionOverflowCheck { void operator()(const int128_t& x, const int128_t& y) { - if (x == Decimal::maxInt128 && y == -1) + if (x == Decimal::minInt128 && y == -1) { throw logging::OperationOverflowExcept( "Decimal::division produces an overflow."); } } + void operator()(const int64_t x, const int64_t y) + { + if (x == std::numeric_limits::min() && y == -1) + { + throw logging::OperationOverflowExcept( + "Decimal::division produces an overflow."); + } + } }; /** @brief The structure contains an overflow check for int128 - addition. + and int64_t multiplication. */ struct MultiplicationOverflowCheck { void operator()(const int128_t& x, const int128_t& y) @@ -189,7 +276,7 @@ struct MultiplicationOverflowCheck { { throw logging::OperationOverflowExcept( "Decimal::multiplication or scale multiplication \ - produces an overflow."); +produces an overflow."); } } bool operator()(const int128_t& x, const int128_t& y, int128_t& r) @@ -198,16 +285,45 @@ struct MultiplicationOverflowCheck { { throw logging::OperationOverflowExcept( "Decimal::multiplication or scale multiplication \ - produces an overflow."); +produces an overflow."); } return true; } + void operator()(const int64_t x, const int64_t y) + { + if (x * y / y != x) + { + throw logging::OperationOverflowExcept( + "Decimal::multiplication or scale multiplication \ +produces an overflow."); + } + } + bool operator()(const int64_t x, const int64_t y, int64_t& r) + { + if ((r = x * y) / y != x) + { + throw logging::OperationOverflowExcept( + "Decimal::multiplication or scale multiplication \ +produces an overflow."); + } + return true; + } +}; +/** + @brief The strucuture runs an empty overflow check for int128 + multiplication operation. +*/ +struct MultiplicationNoOverflowCheck { + void operator()(const int128_t& x, const int128_t& y, int128_t& r) + { + r = x * y; + } }; /** @brief The structure contains an overflow check for int128 - addition. + and int64 addition. */ struct AdditionOverflowCheck { void operator()(const int128_t& x, const int128_t& y) @@ -219,6 +335,40 @@ struct AdditionOverflowCheck { "Decimal::addition produces an overflow."); } } + void operator()(const int64_t x, const int64_t y) + { + if ((y > 0 && x > std::numeric_limits::max() - y) + || (y < 0 && x < std::numeric_limits::min() - y)) + { + throw logging::OperationOverflowExcept( + "Decimal::addition produces an overflow."); + } + } +}; + +/** + @brief The structure contains an overflow check for int128 + subtraction. +*/ +struct SubtractionOverflowCheck { + void operator()(const int128_t& x, const int128_t& y) + { + if ((y > 0 && x < Decimal::minInt128 + y) + || (y < 0 && x > Decimal::maxInt128 + y)) + { + throw logging::OperationOverflowExcept( + "Decimal::subtraction produces an overflow."); + } + } + void operator()(const int64_t x, const int64_t y) + { + if ((y > 0 && x < std::numeric_limits::min() + y) + || (y < 0 && x > std::numeric_limits::max() + y)) + { + throw logging::OperationOverflowExcept( + "Decimal::subtraction produces an overflow."); + } + } }; /** diff --git a/dbcon/execplan/arithmeticoperator.cpp b/dbcon/execplan/arithmeticoperator.cpp index 610bf2ad7..3f4fa695c 100644 --- a/dbcon/execplan/arithmeticoperator.cpp +++ b/dbcon/execplan/arithmeticoperator.cpp @@ -35,19 +35,19 @@ namespace execplan * Constructors/Destructors */ ArithmeticOperator::ArithmeticOperator() : Operator(), - fDecimalOverflowCheck(true) + fDecimalOverflowCheck(false) { } ArithmeticOperator::ArithmeticOperator(const string& operatorName): Operator(operatorName), - fDecimalOverflowCheck(true) + fDecimalOverflowCheck(false) { } ArithmeticOperator::ArithmeticOperator(const ArithmeticOperator& rhs): Operator(rhs), fTimeZone(rhs.timeZone()), - fDecimalOverflowCheck(true) + fDecimalOverflowCheck(false) { } diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index 88c4c1f11..5286ab25c 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -196,7 +196,7 @@ public: return TreeNode::getBoolVal(); } void adjustResultType(const CalpontSystemCatalog::ColType& m); - constexpr inline bool getOverflowCheck() + constexpr inline bool getOverflowCheck() const { return fDecimalOverflowCheck; } @@ -209,7 +209,6 @@ private: template inline result_t execute(result_t op1, result_t op2, bool& isNull); inline void execute(IDB_Decimal& result, IDB_Decimal op1, IDB_Decimal op2, bool& isNull); - inline void execute(IDB_Decimal& result, IDB_Decimal op1, IDB_Decimal op2, bool& isNull, cscType& resultCscType); std::string fTimeZone; bool fDecimalOverflowCheck; }; @@ -250,7 +249,7 @@ inline void ArithmeticOperator::evaluate(rowgroup::Row& row, bool& isNull, Parse // WIP MCOL-641 case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: - execute(fResult.decimalVal, lop->getDecimalVal(row, isNull), rop->getDecimalVal(row, isNull), isNull, fOperationType); + execute(fResult.decimalVal, lop->getDecimalVal(row, isNull), rop->getDecimalVal(row, isNull), isNull); break; default: { @@ -292,20 +291,36 @@ inline result_t ArithmeticOperator::execute(result_t op1, result_t op2, bool& is } } -inline void ArithmeticOperator::execute(IDB_Decimal& result, IDB_Decimal op1, IDB_Decimal op2, bool& isNull, cscType& resultCscType) +inline void ArithmeticOperator::execute(IDB_Decimal& result, IDB_Decimal op1, IDB_Decimal op2, bool& isNull) { switch (fOp) { case OP_ADD: - if (resultCscType.colWidth == datatypes::MAXDECIMALWIDTH) + if (fOperationType.colWidth == datatypes::MAXDECIMALWIDTH) { - datatypes::Decimal::addition( - op1, op2, result); + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::addition( + op1, op2, result); + } + else + { + datatypes::Decimal::addition( + op1, op2, result); + } } - else if (resultCscType.colWidth == utils::MAXLEGACYWIDTH) + else if (fOperationType.colWidth == utils::MAXLEGACYWIDTH) { - datatypes::Decimal::addition( - op1, op2, result); + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::addition( + op1, op2, result); + } + else + { + datatypes::Decimal::addition( + op1, op2, result); + } } else { @@ -315,52 +330,95 @@ inline void ArithmeticOperator::execute(IDB_Decimal& result, IDB_Decimal op1, ID break; case OP_SUB: - if (result.scale == op1.scale && result.scale == op2.scale) + if (fOperationType.colWidth == datatypes::MAXDECIMALWIDTH) { - result.value = op1.value - op2.value; - break; + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::subtraction( + op1, op2, result); + } + else + { + datatypes::Decimal::subtraction( + op1, op2, result); + } + } + else if (fOperationType.colWidth == utils::MAXLEGACYWIDTH) + { + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::subtraction( + op1, op2, result); + } + else + { + datatypes::Decimal::subtraction( + op1, op2, result); + } } - - if (result.scale >= op1.scale) - op1.value *= IDB_pow[result.scale - op1.scale]; else - op1.value = (int64_t)(op1.value > 0 ? - (double)op1.value / IDB_pow[op1.scale - result.scale] + 0.5 : - (double)op1.value / IDB_pow[op1.scale - result.scale] - 0.5); - - if (result.scale >= op2.scale) - op2.value *= IDB_pow[result.scale - op2.scale]; - else - op2.value = (int64_t)(op2.value > 0 ? - (double)op2.value / IDB_pow[op2.scale - result.scale] + 0.5 : - (double)op2.value / IDB_pow[op2.scale - result.scale] - 0.5); - - result.value = op1.value - op2.value; + { + throw logging::InvalidArgumentExcept( + "Unexpected result width"); + } break; case OP_MUL: - if (result.scale >= op1.scale + op2.scale) - result.value = op1.value * op2.value * IDB_pow[result.scale - (op1.scale + op2.scale)]; + if (fOperationType.colWidth == datatypes::MAXDECIMALWIDTH) + { + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::multiplication( + op1, op2, result); + } + else + { + datatypes::Decimal::multiplication( + op1, op2, result); + } + } + else if (fOperationType.colWidth == utils::MAXLEGACYWIDTH) + { + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::multiplication( + op1, op2, result); + } + else + { + datatypes::Decimal::multiplication( + op1, op2, result); + } + } else - result.value = (int64_t)(( (op1.value > 0 && op2.value > 0) || (op1.value < 0 && op2.value < 0) ? - (double)op1.value * op2.value / IDB_pow[op1.scale + op2.scale - result.scale] + 0.5 : - (double)op1.value * op2.value / IDB_pow[op1.scale + op2.scale - result.scale] - 0.5)); - + { + throw logging::InvalidArgumentExcept( + "Unexpected result width"); + } break; case OP_DIV: - if (resultCscType.colWidth == 16) + if (fOperationType.colWidth == datatypes::MAXDECIMALWIDTH) { - if (op2.s128Value == 0) + if ((datatypes::Decimal::isWideDecimalType(op2.precision) && op2.s128Value == 0) + || (!datatypes::Decimal::isWideDecimalType(op2.precision) && op2.value == 0)) { isNull = true; break; } - datatypes::Decimal::division( - op1, op2, result); + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::division( + op1, op2, result); + } + else + { + datatypes::Decimal::division( + op1, op2, result); + } } - else if (resultCscType.colWidth == 8) + else if (fOperationType.colWidth == utils::MAXLEGACYWIDTH) { if (op2.value == 0) { @@ -368,8 +426,16 @@ inline void ArithmeticOperator::execute(IDB_Decimal& result, IDB_Decimal op1, ID break; } - datatypes::Decimal::division( - op1, op2, result); + if (LIKELY(!fDecimalOverflowCheck)) + { + datatypes::Decimal::division( + op1, op2, result); + } + else + { + datatypes::Decimal::division( + op1, op2, result); + } } else { diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 5a3260b65..20a2158bc 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -3105,17 +3105,13 @@ CalpontSystemCatalog::ColType colType_MysqlToIDB (const Item* item) case DECIMAL_RESULT: { Item_decimal* idp = (Item_decimal*)item; + ct.colDataType = CalpontSystemCatalog::DECIMAL; - ct.colWidth = (idp->max_length >= datatypes::INT64MAXPRECISION) - ? datatypes::MAXDECIMALWIDTH : utils::MAXLEGACYWIDTH; - ct.scale = idp->decimals; - // WIP MCOL-641 - if (ct.scale == 0) - ct.precision = (idp->max_length > datatypes::INT128MAXPRECISION) - ? datatypes::INT128MAXPRECISION : idp->max_length - 1; - else - ct.precision = (idp->max_length > datatypes::INT128MAXPRECISION ) - ? datatypes::INT128MAXPRECISION : idp->max_length - idp->decimals; + + unsigned int precision = idp->decimal_precision(); + unsigned int scale = idp->decimal_scale(); + + datatypes::Decimal::setDecimalScalePrecision(ct, precision, scale); break; } @@ -3141,7 +3137,7 @@ ReturnedColumn* buildReturnedColumn( { ReturnedColumn* rc = NULL; - if ( gwi.thd) + if (gwi.thd) { //if ( ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE ) || ((gwi.thd->lex)->sql_command == SQLCOM_UPDATE_MULTI )) { @@ -3444,6 +3440,7 @@ ArithmeticColumn* buildArithmeticColumn( Item** sfitempp = item->arguments(); ArithmeticOperator* aop = new ArithmeticOperator(item->func_name()); aop->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); + aop->setOverflowCheck(get_decimal_overflow_check(gwi.thd)); ParseTree* pt = new ParseTree(aop); //ReturnedColumn *lhs = 0, *rhs = 0; ParseTree* lhs = 0, *rhs = 0; @@ -3606,6 +3603,60 @@ ArithmeticColumn* buildArithmeticColumn( // decimal arithmetic operation gives double result when the session variable is set. CalpontSystemCatalog::ColType mysql_type = colType_MysqlToIDB(item); + if (mysql_type.colDataType == CalpontSystemCatalog::DECIMAL || + mysql_type.colDataType == CalpontSystemCatalog::UDECIMAL) + { + int32_t leftColWidth = pt->left()->data()->resultType().colWidth; + int32_t rightColWidth = pt->right()->data()->resultType().colWidth; + + // Revert back to legacy values of scale and precision + // if the 2 columns involved in the expression are not wide + if (leftColWidth <= utils::MAXLEGACYWIDTH && + rightColWidth <= utils::MAXLEGACYWIDTH) + { + Item_decimal* idp = (Item_decimal*)item; + + mysql_type.colWidth = 8; + + unsigned int precision = idp->max_length; + unsigned int scale = idp->decimals; + + datatypes::Decimal::setDecimalScalePrecisionLegacy(mysql_type, precision, scale); + } + else + { + + if (leftColWidth == datatypes::MAXDECIMALWIDTH || + rightColWidth == datatypes::MAXDECIMALWIDTH) + mysql_type.colWidth = datatypes::MAXDECIMALWIDTH; + + if (mysql_type.colWidth == datatypes::MAXDECIMALWIDTH) + { + string funcName = item->func_name(); + + int32_t scale1 = pt->left()->data()->resultType().scale; + int32_t scale2 = pt->right()->data()->resultType().scale; + + if (funcName == "/" && + (mysql_type.scale - (scale1 - scale2)) > datatypes::INT128MAXPRECISION) + { + Item_decimal* idp = (Item_decimal*)item; + + unsigned int precision = idp->decimal_precision(); + unsigned int scale = idp->decimal_scale(); + + datatypes::Decimal::setDecimalScalePrecisionHeuristic(mysql_type, precision, scale); + + if (mysql_type.scale < scale1) + mysql_type.scale = scale1; + + if (mysql_type.precision < mysql_type.scale) + mysql_type.precision = mysql_type.scale; + } + } + } + } + if (get_double_for_decimal_math(current_thd) == true) aop->adjustResultType(mysql_type); else diff --git a/dbcon/mysql/ha_mcs_sysvars.cpp b/dbcon/mysql/ha_mcs_sysvars.cpp index 02f89ab50..fc037005b 100644 --- a/dbcon/mysql/ha_mcs_sysvars.cpp +++ b/dbcon/mysql/ha_mcs_sysvars.cpp @@ -159,6 +159,15 @@ static MYSQL_THDVAR_BOOL( 0 ); +static MYSQL_THDVAR_BOOL( + decimal_overflow_check, + PLUGIN_VAR_NOCMDARG, + "Enable/disable for ColumnStore to check for overflow in arithmetic operation.", + NULL, + NULL, + 0 +); + static MYSQL_THDVAR_BOOL( ordered_only, PLUGIN_VAR_NOCMDARG, @@ -353,6 +362,7 @@ st_mysql_sys_var* mcs_system_variables[] = MYSQL_SYSVAR(diskjoin_bucketsize), MYSQL_SYSVAR(um_mem_limit), MYSQL_SYSVAR(double_for_decimal_math), + MYSQL_SYSVAR(decimal_overflow_check), MYSQL_SYSVAR(local_query), MYSQL_SYSVAR(use_import_for_batchinsert), MYSQL_SYSVAR(import_for_batchinsert_delimiter), @@ -557,6 +567,15 @@ void set_double_for_decimal_math(THD* thd, bool value) THDVAR(thd, double_for_decimal_math) = value; } +bool get_decimal_overflow_check(THD* thd) +{ + return ( thd == NULL ) ? false : THDVAR(thd, decimal_overflow_check); +} +void set_decimal_overflow_check(THD* thd, bool value) +{ + THDVAR(thd, decimal_overflow_check) = value; +} + ulong get_local_query(THD* thd) { return ( thd == NULL ) ? 0 : THDVAR(thd, local_query); diff --git a/dbcon/mysql/ha_mcs_sysvars.h b/dbcon/mysql/ha_mcs_sysvars.h index 1bb01307f..b8e3884b7 100644 --- a/dbcon/mysql/ha_mcs_sysvars.h +++ b/dbcon/mysql/ha_mcs_sysvars.h @@ -101,6 +101,9 @@ void set_varbin_always_hex(THD* thd, bool value); bool get_double_for_decimal_math(THD* thd); void set_double_for_decimal_math(THD* thd, bool value); +bool get_decimal_overflow_check(THD* thd); +void set_decimal_overflow_check(THD* thd, bool value); + ulong get_local_query(THD* thd); void set_local_query(THD* thd, ulong value); diff --git a/tests/mcs_decimal-tests.cpp b/tests/mcs_decimal-tests.cpp index b3a041867..53a40c533 100644 --- a/tests/mcs_decimal-tests.cpp +++ b/tests/mcs_decimal-tests.cpp @@ -15,11 +15,15 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ +#include + #include "gtest/gtest.h" #include "treenode.h" #include "mcs_decimal.h" #include "widedecimalutils.h" +#include "dataconvert.h" +#include "calpontsystemcatalog.h" TEST(Decimal, compareCheck) { @@ -82,6 +86,9 @@ TEST(Decimal, compareCheck) TEST(Decimal, additionNoOverflowCheck) { + execplan::CalpontSystemCatalog::ColType ct; + ct.colDataType = execplan::CalpontSystemCatalog::DECIMAL; + char buf[42]; // Addition w/o overflow check execplan::IDB_Decimal l, r, result; // same precision, same scale, both positive values @@ -98,61 +105,33 @@ TEST(Decimal, additionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); EXPECT_EQ(462, result.s128Value); + // same precision, same scale, both negative values - l.scale = 38; - l.precision = 38; l.s128Value = -42; - - r.scale = 38; - r.precision = 38; r.s128Value = -420; - - result.scale = 38; - result.precision = 38; result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); EXPECT_EQ(-462, result.s128Value); + // same precision, same scale, +- values - l.scale = 38; - l.precision = 38; l.s128Value = 42; - - r.scale = 38; - r.precision = 38; r.s128Value = -420; - - result.scale = 38; - result.precision = 38; result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); EXPECT_EQ(-378, result.s128Value); - // same precision, same scale, both 0 - l.scale = 38; - l.precision = 38; - l.s128Value = 0; - r.scale = 38; - r.precision = 38; + // same precision, same scale, both 0 + l.s128Value = 0; r.s128Value = 0; - - result.scale = 38; - result.precision = 38; result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); EXPECT_EQ(0, result.s128Value); - // diff scale + + // different scale // same precision, L scale > R scale, both positive values l.scale = 38; l.precision = 38; @@ -167,65 +146,35 @@ TEST(Decimal, additionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); - int128_t s128ScaleMultiplier1 = - static_cast(10000000000000)*10000000000; - int128_t s128Result = r.s128Value*s128ScaleMultiplier1+l.s128Value; - EXPECT_EQ(s128Result, result.s128Value); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("0.00000000000042000000000000000000000042", std::string(buf)); + // same precision, L scale > R scale, both negative values - l.scale = 38; - l.precision = 38; l.s128Value = -42; - - r.scale = 15; - r.precision = 38; r.s128Value = -420; - - result.scale = 38; - result.precision = 38; result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); - s128Result = r.s128Value*s128ScaleMultiplier1+l.s128Value; - EXPECT_EQ(s128Result, result.s128Value); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("-0.00000000000042000000000000000000000042", std::string(buf)); + // same precision, L scale > R scale, +- values - l.scale = 38; - l.precision = 38; l.s128Value = 42; - - r.scale = 15; - r.precision = 38; r.s128Value = -420; - - result.scale = 38; - result.precision = 38; result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); - s128Result = r.s128Value*s128ScaleMultiplier1+l.s128Value; - EXPECT_EQ(s128Result, result.s128Value); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("-0.00000000000041999999999999999999999958", std::string(buf)); + // same precision, L scale > R scale, both 0 - l.scale = 38; - l.precision = 38; l.s128Value = 0; - - r.scale = 15; - r.precision = 38; r.s128Value = 0; - - result.scale = 38; - result.precision = 38; result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); EXPECT_EQ(0, result.s128Value); + // same precision, L scale < R scale, both positive values l.scale = 15; l.precision = 38; @@ -240,72 +189,45 @@ TEST(Decimal, additionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); - s128Result = l.s128Value*s128ScaleMultiplier1+r.s128Value; - EXPECT_EQ(s128Result, result.s128Value); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("0.00000000000004200000000000000000000420", std::string(buf)); + // same precision, L scale < R scale, both negative values - l.scale = 15; - l.precision = 38; l.s128Value = -42; - - r.scale = 38; - r.precision = 38; r.s128Value = -420; - - result.scale = 38; - result.precision = 38; result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); - s128Result = l.s128Value*s128ScaleMultiplier1+r.s128Value; - EXPECT_EQ(s128Result, result.s128Value); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("-0.00000000000004200000000000000000000420", std::string(buf)); + // same precision, L scale < R scale, +- values - l.scale = 15; - l.precision = 38; l.s128Value = 42; - - r.scale = 38; - r.precision = 38; r.s128Value = -420; - - result.scale = 38; - result.precision = 38; result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); - s128Result = l.s128Value*s128ScaleMultiplier1+r.s128Value; - EXPECT_EQ(s128Result, result.s128Value); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("0.00000000000004199999999999999999999580", std::string(buf)); + // same precision, L scale < R scale, both 0 - l.scale = 15; - l.precision = 38; l.s128Value = 0; - - r.scale = 38; - r.precision = 38; r.s128Value = 0; - - result.scale = 38; - result.precision = 38; result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); - s128Result = l.s128Value*s128ScaleMultiplier1+r.s128Value; - EXPECT_EQ(s128Result, result.s128Value); + EXPECT_EQ(0, result.s128Value); } TEST(Decimal, divisionNoOverflowCheck) { // DIVISION // same precision, same scale, both positive values - std::string decimalStr; + execplan::CalpontSystemCatalog::ColType ct; + ct.colDataType = execplan::CalpontSystemCatalog::DECIMAL; + char buf[42]; execplan::IDB_Decimal l, r, result; + l.scale = 38; l.precision = 38; l.s128Value = 43; @@ -314,244 +236,104 @@ TEST(Decimal, divisionNoOverflowCheck) r.precision = 38; r.s128Value = 420; - result.scale = r.scale; + result.scale = 10; result.precision = 38; result.s128Value = 0; datatypes::Decimal::division(r, l, result); - - EXPECT_EQ(r.scale, result.scale); - EXPECT_EQ(result.precision, result.precision); - EXPECT_EQ(r.s128Value/l.s128Value, result.s128Value); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("9.7674418605", std::string(buf)); + // same precision, same scale, both negative values - l.scale = 38; - l.precision = 38; - l.s128Value = -42; - - r.scale = 38; - r.precision = 38; + l.s128Value = -43; r.s128Value = -420; - - result.scale = r.scale; - result.precision = r.precision; result.s128Value = 0; datatypes::Decimal::division(r, l, result); - EXPECT_EQ(r.scale, result.scale); - EXPECT_EQ(r.precision, result.precision); - EXPECT_EQ(r.s128Value/l.s128Value, result.s128Value); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("9.7674418605", std::string(buf)); + // same precision, same scale, +- values - l.scale = 38; - l.precision = 38; - l.s128Value = 42; - - r.scale = 38; - r.precision = 38; - r.s128Value = -420; - - result.scale = 38; - result.precision = 38; + l.s128Value = 2200000; + r.s128Value = -1900; result.s128Value = 0; datatypes::Decimal::division(l, r, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); - EXPECT_EQ(l.s128Value/r.s128Value, result.s128Value); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("-1157.8947368421", std::string(buf)); + // same precision, same scale, l = 0 - l.scale = 38; - l.precision = 38; l.s128Value = 0; - - r.scale = 38; - r.precision = 38; r.s128Value = 42424242; - - result.scale = 38; - result.precision = 38; result.s128Value = 0; datatypes::Decimal::division(l, r, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); EXPECT_EQ(0, result.s128Value); + // diff scale // same precision, L scale > R scale, both positive values l.scale = 38; - l.precision = 38; - l.s128Value = 42; + l.s128Value = 19; r.scale = 15; - r.precision = 38; - r.s128Value = 420; + r.s128Value = 22; - result.scale = 38; - result.precision = 38; + result.scale = 10; result.s128Value = 0; datatypes::Decimal::division(r, l, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); - int128_t s128ScaleMultiplier1 = - static_cast(10000000000000)*10000000000; - int128_t s128Result = r.s128Value*s128ScaleMultiplier1/l.s128Value; - EXPECT_EQ(s128Result, result.s128Value); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("115789473684210526315789.4736842105", std::string(buf)); + // same precision, L scale > R scale, both negative values - l.scale = 38; - l.precision = 38; - l.s128Value = -42; - - r.scale = 15; - r.precision = 38; - r.s128Value = -420; - - result.scale = 38; - result.precision = 38; + l.s128Value = -22; + r.s128Value = -19; result.s128Value = 0; datatypes::Decimal::division(r, l, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); - s128Result = r.s128Value*s128ScaleMultiplier1/l.s128Value; - EXPECT_EQ(s128Result, result.s128Value); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("86363636363636363636363.6363636364", std::string(buf)); + // same precision, L scale > R scale, +- values - l.scale = 38; - l.precision = 38; - l.s128Value = 42; - - r.scale = 15; - r.precision = 38; - r.s128Value = -420; - - result.scale = 38; - result.precision = 38; + l.s128Value = 19; + r.s128Value = -22; result.s128Value = 0; datatypes::Decimal::division(r, l, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); - s128Result = r.s128Value*s128ScaleMultiplier1/l.s128Value; - EXPECT_EQ(s128Result, result.s128Value); - // same precision, L scale > R scale, L = 0 - l.scale = 38; - l.precision = 38; - l.s128Value = 0; + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("-115789473684210526315789.4736842105", std::string(buf)); - r.scale = 15; - r.precision = 38; - r.s128Value = 424242; - - result.scale = 38; - result.precision = 38; + // same precision, L scale > R scale, R = 0 + l.s128Value = 424242; + r.s128Value = 0; result.s128Value = 0; - datatypes::Decimal::division(l, r, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); + datatypes::Decimal::division(r, l, result); EXPECT_EQ(0, result.s128Value); + // same precision, L scale > R scale, both MAX positive values - // WIP Investigate the next two - l.scale = 38; - l.precision = 38; - l.s128Value = 0; utils::int128Max(l.s128Value); - - r.scale = 15; - r.precision = 38; - r.s128Value = 0; utils::int128Max(r.s128Value); - - result.scale = 38; - result.precision = 38; - result.s128Value = 0; - - datatypes::Decimal::division(l, r, result); - // Use as an examplar + utils::int128Max(l.s128Value); utils::int128Max(r.s128Value); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); - s128Result = r.s128Value*s128ScaleMultiplier1/l.s128Value; - // WIP - //EXPECT_EQ(s128Result, result.s128Value); - // same precision, L scale > R scale, both MIN negative values - l.scale = 38; - l.precision = 38; - l.s128Value = 0; utils::int128Min(l.s128Value); - - r.scale = 15; - r.precision = 38; - r.s128Value = 0; utils::int128Min(l.s128Value); - - result.scale = 38; - result.precision = 38; result.s128Value = 0; - //datatypes::Decimal::division(l, r, result); - // Use as an examplar + datatypes::Decimal::division(r, l, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("100000000000000000000000.0000000000", std::string(buf)); + + // same precision, L scale > R scale, both MIN negative values + utils::int128Min(l.s128Value); utils::int128Min(r.s128Value); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); - s128Result = r.s128Value*s128ScaleMultiplier1/l.s128Value; - //EXPECT_EQ(s128Result, result.s128Value); - // WIP - //EXPECT_EQ(r.s128Value, result.s128Value); + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("100000000000000000000000.0000000000", std::string(buf)); // same precision, L scale < R scale, both positive values l.scale = 15; l.precision = 38; l.s128Value = 42; - r.scale = 38; - r.precision = 38; - r.s128Value = 420; - - result.scale = 38; - result.precision = 38; - result.s128Value = 0; - - datatypes::Decimal::division(l, r, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); - s128Result = l.s128Value*s128ScaleMultiplier1/r.s128Value; - EXPECT_EQ(s128Result, result.s128Value); - // same precision, L scale < R scale, both negative values - l.scale = 15; - l.precision = 38; - l.s128Value = -42; - - r.scale = 38; - r.precision = 38; - r.s128Value = -420; - - result.scale = 38; - result.precision = 38; - result.s128Value = 0; - - datatypes::Decimal::division(l, r, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); - s128Result = l.s128Value*s128ScaleMultiplier1/r.s128Value; - EXPECT_EQ(s128Result, result.s128Value); - // same precision, L scale < R scale, +- values - l.scale = 15; - l.precision = 38; - l.s128Value = 42; - - r.scale = 38; - r.precision = 38; - r.s128Value = -420; - - result.scale = 38; - result.precision = 38; - result.s128Value = 0; - - datatypes::Decimal::division(l, r, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); - s128Result = l.s128Value*s128ScaleMultiplier1/r.s128Value; - EXPECT_EQ(s128Result, result.s128Value); - // same precision, L scale < R scale, L = 0 - l.scale = 15; - l.precision = 38; - l.s128Value = 0; - r.scale = 38; r.precision = 38; r.s128Value = 42; @@ -560,73 +342,140 @@ TEST(Decimal, divisionNoOverflowCheck) result.precision = 38; result.s128Value = 0; - datatypes::Decimal::division(l, r, result); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); - s128Result = l.s128Value*s128ScaleMultiplier1/r.s128Value; - EXPECT_EQ(s128Result, result.s128Value); + datatypes::Decimal::division(r, l, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("0.00000000000000000000001000000000000000", std::string(buf)); + + // same precision, L scale < R scale, both negative values + l.s128Value = -22; + r.s128Value = -19; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("0.00000000000000000000000863636363636364", std::string(buf)); + + // same precision, L scale < R scale, +- values + l.s128Value = 22; + r.s128Value = -19; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("-0.00000000000000000000000863636363636364", std::string(buf)); + + // same precision, L scale < R scale, R = 0 + l.s128Value = 42; + r.s128Value = 0; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ(0, result.s128Value); + // same precision, L scale < R scale, both MAX positive values // WIP Investigate the next two - l.scale = 15; - l.precision = 38; - l.s128Value = 0; utils::int128Max(l.s128Value); - - r.scale = 38; - r.precision = 38; - r.s128Value = 0; utils::int128Max(r.s128Value); - - result.scale = 38; - result.precision = 38; - result.s128Value = 0; - - datatypes::Decimal::division(l, r, result); - // Use as an examplar + utils::int128Max(l.s128Value); utils::int128Max(r.s128Value); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); - s128Result = l.s128Value*s128ScaleMultiplier1/r.s128Value; - // WIP - //EXPECT_EQ(s128Result, result.s128Value); - //EXPECT_EQ(r.s128Value, result.s128Value); + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("0.00000000000000000000001000000000000000", std::string(buf)); + // same precision, L scale < R scale, both MIN negative values - l.scale = 15; + utils::int128Min(l.s128Value); + utils::int128Min(r.s128Value); + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("0.00000000000000000000001000000000000000", std::string(buf)); + + // same precision, L scale < R scale, result.scale < (r.scale-l.scale) + // both positive values + l.scale = 37; l.precision = 38; - l.s128Value = 0; utils::int128Min(l.s128Value); + l.s128Value = 43; r.scale = 38; r.precision = 38; - r.s128Value = 0; utils::int128Min(l.s128Value); + r.s128Value = 420; - result.scale = 38; + result.scale = 0; result.precision = 38; result.s128Value = 0; - //datatypes::Decimal::division(l, r, result); - // Use as an examplar + datatypes::Decimal::division(r, l, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("1", std::string(buf)); + + // same precision, L scale < R scale, result.scale < (r.scale-l.scale) + // both negative values + l.s128Value = -22; + r.s128Value = -1900; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("9", std::string(buf)); + + // same precision, L scale < R scale, result.scale < (r.scale-l.scale) + // +- values + l.s128Value = 22; + r.s128Value = -1900; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("-9", std::string(buf)); + + // same precision, L scale < R scale, result.scale < (r.scale-l.scale) + // R = 0 + l.s128Value = 42; + r.s128Value = 0; + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + EXPECT_EQ(0, result.s128Value); + + // same precision, L scale < R scale, result.scale < (r.scale-l.scale) + // both MAX positive values + utils::int128Max(l.s128Value); + utils::int128Max(r.s128Value); + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("0", std::string(buf)); + + // same precision, L scale < R scale, result.scale < (r.scale-l.scale) + // both MIN negative values + utils::int128Min(l.s128Value); utils::int128Min(r.s128Value); - EXPECT_EQ(38, result.scale); - EXPECT_EQ(38, result.precision); - s128Result = l.s128Value*s128ScaleMultiplier1/r.s128Value; - // WIP - // EXPECT_EQ(s128Result, result.s128Value); - //EXPECT_EQ(r.s128Value, result.s128Value); + result.s128Value = 0; + + datatypes::Decimal::division(r, l, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("0", std::string(buf)); } -void doDiv(execplan::IDB_Decimal& l, - execplan::IDB_Decimal& r, - execplan::IDB_Decimal& result) +void doDiv(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, + execplan::IDB_Decimal& result) { datatypes::Decimal::division(l, r, result); } TEST(Decimal, divisionWithOverflowCheck) { - - // Divide max int128 by -1 + execplan::CalpontSystemCatalog::ColType ct; + ct.colDataType = execplan::CalpontSystemCatalog::DECIMAL; + char buf[42]; + // Divide min int128 by -1 execplan::IDB_Decimal l, r, result; l.scale = 0; l.precision = 38; - l.s128Value = datatypes::Decimal::maxInt128; + l.s128Value = datatypes::Decimal::minInt128; r.scale = 0; r.precision = 38; @@ -636,71 +485,63 @@ TEST(Decimal, divisionWithOverflowCheck) result.precision = 38; result.s128Value = 42; EXPECT_THROW(doDiv(l, r, result), logging::OperationOverflowExcept); - // Divide two ints one of which overflows after the scaling. - l.scale = 0; - l.precision = 38; - l.s128Value = datatypes::Decimal::maxInt128; + // Divide two ints one of which overflows after the scaling. + // TODO We currently do not test overflow due to scaling in + // case of division. Re-enable this test when we check for overflow + /*l.s128Value = datatypes::Decimal::maxInt128; r.scale = 1; - r.precision = 38; r.s128Value = 42; result.scale = 1; - result.precision = 38; result.s128Value = 42; - EXPECT_THROW(doDiv(l, r, result), logging::OperationOverflowExcept); + EXPECT_THROW(doDiv(l, r, result), logging::OperationOverflowExcept);*/ + // Normal execution w/o overflow l.scale = 0; - l.precision = 38; - l.s128Value = datatypes::Decimal::maxInt128-1; + l.s128Value = datatypes::Decimal::maxInt128 - 1; r.scale = 0; - r.precision = 38; r.s128Value = 0xFFFFFFFFFFFFFFFF; result.scale = 0; - result.precision = 38; result.s128Value = 0; EXPECT_NO_THROW(doDiv(l, r, result)); - - l.s128Value /= r.s128Value; - - EXPECT_EQ(0, result.scale); - EXPECT_EQ(38, result.precision); - EXPECT_EQ(l.s128Value, result.s128Value); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("9223372036854775809", std::string(buf)); } -void doAdd(execplan::IDB_Decimal& l, - execplan::IDB_Decimal& r, - execplan::IDB_Decimal& result) +void doAdd(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, + execplan::IDB_Decimal& result) { datatypes::Decimal::addition(l, r, result); } TEST(Decimal, additionWithOverflowCheck) { + execplan::CalpontSystemCatalog::ColType ct; + ct.colDataType = execplan::CalpontSystemCatalog::DECIMAL; + char buf[42]; // Add two max ints execplan::IDB_Decimal l, r, result; l.scale = 0; l.precision = 38; - l.s128Value = datatypes::Decimal::maxInt128-1; + l.s128Value = datatypes::Decimal::maxInt128 - 1; r.scale = 0; r.precision = 38; - r.s128Value = datatypes::Decimal::maxInt128-1; + r.s128Value = datatypes::Decimal::maxInt128 - 1; result.scale = 0; result.precision = 38; result.s128Value = 42; EXPECT_THROW(doAdd(l, r, result), logging::OperationOverflowExcept); - // Add two ints one of which overflows after the scaling. - l.scale = 0; - l.precision = 38; - l.s128Value = datatypes::Decimal::maxInt128-1; + // Add two ints one of which overflows after the scaling. + l.s128Value = datatypes::Decimal::maxInt128 - 1; r.scale = 1; - r.precision = 38; r.s128Value = 0xFFFFFFFFFFFFFFFF; result.scale = 1; @@ -708,36 +549,407 @@ TEST(Decimal, additionWithOverflowCheck) result.s128Value = 0; EXPECT_THROW(doAdd(l, r, result), logging::OperationOverflowExcept); + // Normal execution w/o overflow l.scale = 0; - l.precision = 38; - l.s128Value = datatypes::Decimal::maxInt128-1; + l.s128Value = datatypes::Decimal::minInt128; r.scale = 0; - r.precision = 38; r.s128Value = 0xFFFFFFFFFFFFFFFF; - + result.scale = 0; + result.s128Value = 0; + + EXPECT_NO_THROW(doAdd(l, r, result)); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("-170141183460469231713240559642174554113", std::string(buf)); +} + +TEST(Decimal, subtractionNoOverflowCheck) +{ + execplan::CalpontSystemCatalog::ColType ct; + ct.colDataType = execplan::CalpontSystemCatalog::DECIMAL; + char buf[42]; + // Subtractio w/o overflow check + execplan::IDB_Decimal l, r, result; + // same precision, same scale, both positive values + l.scale = 38; + l.precision = 38; + l.s128Value = 42; + + r.scale = 38; + r.precision = 38; + r.s128Value = 420; + + result.scale = 38; result.precision = 38; result.s128Value = 0; - EXPECT_NO_THROW(doDiv(l, r, result)); + datatypes::Decimal::subtraction(l, r, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("-0.00000000000000000000000000000000000378", std::string(buf)); - l.s128Value /= r.s128Value; + // same precision, same scale, both negative values + l.s128Value = -42; + r.s128Value = -420; + result.s128Value = 0; - EXPECT_EQ(0, result.scale); - EXPECT_EQ(38, result.precision); - EXPECT_EQ(l.s128Value, result.s128Value); + datatypes::Decimal::subtraction(l, r, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("0.00000000000000000000000000000000000378", std::string(buf)); + + // same precision, same scale, +- values + l.s128Value = 42; + r.s128Value = -420; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("0.00000000000000000000000000000000000462", std::string(buf)); + + // same precision, same scale, both 0 + l.s128Value = 0; + r.s128Value = 0; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ(0, result.s128Value); + + // different scale + // same precision, L scale > R scale, both positive values + l.scale = 38; + l.precision = 38; + l.s128Value = 42; + + r.scale = 15; + r.precision = 38; + r.s128Value = 420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("-0.00000000000041999999999999999999999958", std::string(buf)); + + // same precision, L scale > R scale, both negative values + l.s128Value = -42; + r.s128Value = -420; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("0.00000000000041999999999999999999999958", std::string(buf)); + + // same precision, L scale > R scale, +- values + l.s128Value = 42; + r.s128Value = -420; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("0.00000000000042000000000000000000000042", std::string(buf)); + + // same precision, L scale > R scale, both 0 + l.s128Value = 0; + r.s128Value = 0; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ(0, result.s128Value); + + // same precision, L scale < R scale, both positive values + l.scale = 15; + l.precision = 38; + l.s128Value = 42; + + r.scale = 38; + r.precision = 38; + r.s128Value = 420; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("0.00000000000004199999999999999999999580", std::string(buf)); + + // same precision, L scale < R scale, both negative values + l.s128Value = -42; + r.s128Value = -420; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("-0.00000000000004199999999999999999999580", std::string(buf)); + + // same precision, L scale < R scale, +- values + l.s128Value = 42; + r.s128Value = -420; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("0.00000000000004200000000000000000000420", std::string(buf)); + + // same precision, L scale < R scale, both 0 + l.s128Value = 0; + r.s128Value = 0; + result.s128Value = 0; + + datatypes::Decimal::subtraction(l, r, result); + EXPECT_EQ(0, result.s128Value); +} + +void doSubtract(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, + execplan::IDB_Decimal& result) +{ + datatypes::Decimal::subtraction(l, r, result); +} + +TEST(Decimal, subtractionWithOverflowCheck) +{ + execplan::CalpontSystemCatalog::ColType ct; + ct.colDataType = execplan::CalpontSystemCatalog::DECIMAL; + char buf[42]; + // Subtract a max int from a min int + execplan::IDB_Decimal l, r, result; + l.scale = 0; + l.precision = 38; + l.s128Value = datatypes::Decimal::minInt128 + 1; + + r.scale = 0; + r.precision = 38; + r.s128Value = datatypes::Decimal::maxInt128 - 1; + + result.scale = 0; + result.precision = 38; + result.s128Value = 42; + EXPECT_THROW(doSubtract(l, r, result), logging::OperationOverflowExcept); + + // Subtract two ints one of which overflows after the scaling. + l.s128Value = datatypes::Decimal::minInt128 + 1; + r.scale = 1; + r.s128Value = 0xFFFFFFFFFFFFFFFF; + + result.scale = 1; + result.precision = 38; + result.s128Value = 0; + + EXPECT_THROW(doSubtract(l, r, result), logging::OperationOverflowExcept); + + // Normal execution w/o overflow + l.scale = 0; + l.s128Value = datatypes::Decimal::maxInt128; + + r.scale = 0; + r.s128Value = 0xFFFFFFFFFFFFFFFF; + + result.scale = 0; + result.s128Value = 0; + + EXPECT_NO_THROW(doSubtract(l, r, result)); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("170141183460469231713240559642174554112", std::string(buf)); +} + +TEST(Decimal, multiplicationNoOverflowCheck) +{ + // Multiplication + // same precision, l.scale + r.scale = result.scale, both positive values + execplan::CalpontSystemCatalog::ColType ct; + ct.colDataType = execplan::CalpontSystemCatalog::DECIMAL; + char buf[42]; + execplan::IDB_Decimal l, r, result; + + l.scale = 19; + l.precision = 38; + l.s128Value = 4611686018427387904; + + r.scale = 19; + r.precision = 38; + r.s128Value = UINT64_MAX; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::multiplication(r, l, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("0.85070591730234615861231965839514664960", std::string(buf)); + + // same precision, l.scale + r.scale = result.scale, both negative values + l.s128Value = -4611686018427387904; + r.s128Value = UINT64_MAX; + r.s128Value = -r.s128Value; + result.s128Value = 0; + + datatypes::Decimal::multiplication(r, l, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("0.85070591730234615861231965839514664960", std::string(buf)); + + // same precision, l.scale + r.scale = result.scale, +- values + l.s128Value = -4611686018427387904; + r.s128Value = UINT64_MAX; + result.s128Value = 0; + + datatypes::Decimal::multiplication(l, r, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("-0.85070591730234615861231965839514664960", std::string(buf)); + + // same precision, l.scale + r.scale = result.scale, l = 0 + l.s128Value = 0; + r.s128Value = UINT64_MAX; + result.s128Value = 0; + + datatypes::Decimal::multiplication(l, r, result); + EXPECT_EQ(0, result.s128Value); + + // same precision, l.scale + r.scale < result.scale, both positive values + l.scale = 18; + l.precision = 38; + l.s128Value = 72057594037927936; + + r.scale = 18; + r.precision = 38; + r.s128Value = 9223372036854775808ULL; + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::multiplication(r, l, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("0.66461399789245793645190353014017228800", std::string(buf)); + + // same precision, l.scale + r.scale < result.scale, both negative values + l.s128Value = -72057594037927936; + r.s128Value = 9223372036854775808ULL; + r.s128Value = -r.s128Value; + result.s128Value = 0; + + datatypes::Decimal::multiplication(r, l, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("0.66461399789245793645190353014017228800", std::string(buf)); + + // same precision, l.scale + r.scale < result.scale, +- values + l.s128Value = -72057594037927936; + r.s128Value = 9223372036854775808ULL; + result.s128Value = 0; + + datatypes::Decimal::multiplication(l, r, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("-0.66461399789245793645190353014017228800", std::string(buf)); + + // same precision, l.scale + r.scale < result.scale, l = 0 + l.s128Value = 0; + r.s128Value = 9223372036854775808ULL; + result.s128Value = 0; + + datatypes::Decimal::multiplication(l, r, result); + EXPECT_EQ(0, result.s128Value); + + // same precision, l.scale + r.scale > result.scale, both positive values + l.scale = 38; + l.precision = 38; + l.s128Value = (((int128_t)1234567890123456789ULL * 10000000000000000000ULL) + + 1234567890123456789); + + r.scale = 38; + r.precision = 38; + r.s128Value = (((int128_t)1234567890123456789ULL * 10000000000000000000ULL) + + 1234567890123456789ULL); + + result.scale = 38; + result.precision = 38; + result.s128Value = 0; + + datatypes::Decimal::multiplication(r, l, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("0.01524157875323883675019051998750190521", std::string(buf)); + + // same precision, l.scale + r.scale > result.scale, both negative values + l.s128Value = -l.s128Value; + r.s128Value = -r.s128Value; + result.s128Value = 0; + + datatypes::Decimal::multiplication(r, l, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("0.01524157875323883675019051998750190521", std::string(buf)); + + // same precision, l.scale + r.scale > result.scale, +- values + r.s128Value = -r.s128Value; + result.s128Value = 0; + + datatypes::Decimal::multiplication(l, r, result); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("-0.01524157875323883675019051998750190521", std::string(buf)); + + // same precision, l.scale + r.scale > result.scale, l = 0 + r.s128Value = 0; + result.s128Value = 0; + + datatypes::Decimal::multiplication(l, r, result); + EXPECT_EQ(0, result.s128Value); +} + +void doMultiply(const execplan::IDB_Decimal& l, + const execplan::IDB_Decimal& r, + execplan::IDB_Decimal& result) +{ + datatypes::Decimal::multiplication(l, r, result); } TEST(Decimal, multiplicationWithOverflowCheck) { - datatypes::MultiplicationOverflowCheck mul; - int128_t x = 42, y = 42, r = 0; - execplan::IDB_Decimal d; - EXPECT_NO_THROW(mul(x, y, r)); - EXPECT_EQ(x*y, r); + execplan::CalpontSystemCatalog::ColType ct; + ct.colDataType = execplan::CalpontSystemCatalog::DECIMAL; + char buf[42]; + execplan::IDB_Decimal l, r, result; + // result.scale >= l.scale + r.scale + l.scale = 0; + l.precision = 38; + l.s128Value = UINT64_MAX; - x = datatypes::Decimal::maxInt128, y = 42, r = 0; - EXPECT_THROW(mul(x, y, r), logging::OperationOverflowExcept); + r.scale = 0; + r.precision = 38; + r.s128Value = UINT64_MAX; + + result.scale = 0; + result.precision = 38; + result.s128Value = 42; + EXPECT_THROW(doMultiply(l, r, result), logging::OperationOverflowExcept); + + // result.scale < l.scale + r.scale + l.scale = 36; + l.precision = 38; + l.s128Value = (((int128_t)1234567890123456789ULL * 10000000000000000000ULL) + + 1234567890123456789); + + r.scale = 36; + r.precision = 38; + r.s128Value = (((int128_t)1234567890123456789ULL * 10000000000000000000ULL) + + 1234567890123456789); + + result.scale = 38; + result.precision = 38; + result.s128Value = 42; + EXPECT_THROW(doMultiply(l, r, result), logging::OperationOverflowExcept); + + // Normal execution w/o overflow + l.scale = 0; + l.s128Value = 4611686018427387904; + + r.scale = 0; + r.s128Value = 4611686018427387904; + + result.scale = 0; + result.s128Value = 0; + + EXPECT_NO_THROW(doMultiply(l, r, result)); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + EXPECT_EQ("21267647932558653966460912964485513216", std::string(buf)); } From cfe35b5c7fc2c5ad91aa19a93b2758d0f72e0e27 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Sun, 19 Apr 2020 23:27:43 -0400 Subject: [PATCH 34/78] MCOL-641 Add support for functions (Part 1). --- datatypes/mcs_decimal.h | 97 ++++ dbcon/execplan/functioncolumn.h | 22 +- dbcon/execplan/treenode.h | 168 ++++++- dbcon/joblist/tupleconstantstep.cpp | 4 +- dbcon/mysql/ha_mcs_execplan.cpp | 31 +- utils/dataconvert/dataconvert.cpp | 24 - utils/dataconvert/dataconvert.h | 24 + utils/funcexp/CMakeLists.txt | 2 +- utils/funcexp/func_abs.cpp | 6 +- utils/funcexp/func_bitwise.cpp | 56 ++- utils/funcexp/func_cast.cpp | 667 +++++++++++++++++++++------- utils/funcexp/func_ceil.cpp | 253 ++++++++++- utils/funcexp/func_char.cpp | 51 ++- utils/funcexp/func_elt.cpp | 37 +- utils/funcexp/func_floor.cpp | 236 ++++++++-- utils/funcexp/func_inet_aton.cpp | 31 +- utils/funcexp/func_inet_ntoa.cpp | 2 - utils/funcexp/func_makedate.cpp | 95 +++- utils/funcexp/func_maketime.cpp | 111 ++++- utils/funcexp/func_mod.cpp | 180 +++++++- utils/funcexp/func_monthname.cpp | 6 +- utils/funcexp/func_period_diff.cpp | 38 +- utils/funcexp/func_round.cpp | 205 ++++++--- utils/funcexp/func_sec_to_time.cpp | 17 +- utils/funcexp/func_truncate.cpp | 184 ++++++-- utils/funcexp/funchelpers.h | 3 +- utils/funcexp/functor_real.h | 10 + utils/rowgroup/rowgroup.h | 4 + 28 files changed, 2102 insertions(+), 462 deletions(-) diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index 54071c03b..cf70a357d 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -19,6 +19,7 @@ #define H_DECIMALDATATYPE #include +#include #include "calpontsystemcatalog.h" @@ -240,6 +241,102 @@ class Decimal ct.scale = 0; } } + + /** + @brief The method converts a __float128 value to an int64_t. + */ + static inline int64_t getInt64FromFloat128(const __float128& value) + { + if (value > static_cast<__float128>(INT64_MAX)) + return INT64_MAX; + else if (value < static_cast<__float128>(INT64_MIN)) + return INT64_MIN; + + return static_cast(value); + } + + /** + @brief The method converts a __float128 value to an uint64_t. + */ + static inline uint64_t getUInt64FromFloat128(const __float128& value) + { + if (value > static_cast<__float128>(UINT64_MAX)) + return UINT64_MAX; + else if (value < 0) + return 0; + + return static_cast(value); + } + + /** + @brief The method converts a __float128 value to a double. + */ + static inline double getDoubleFromFloat128(const __float128& value) + { + if (value > static_cast<__float128>(DBL_MAX)) + return DBL_MAX; + else if (value < -static_cast<__float128>(DBL_MAX)) + return -DBL_MAX; + + return static_cast(value); + } + + /** + @brief The method converts a wide decimal value to a double. + */ + static inline double getDoubleFromWideDecimal(const int128_t& value, int8_t scale) + { + int128_t scaleDivisor; + + getScaleDivisor(scaleDivisor, scale); + + __float128 tmpval = (__float128) value / scaleDivisor; + + return getDoubleFromFloat128(tmpval); + } + + /** + @brief The method converts a wide decimal value to a double. + */ + static inline double getDoubleFromWideDecimal(const int128_t& value) + { + return getDoubleFromFloat128(static_cast<__float128>(value)); + } + + /** + @brief The method converts a __float128 value to a long double. + */ + static inline long double getLongDoubleFromFloat128(const __float128& value) + { + if (value > static_cast<__float128>(LDBL_MAX)) + return LDBL_MAX; + else if (value < -static_cast<__float128>(LDBL_MAX)) + return -LDBL_MAX; + + return static_cast(value); + } + + /** + @brief The method converts a wide decimal value to a long double. + */ + static inline long double getLongDoubleFromWideDecimal(const int128_t& value) + { + return getLongDoubleFromFloat128(static_cast<__float128>(value)); + } + + /** + @brief The method converts a wide decimal value to an int64_t, + saturating the value if necessary. + */ + static inline int64_t getInt64FromWideDecimal(const int128_t& value) + { + if (value > static_cast(INT64_MAX)) + return INT64_MAX; + else if (value < static_cast(INT64_MIN)) + return INT64_MIN; + + return static_cast(value); + } }; /** diff --git a/dbcon/execplan/functioncolumn.h b/dbcon/execplan/functioncolumn.h index 9952910ad..4d398d9a8 100644 --- a/dbcon/execplan/functioncolumn.h +++ b/dbcon/execplan/functioncolumn.h @@ -256,14 +256,24 @@ public: (datatypes::Decimal::isWideDecimalType(decimal.precision)) ? decimal.s128Value : decimal.value; - int128_t scaleMultiplier, result; + int128_t scaleMultiplier; int32_t scaleDiff = fResultType.scale - decimal.scale; datatypes::getScaleDivisor(scaleMultiplier, abs(scaleDiff)); - // WIP MCOL-641 Unconditionall overflow check - datatypes::MultiplicationOverflowCheck mul; - decimal.s128Value = (scaleDiff > 0 - && mul(decimal.s128Value, scaleMultiplier, result)) - ? result : decimal.s128Value / scaleMultiplier; + if (scaleMultiplier > 1) + { + if (scaleDiff > 0) + { + // WIP MCOL-641 Unconditionall overflow check + datatypes::MultiplicationNoOverflowCheck mul; + mul(decimal.s128Value, scaleMultiplier, decimal.s128Value); + } + else + { + decimal.s128Value = (int128_t)(decimal.s128Value > 0 ? + (__float128)decimal.s128Value / scaleMultiplier + 0.5 : + (__float128)decimal.s128Value / scaleMultiplier - 0.5); + } + } } else if (fResultType.colWidth == utils::MAXLEGACYWIDTH) { diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index be105cf92..56f21ccdd 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -63,17 +63,16 @@ typedef execplan::CalpontSystemCatalog::ColType Type; */ struct IDB_Decimal { - IDB_Decimal(): value(0), scale(0), precision(0) + IDB_Decimal(): s128Value(0), value(0), scale(0), precision(0) { - s128Value = 0; } - IDB_Decimal(int64_t val, int8_t s, uint8_t p) : - value (val), + IDB_Decimal(int64_t val, int8_t s, uint8_t p, int128_t val128 = 0) : + s128Value(val128), + value(val), scale(s), precision(p) { - s128Value = 0; } int decimalComp(const IDB_Decimal& d) const @@ -117,13 +116,32 @@ struct IDB_Decimal bool operator==(const IDB_Decimal& rhs) const { - if (utils::widthByPrecision(precision) == 16) + if (precision > datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) { if (scale == rhs.scale) return s128Value == rhs.s128Value; else return (datatypes::Decimal::compare(*this, rhs) == 0); } + else if (precision > datatypes::INT64MAXPRECISION && + rhs.precision <= datatypes::INT64MAXPRECISION) + { + const_cast(rhs).s128Value = rhs.value; + + if (scale == rhs.scale) + return s128Value == rhs.s128Value; + else + return (datatypes::Decimal::compare(*this, rhs) == 0); + } + else if (precision <= datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return (int128_t) value == rhs.s128Value; + else + return (datatypes::Decimal::compare(IDB_Decimal(0, scale, precision, (int128_t) value), rhs) == 0); + } else { if (scale == rhs.scale) @@ -135,13 +153,32 @@ struct IDB_Decimal bool operator>(const IDB_Decimal& rhs) const { - if (utils::widthByPrecision(precision) == 16) + if (precision > datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) { if (scale == rhs.scale) return s128Value > rhs.s128Value; else return (datatypes::Decimal::compare(*this, rhs) > 0); } + else if (precision > datatypes::INT64MAXPRECISION && + rhs.precision <= datatypes::INT64MAXPRECISION) + { + IDB_Decimal rhstmp(0, rhs.scale, rhs.precision, (int128_t) rhs.value); + + if (scale == rhstmp.scale) + return s128Value > rhstmp.s128Value; + else + return (datatypes::Decimal::compare(*this, rhstmp) > 0); + } + else if (precision <= datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return (int128_t) value > rhs.s128Value; + else + return (datatypes::Decimal::compare(IDB_Decimal(0, scale, precision, (int128_t) value), rhs) > 0); + } else { if (scale == rhs.scale) @@ -153,13 +190,32 @@ struct IDB_Decimal bool operator<(const IDB_Decimal& rhs) const { - if (utils::widthByPrecision(precision) == 16) + if (precision > datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) { if (scale == rhs.scale) return s128Value < rhs.s128Value; else return (datatypes::Decimal::compare(*this, rhs) < 0); } + else if (precision > datatypes::INT64MAXPRECISION && + rhs.precision <= datatypes::INT64MAXPRECISION) + { + IDB_Decimal rhstmp(0, rhs.scale, rhs.precision, (int128_t) rhs.value); + + if (scale == rhstmp.scale) + return s128Value < rhstmp.s128Value; + else + return (datatypes::Decimal::compare(*this, rhstmp) < 0); + } + else if (precision <= datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return (int128_t) value < rhs.s128Value; + else + return (datatypes::Decimal::compare(IDB_Decimal(0, scale, precision, (int128_t) value), rhs) < 0); + } else { if (scale == rhs.scale) @@ -171,13 +227,32 @@ struct IDB_Decimal bool operator>=(const IDB_Decimal& rhs) const { - if (utils::widthByPrecision(precision) == 16) + if (precision > datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) { if (scale == rhs.scale) return s128Value >= rhs.s128Value; else return (datatypes::Decimal::compare(*this, rhs) >= 0); } + else if (precision > datatypes::INT64MAXPRECISION && + rhs.precision <= datatypes::INT64MAXPRECISION) + { + IDB_Decimal rhstmp(0, rhs.scale, rhs.precision, (int128_t) rhs.value); + + if (scale == rhstmp.scale) + return s128Value >= rhstmp.s128Value; + else + return (datatypes::Decimal::compare(*this, rhstmp) >= 0); + } + else if (precision <= datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return (int128_t) value >= rhs.s128Value; + else + return (datatypes::Decimal::compare(IDB_Decimal(0, scale, precision, (int128_t) value), rhs) >= 0); + } else { if (scale == rhs.scale) @@ -189,13 +264,32 @@ struct IDB_Decimal bool operator<=(const IDB_Decimal& rhs) const { - if (utils::widthByPrecision(precision) == 16) + if (precision > datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) { if (scale == rhs.scale) return s128Value <= rhs.s128Value; else return (datatypes::Decimal::compare(*this, rhs) <= 0); } + else if (precision > datatypes::INT64MAXPRECISION && + rhs.precision <= datatypes::INT64MAXPRECISION) + { + IDB_Decimal rhstmp(0, rhs.scale, rhs.precision, (int128_t) rhs.value); + + if (scale == rhstmp.scale) + return s128Value <= rhstmp.s128Value; + else + return (datatypes::Decimal::compare(*this, rhstmp) <= 0); + } + else if (precision <= datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return (int128_t) value <= rhs.s128Value; + else + return (datatypes::Decimal::compare(IDB_Decimal(0, scale, precision, (int128_t) value), rhs) <= 0); + } else { if (scale == rhs.scale) @@ -207,13 +301,32 @@ struct IDB_Decimal bool operator!=(const IDB_Decimal& rhs) const { - if (utils::widthByPrecision(precision) == 16) + if (precision > datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) { if (scale == rhs.scale) return s128Value != rhs.s128Value; else return (datatypes::Decimal::compare(*this, rhs) != 0); } + else if (precision > datatypes::INT64MAXPRECISION && + rhs.precision <= datatypes::INT64MAXPRECISION) + { + IDB_Decimal rhstmp(0, rhs.scale, rhs.precision, (int128_t) rhs.value); + + if (scale == rhstmp.scale) + return s128Value != rhstmp.s128Value; + else + return (datatypes::Decimal::compare(*this, rhstmp) != 0); + } + else if (precision <= datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return (int128_t) value != rhs.s128Value; + else + return (datatypes::Decimal::compare(IDB_Decimal(0, scale, precision, (int128_t) value), rhs) != 0); + } else { if (scale == rhs.scale) @@ -308,7 +421,7 @@ struct Result { Result(): intVal(0), uintVal(0), origIntVal(0), dummy(0), doubleVal(0), longDoubleVal(0), floatVal(0), boolVal(false), - strVal(""), decimalVal(IDB_Decimal(0, 0, 0)), + strVal(""), decimalVal(IDB_Decimal()), valueConverted(false) {} int64_t intVal; uint64_t uintVal; @@ -783,7 +896,10 @@ inline const std::string& TreeNode::getStrVal(const std::string& timeZone) case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - dataconvert::DataConvert::decimalToString(fResult.decimalVal.value, fResult.decimalVal.scale, tmp, 22, fResultType.colDataType); + if (fResultType.colWidth == datatypes::MAXDECIMALWIDTH) + dataconvert::DataConvert::decimalToString(&fResult.decimalVal.s128Value, fResult.decimalVal.scale, tmp, utils::MAXLENGTH16BYTES, fResultType.colDataType); + else + dataconvert::DataConvert::decimalToString(fResult.decimalVal.value, fResult.decimalVal.scale, tmp, 22, fResultType.colDataType); fResult.strVal = std::string(tmp); break; } @@ -880,7 +996,20 @@ inline int64_t TreeNode::getIntVal() case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - return (int64_t)(fResult.decimalVal.value / pow((double)10, fResult.decimalVal.scale)); + if (fResultType.colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t scaleDivisor; + + datatypes::getScaleDivisor(scaleDivisor, fResult.decimalVal.scale); + + int128_t tmpval = fResult.decimalVal.s128Value / scaleDivisor; + + return datatypes::Decimal::getInt64FromWideDecimal(tmpval); + } + else + { + return (int64_t)(fResult.decimalVal.value / pow((double)10, fResult.decimalVal.scale)); + } } case CalpontSystemCatalog::DATE: @@ -1062,8 +1191,15 @@ inline double TreeNode::getDoubleVal() case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - // this may not be accurate. if this is problematic, change to pre-calculated power array. - return (double)(fResult.decimalVal.value / pow((double)10, fResult.decimalVal.scale)); + if (fResultType.colWidth == datatypes::MAXDECIMALWIDTH) + { + return datatypes::Decimal::getDoubleFromWideDecimal(fResult.decimalVal.s128Value, fResult.decimalVal.scale); + } + else + { + // this may not be accurate. if this is problematic, change to pre-calculated power array. + return (double)(fResult.decimalVal.value / pow((double)10, fResult.decimalVal.scale)); + } } case CalpontSystemCatalog::DATE: diff --git a/dbcon/joblist/tupleconstantstep.cpp b/dbcon/joblist/tupleconstantstep.cpp index 5b8f53810..2a6a2ee7f 100644 --- a/dbcon/joblist/tupleconstantstep.cpp +++ b/dbcon/joblist/tupleconstantstep.cpp @@ -529,7 +529,9 @@ void TupleConstantStep::fillInConstants(const rowgroup::Row& rowIn, rowgroup::Ro //fRowConst.copyField(rowOut.getData()+2, 0); // hardcoded 2 for rid length for (uint32_t i = 1; i < rowOut.getColumnCount(); i++) // WIP MCOL-641 implement copyBinaryField inside copyField - if (UNLIKELY(utils::isWide(rowIn.getColumnWidth(i - 1)))) + if (UNLIKELY(utils::isWide(rowIn.getColumnWidth(i - 1)) && + (rowIn.getColType(i - 1) == CalpontSystemCatalog::DECIMAL || + rowIn.getColType(i - 1) == CalpontSystemCatalog::UDECIMAL))) rowIn.copyBinaryField(rowOut, i, i - 1); else rowIn.copyField(rowOut, i, i - 1); diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 20a2158bc..a3a72428f 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -4095,6 +4095,20 @@ ReturnedColumn* buildFunctionColumn( fc->functionParms(funcParms); fc->resultType(colType_MysqlToIDB(ifp)); + // if the result type is DECIMAL and any function parameter is a wide decimal + // column, set the result colwidth to wide + if (fc->resultType().colDataType == CalpontSystemCatalog::DECIMAL) + { + for (size_t i = 0; i < funcParms.size(); i++) + { + if (datatypes::Decimal::isWideDecimalType(funcParms[i]->data()->resultType())) + { + fc->resultType().colWidth = datatypes::MAXDECIMALWIDTH; + break; + } + } + } + // MySQL give string result type for date function, but has the flag set. // we should set the result type to be datetime for comparision. if (ifp->field_type() == MYSQL_TYPE_DATETIME || @@ -4410,6 +4424,14 @@ ConstantColumn* buildDecimalColumn(Item* item, gp_walk_info& gwi) i = 1; } + bool specialPrecision = false; + + // handle the case when the constant value is 0.12345678901234567890123456789012345678 + // In this case idp->decimal_precision() = 39, but we can + if (((i + 1) < str->length()) && + str->ptr()[i] == '0' && str->ptr()[i + 1] == '.') + specialPrecision = true; + for (; i < str->length(); i++) { if (str->ptr()[i] == '.') @@ -4420,7 +4442,9 @@ ConstantColumn* buildDecimalColumn(Item* item, gp_walk_info& gwi) if (idp->decimal_precision() <= datatypes::INT64MAXPRECISION) columnstore_decimal.value = strtoll(columnstore_decimal_val.str().c_str(), 0, 10); - else if (idp->decimal_precision() <= datatypes::INT128MAXPRECISION) + else if (idp->decimal_precision() <= datatypes::INT128MAXPRECISION || + (idp->decimal_precision() <= datatypes::INT128MAXPRECISION + 1 && + specialPrecision)) { bool dummy = false; columnstore_decimal.s128Value = @@ -4435,9 +4459,10 @@ ConstantColumn* buildDecimalColumn(Item* item, gp_walk_info& gwi) columnstore_decimal.value = (int64_t)(val > 0 ? val + 0.5 : val - 0.5); } else - columnstore_decimal.scale = idp->decimals; + columnstore_decimal.scale = idp->decimal_scale(); - columnstore_decimal.precision = idp->max_length - idp->decimals; + columnstore_decimal.precision = (idp->decimal_precision() > datatypes::INT128MAXPRECISION) ? + datatypes::INT128MAXPRECISION : idp->decimal_precision(); ConstantColumn* cc = new ConstantColumn(valStr, columnstore_decimal); cc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); cc->charsetNumber(idp->collation.collation->number); diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 546c24057..b7a0f9985 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -78,30 +78,6 @@ const int64_t columnstore_precision[19] = 999999999999999999LL }; -const string columnstore_big_precision[20] = -{ - "9999999999999999999", - "99999999999999999999", - "999999999999999999999", - "9999999999999999999999", - "99999999999999999999999", - "999999999999999999999999", - "9999999999999999999999999", - "99999999999999999999999999", - "999999999999999999999999999", - "9999999999999999999999999999", - "99999999999999999999999999999", - "999999999999999999999999999999", - "9999999999999999999999999999999", - "99999999999999999999999999999999", - "999999999999999999999999999999999", - "9999999999999999999999999999999999", - "99999999999999999999999999999999999", - "999999999999999999999999999999999999", - "9999999999999999999999999999999999999", - "99999999999999999999999999999999999999" -}; - template bool from_string(T& t, const std::string& s, std::ios_base & (*f)(std::ios_base&)) { diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index f70ec7f87..7c9f69278 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -118,6 +118,30 @@ const int64_t IDB_pow[19] = 1000000000000000000LL }; +const std::string columnstore_big_precision[20] = +{ + "9999999999999999999", + "99999999999999999999", + "999999999999999999999", + "9999999999999999999999", + "99999999999999999999999", + "999999999999999999999999", + "9999999999999999999999999", + "99999999999999999999999999", + "999999999999999999999999999", + "9999999999999999999999999999", + "99999999999999999999999999999", + "999999999999999999999999999999", + "9999999999999999999999999999999", + "99999999999999999999999999999999", + "999999999999999999999999999999999", + "9999999999999999999999999999999999", + "99999999999999999999999999999999999", + "999999999999999999999999999999999999", + "9999999999999999999999999999999999999", + "99999999999999999999999999999999999999" +}; + const int32_t SECS_PER_MIN = 60; const int32_t MINS_PER_HOUR = 60; const int32_t HOURS_PER_DAY = 24; diff --git a/utils/funcexp/CMakeLists.txt b/utils/funcexp/CMakeLists.txt index 889b44d64..8d194f105 100644 --- a/utils/funcexp/CMakeLists.txt +++ b/utils/funcexp/CMakeLists.txt @@ -112,7 +112,7 @@ add_library(funcexp SHARED ${funcexp_LIB_SRCS}) add_dependencies(funcexp loggingcpp) -target_link_libraries(funcexp ${NETSNMP_LIBRARIES}) +target_link_libraries(funcexp ${NETSNMP_LIBRARIES} quadmath) install(TARGETS funcexp DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) diff --git a/utils/funcexp/func_abs.cpp b/utils/funcexp/func_abs.cpp index 7b04408ba..830a48a6d 100644 --- a/utils/funcexp/func_abs.cpp +++ b/utils/funcexp/func_abs.cpp @@ -66,7 +66,11 @@ IDB_Decimal Func_abs::getDecimalVal(Row& row, CalpontSystemCatalog::ColType&) { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - d.value = llabs(d.value); + + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + d.s128Value = (d.s128Value < 0) ? -d.s128Value : d.s128Value; + else + d.value = llabs(d.value); return d; } diff --git a/utils/funcexp/func_bitwise.cpp b/utils/funcexp/func_bitwise.cpp index 2154f125b..beef7d290 100644 --- a/utils/funcexp/func_bitwise.cpp +++ b/utils/funcexp/func_bitwise.cpp @@ -102,22 +102,56 @@ bool getUIntValFromParm( { IDB_Decimal d = parm->data()->getDecimalVal(row, isNull); - if (parm->data()->resultType().colDataType == execplan::CalpontSystemCatalog::UDECIMAL && - d.value < 0) + if (parm->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) { - d.value = 0; + if (parm->data()->resultType().colDataType == execplan::CalpontSystemCatalog::UDECIMAL && + d.value < 0) + { + value = 0; + break; + } + + int128_t scaleDivisor, scaleDivisor2; + + datatypes::getScaleDivisor(scaleDivisor, d.scale); + + scaleDivisor2 = (scaleDivisor <= 10) ? 1 : (scaleDivisor / 10); + + int128_t tmpval = d.s128Value / scaleDivisor; + int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; + + if (tmpval >= 0 && lefto > 4) + tmpval++; + + if (tmpval < 0 && lefto < -4) + tmpval--; + + if (tmpval > static_cast(INT64_MAX)) + tmpval = INT64_MAX; + else if (tmpval < static_cast(INT64_MIN)) + tmpval = INT64_MIN; + + value = tmpval; } - double dscale = d.scale; - int64_t tmpval = d.value / pow(10.0, dscale); - int lefto = (d.value - tmpval * pow(10.0, dscale)) / pow(10.0, dscale - 1); + else + { + if (parm->data()->resultType().colDataType == execplan::CalpontSystemCatalog::UDECIMAL && + d.value < 0) + { + d.value = 0; + } + double dscale = d.scale; + int64_t tmpval = d.value / pow(10.0, dscale); + int lefto = (d.value - tmpval * pow(10.0, dscale)) / pow(10.0, dscale - 1); - if ( tmpval >= 0 && lefto > 4 ) - tmpval++; + if (tmpval >= 0 && lefto > 4) + tmpval++; - if ( tmpval < 0 && lefto < -4 ) - tmpval--; + if (tmpval < 0 && lefto < -4) + tmpval--; - value = tmpval; + value = tmpval; + } } break; diff --git a/utils/funcexp/func_cast.cpp b/utils/funcexp/func_cast.cpp index 3584b7102..a4c9fbe49 100644 --- a/utils/funcexp/func_cast.cpp +++ b/utils/funcexp/func_cast.cpp @@ -22,6 +22,8 @@ #include using namespace std; +#include + #include "functor_dtm.h" #include "functor_int.h" #include "functor_real.h" @@ -183,17 +185,40 @@ int64_t Func_cast_signed::getIntVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - double dscale = d.scale; - int64_t value = d.value / pow(10.0, dscale); - int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1); - if ( value >= 0 && lefto > 4 ) - value++; + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t scaleDivisor, scaleDivisor2; - if ( value < 0 && lefto < -4 ) - value--; + datatypes::getScaleDivisor(scaleDivisor, d.scale); - return value; + scaleDivisor2 = (scaleDivisor <= 10) ? 1 : (scaleDivisor / 10); + + int128_t tmpval = d.s128Value / scaleDivisor; + int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; + + if (tmpval >= 0 && lefto > 4) + tmpval++; + + if (tmpval < 0 && lefto < -4) + tmpval--; + + return datatypes::Decimal::getInt64FromWideDecimal(tmpval); + } + else + { + double dscale = d.scale; + int64_t value = d.value / pow(10.0, dscale); + int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1); + + if ( value >= 0 && lefto > 4 ) + value++; + + if ( value < 0 && lefto < -4 ) + value--; + + return value; + } } break; @@ -341,20 +366,48 @@ uint64_t Func_cast_unsigned::getUintVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - double dscale = d.scale; - if (d.value < 0) + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) { - return 0; - } + if (d.s128Value < 0) + { + return 0; + } - uint64_t value = d.value / pow(10.0, dscale); - int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1); + int128_t scaleDivisor, scaleDivisor2; + + datatypes::getScaleDivisor(scaleDivisor, d.scale); + + scaleDivisor2 = (scaleDivisor <= 10) ? 1 : (scaleDivisor / 10); + + uint128_t tmpval = d.s128Value / scaleDivisor; + int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; + + if (tmpval >= 0 && lefto > 4) + tmpval++; + + if (tmpval > static_cast(UINT64_MAX)) + tmpval = UINT64_MAX; + + return static_cast(tmpval); + } + else + { + if (d.value < 0) + { + return 0; + } + + double dscale = d.scale; + + uint64_t value = d.value / pow(10.0, dscale); + int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1); if ( utils::is_nonnegative(value) && lefto > 4 ) value++; - return value; + return value; + } } break; @@ -491,7 +544,10 @@ string Func_cast_char::getStrVal(Row& row, char buf[80]; - dataconvert::DataConvert::decimalToString( d.value, d.scale, buf, 80, parm[0]->data()->resultType().colDataType); + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + dataconvert::DataConvert::decimalToString( &d.s128Value, d.scale, buf, 80, parm[0]->data()->resultType().colDataType); + else + dataconvert::DataConvert::decimalToString( d.value, d.scale, buf, 80, parm[0]->data()->resultType().colDataType); string sbuf = buf; return sbuf.substr(0, length); @@ -583,10 +639,16 @@ IDB_Decimal Func_cast_date::getDecimalVal(Row& row, { IDB_Decimal decimal; - decimal.value = Func_cast_date::getDatetimeIntVal(row, - parm, - isNull, - operationColType); + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + decimal.s128Value = Func_cast_date::getDatetimeIntVal(row, + parm, + isNull, + operationColType); + else + decimal.value = Func_cast_date::getDatetimeIntVal(row, + parm, + isNull, + operationColType); return decimal; } @@ -886,10 +948,16 @@ IDB_Decimal Func_cast_datetime::getDecimalVal(Row& row, { IDB_Decimal decimal; - decimal.value = Func_cast_datetime::getDatetimeIntVal(row, - parm, - isNull, - operationColType); + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + decimal.s128Value = Func_cast_datetime::getDatetimeIntVal(row, + parm, + isNull, + operationColType); + else + decimal.value = Func_cast_datetime::getDatetimeIntVal(row, + parm, + isNull, + operationColType); return decimal; } @@ -1139,6 +1207,17 @@ int64_t Func_cast_decimal::getIntVal(Row& row, isNull, operationColType); + if (decimal.precision > datatypes::INT64MAXPRECISION) + { + int128_t scaleDivisor; + + datatypes::getScaleDivisor(scaleDivisor, decimal.scale); + + int128_t tmpval = decimal.s128Value / scaleDivisor; + + return datatypes::Decimal::getInt64FromWideDecimal(tmpval); + } + return (int64_t) decimal.value / helpers::powerOf10_c[decimal.scale]; } @@ -1155,7 +1234,10 @@ string Func_cast_decimal::getStrVal(Row& row, char buf[80]; - dataconvert::DataConvert::decimalToString( decimal.value, decimal.scale, buf, 80, operationColType.colDataType); + if (decimal.precision > datatypes::INT64MAXPRECISION) + dataconvert::DataConvert::decimalToString( &decimal.s128Value, decimal.scale, buf, 80, operationColType.colDataType); + else + dataconvert::DataConvert::decimalToString( decimal.value, decimal.scale, buf, 80, operationColType.colDataType); string value = buf; return value; @@ -1173,12 +1255,10 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, int32_t decimals = parm[1]->data()->getIntVal(row, isNull); int64_t max_length = parm[2]->data()->getIntVal(row, isNull); - // As of 2.0, max length columnStore can support is 18 - // decimal(0,0) is valid, and no limit on integer number - if (max_length > 18 || max_length <= 0) - max_length = 18; + if (max_length > datatypes::INT128MAXPRECISION || max_length <= 0) + max_length = datatypes::INT128MAXPRECISION; - int64_t max_number_decimal = helpers::maxNumber_c[max_length]; + decimal.precision = max_length; switch (parm[0]->data()->resultType().colDataType) { @@ -1188,19 +1268,45 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, case execplan::CalpontSystemCatalog::TINYINT: case execplan::CalpontSystemCatalog::SMALLINT: { - decimal.value = parm[0]->data()->getIntVal(row, isNull); - decimal.scale = 0; - int64_t value = decimal.value * helpers::powerOf10_c[decimals]; + if (max_length > datatypes::INT64MAXPRECISION) + { + bool dummy = false; + char *ep = NULL; + int128_t max_number_decimal = dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); + decimal.s128Value = parm[0]->data()->getIntVal(row, isNull); + decimal.scale = 0; + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, decimals); + int128_t value = decimal.s128Value * scaleDivisor; - if ( value > max_number_decimal ) - { - decimal.value = max_number_decimal; - decimal.scale = decimals; + if ( value > max_number_decimal ) + { + decimal.s128Value = max_number_decimal; + decimal.scale = decimals; + } + else if ( value < -max_number_decimal ) + { + decimal.s128Value = -max_number_decimal; + decimal.scale = decimals; + } } - else if ( value < -max_number_decimal ) + else { - decimal.value = -max_number_decimal; - decimal.scale = decimals; + int64_t max_number_decimal = helpers::maxNumber_c[max_length]; + decimal.value = parm[0]->data()->getIntVal(row, isNull); + decimal.scale = 0; + int64_t value = decimal.value * helpers::powerOf10_c[decimals]; + + if ( value > max_number_decimal ) + { + decimal.value = max_number_decimal; + decimal.scale = decimals; + } + else if ( value < -max_number_decimal ) + { + decimal.value = -max_number_decimal; + decimal.scale = decimals; + } } } break; @@ -1211,21 +1317,51 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, case execplan::CalpontSystemCatalog::UTINYINT: case execplan::CalpontSystemCatalog::USMALLINT: { - uint64_t uval = parm[0]->data()->getUintVal(row, isNull); - - if (uval > (uint64_t)numeric_limits::max()) + if (max_length > datatypes::INT64MAXPRECISION) { - uval = numeric_limits::max(); + bool dummy = false; + char *ep = NULL; + int128_t max_number_decimal = dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); + + uint128_t uval = parm[0]->data()->getUintVal(row, isNull); + + if (uval > (uint128_t)datatypes::Decimal::maxInt128) + { + uval = datatypes::Decimal::maxInt128; + } + + decimal.s128Value = uval; + decimal.scale = 0; + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, decimals); + int128_t value = decimal.s128Value * scaleDivisor; + + if ( value > max_number_decimal ) + { + decimal.s128Value = max_number_decimal; + decimal.scale = decimals; + } } - - decimal.value = uval; - decimal.scale = 0; - int64_t value = decimal.value * helpers::powerOf10_c[decimals]; - - if ( value > max_number_decimal ) + else { - decimal.value = max_number_decimal; - decimal.scale = decimals; + int64_t max_number_decimal = helpers::maxNumber_c[max_length]; + + uint64_t uval = parm[0]->data()->getUintVal(row, isNull); + + if (uval > (uint64_t)numeric_limits::max()) + { + uval = numeric_limits::max(); + } + + decimal.value = uval; + decimal.scale = 0; + int64_t value = decimal.value * helpers::powerOf10_c[decimals]; + + if ( value > max_number_decimal ) + { + decimal.value = max_number_decimal; + decimal.scale = decimals; + } } } break; @@ -1235,67 +1371,172 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::UFLOAT: { - double value = parm[0]->data()->getDoubleVal(row, isNull); + if (max_length > datatypes::INT64MAXPRECISION) + { + bool dummy = false; + char *ep = NULL; + int128_t max_number_decimal = dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); - if (value > 0) - decimal.value = (int64_t) (value * helpers::powerOf10_c[decimals] + 0.5); - else if (value < 0) - decimal.value = (int64_t) (value * helpers::powerOf10_c[decimals] - 0.5); + __float128 value = parm[0]->data()->getDoubleVal(row, isNull); + + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, decimals); + + if (value > 0) + decimal.s128Value = (int128_t) (value * scaleDivisor + 0.5); + else if (value < 0) + decimal.s128Value = (int128_t) (value * scaleDivisor - 0.5); + else + decimal.s128Value = 0; + + decimal.scale = decimals; + + if ( value > max_number_decimal ) + decimal.s128Value = max_number_decimal; + else if ( value < -max_number_decimal ) + decimal.s128Value = -max_number_decimal; + } else - decimal.value = 0; + { + int64_t max_number_decimal = helpers::maxNumber_c[max_length]; - decimal.scale = decimals; + double value = parm[0]->data()->getDoubleVal(row, isNull); - if ( value > max_number_decimal ) - decimal.value = max_number_decimal; - else if ( value < -max_number_decimal ) - decimal.value = -max_number_decimal; + if (value > 0) + decimal.value = (int64_t) (value * helpers::powerOf10_c[decimals] + 0.5); + else if (value < 0) + decimal.value = (int64_t) (value * helpers::powerOf10_c[decimals] - 0.5); + else + decimal.value = 0; + + decimal.scale = decimals; + + if ( value > max_number_decimal ) + decimal.value = max_number_decimal; + else if ( value < -max_number_decimal ) + decimal.value = -max_number_decimal; + } } break; case execplan::CalpontSystemCatalog::LONGDOUBLE: { - long double value = parm[0]->data()->getLongDoubleVal(row, isNull); + if (max_length > datatypes::INT64MAXPRECISION) + { + bool dummy = false; + char *ep = NULL; + int128_t max_number_decimal = dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); - if (value > 0) - decimal.value = (int64_t) (value * helpers::powerOf10_c[decimals] + 0.5); - else if (value < 0) - decimal.value = (int64_t) (value * helpers::powerOf10_c[decimals] - 0.5); + __float128 value = parm[0]->data()->getLongDoubleVal(row, isNull); + + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, decimals); + + if (value > 0) + decimal.s128Value = (int128_t) (value * scaleDivisor + 0.5); + else if (value < 0) + decimal.s128Value = (int128_t) (value * scaleDivisor - 0.5); + else + decimal.s128Value = 0; + + decimal.scale = decimals; + + if ( value > max_number_decimal ) + decimal.s128Value = max_number_decimal; + else if ( value < -max_number_decimal ) + decimal.s128Value = -max_number_decimal; + } else - decimal.value = 0; + { + int64_t max_number_decimal = helpers::maxNumber_c[max_length]; - decimal.scale = decimals; + long double value = parm[0]->data()->getLongDoubleVal(row, isNull); - if ( value > max_number_decimal ) - decimal.value = max_number_decimal; - else if ( value < -max_number_decimal ) - decimal.value = -max_number_decimal; + if (value > 0) + decimal.value = (int64_t) (value * helpers::powerOf10_c[decimals] + 0.5); + else if (value < 0) + decimal.value = (int64_t) (value * helpers::powerOf10_c[decimals] - 0.5); + else + decimal.value = 0; + + decimal.scale = decimals; + + if ( value > max_number_decimal ) + decimal.value = max_number_decimal; + else if ( value < -max_number_decimal ) + decimal.value = -max_number_decimal; + } } break; case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - decimal = parm[0]->data()->getDecimalVal(row, isNull); + if (max_length > datatypes::INT64MAXPRECISION) + { + bool dummy = false; + char *ep = NULL; + int128_t max_number_decimal = dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); + decimal = parm[0]->data()->getDecimalVal(row, isNull); - if (decimals > decimal.scale) - decimal.value *= helpers::powerOf10_c[decimals - decimal.scale]; + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, abs(decimals - decimal.scale)); + + if (decimal.precision <= datatypes::INT64MAXPRECISION) + decimal.s128Value = decimal.value; + + decimal.precision = max_length; + + if (scaleDivisor > 1) + { + if (decimals > decimal.scale) + decimal.s128Value *= scaleDivisor; + else + decimal.s128Value = (int128_t)(decimal.s128Value > 0 ? + (__float128)decimal.s128Value / scaleDivisor + 0.5 : + (__float128)decimal.s128Value / scaleDivisor - 0.5); + } + + decimal.scale = decimals; + + if ( decimal.s128Value > max_number_decimal ) + decimal.s128Value = max_number_decimal; + else if ( decimal.s128Value < -max_number_decimal ) + decimal.s128Value = -max_number_decimal; + } else - decimal.value = (int64_t)(decimal.value > 0 ? - (double)decimal.value / helpers::powerOf10_c[decimal.scale - decimals] + 0.5 : - (double)decimal.value / helpers::powerOf10_c[decimal.scale - decimals] - 0.5); + { + int64_t max_number_decimal = helpers::maxNumber_c[max_length]; - decimal.scale = decimals; + decimal = parm[0]->data()->getDecimalVal(row, isNull); + if (decimal.precision > datatypes::INT64MAXPRECISION) + { + if ( decimal.s128Value > (int128_t) max_number_decimal ) + decimal.value = max_number_decimal; + else if ( decimal.s128Value < (int128_t) -max_number_decimal ) + decimal.value = -max_number_decimal; + else + decimal.value = decimal.s128Value; + } + decimal.precision = max_length; - //int64_t value = decimal.value; + if (decimals > decimal.scale) + decimal.value *= helpers::powerOf10_c[decimals - decimal.scale]; + else + decimal.value = (int64_t)(decimal.value > 0 ? + (double)decimal.value / helpers::powerOf10_c[decimal.scale - decimals] + 0.5 : + (double)decimal.value / helpers::powerOf10_c[decimal.scale - decimals] - 0.5); - if ( decimal.value > max_number_decimal ) - decimal.value = max_number_decimal; - else if ( decimal.value < -max_number_decimal ) - decimal.value = -max_number_decimal; + decimal.scale = decimals; + + if ( decimal.value > max_number_decimal ) + decimal.value = max_number_decimal; + else if ( decimal.value < -max_number_decimal ) + decimal.value = -max_number_decimal; + } } break; @@ -1313,9 +1554,6 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, int negate = 1; bool bFoundSign = false; bool bRound = false; - double floatValue; - int64_t value = 0; - int64_t frac = 0; if (strValue.empty()) { @@ -1331,27 +1569,63 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, { if (*s == 'e' || *s == 'E') { - floatValue = strtod(str, 0); + if (max_length > datatypes::INT64MAXPRECISION) + { + bool dummy = false; + char *ep = NULL; + int128_t max_number_decimal = dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); - // If the float value is too large, the saturated result may end up with - // the wrong sign, so we just check first. - if ((int64_t)floatValue > max_number_decimal) - decimal.value = max_number_decimal; - else if ((int64_t)floatValue < -max_number_decimal) - decimal.value = -max_number_decimal; - else if (floatValue > 0) - decimal.value = (int64_t) (floatValue * helpers::powerOf10_c[decimals] + 0.5); - else if (floatValue < 0) - decimal.value = (int64_t) (floatValue * helpers::powerOf10_c[decimals] - 0.5); + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, decimals); + + __float128 floatValue = strtoflt128 (str, 0); + + // If the float value is too large, the saturated result may end up with + // the wrong sign, so we just check first. + if ((int128_t)floatValue > max_number_decimal) + decimal.s128Value = max_number_decimal; + else if ((int128_t)floatValue < -max_number_decimal) + decimal.s128Value = -max_number_decimal; + else if (floatValue > 0) + decimal.s128Value = (int128_t) (floatValue * scaleDivisor + 0.5); + else if (floatValue < 0) + decimal.s128Value = (int128_t) (floatValue * scaleDivisor - 0.5); + else + decimal.s128Value = 0; + + if (decimal.s128Value > max_number_decimal) + decimal.s128Value = max_number_decimal; + else if (decimal.s128Value < -max_number_decimal) + decimal.s128Value = -max_number_decimal; + + return decimal; + } else - decimal.value = 0; + { + int64_t max_number_decimal = helpers::maxNumber_c[max_length]; - if (decimal.value > max_number_decimal) - decimal.value = max_number_decimal; - else if (decimal.value < -max_number_decimal) - decimal.value = -max_number_decimal; + double floatValue = strtod(str, 0); - return decimal; + // If the float value is too large, the saturated result may end up with + // the wrong sign, so we just check first. + if ((int64_t)floatValue > max_number_decimal) + decimal.value = max_number_decimal; + else if ((int64_t)floatValue < -max_number_decimal) + decimal.value = -max_number_decimal; + else if (floatValue > 0) + decimal.value = (int64_t) (floatValue * helpers::powerOf10_c[decimals] + 0.5); + else if (floatValue < 0) + decimal.value = (int64_t) (floatValue * helpers::powerOf10_c[decimals] - 0.5); + else + decimal.value = 0; + + if (decimal.value > max_number_decimal) + decimal.value = max_number_decimal; + else if (decimal.value < -max_number_decimal) + decimal.value = -max_number_decimal; + + return decimal; + } } } @@ -1392,54 +1666,119 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, } } - errno = 0; - - if (firstInt) // Checking to see if we have a decimal point, but no previous digits. + if (max_length > datatypes::INT64MAXPRECISION) { - value = strtoll(firstInt, &endptr, 10); - } + bool dummy = false; + char *ep = NULL; + int128_t max_number_decimal = dataconvert::strtoll128(columnstore_big_precision[max_length - 19].c_str(), dummy, &ep); - if (!errno && endptr) - { - // Scale the integer portion according to the DECIMAL description - value *= helpers::powerOf10_c[decimals]; + int128_t value = 0, frac = 0; - // Get the fractional part. - if (endptr && (*endptr == *convData->decimal_point || *endptr == '.')) + if (firstInt) // Checking to see if we have a decimal point, but no previous digits. { - s = endptr + 1; - - // Get the digits to the right of the decimal - // Only retrieve those that matter based on scale. - for (fracChars = 0; - *s && isdigit(*s) && fracChars < decimals; - ++fracChars, ++s) - { - // Save the frac characters to a side buffer. This way we can limit - // ourselves to the scale without modifying the original string. - fracBuf[fracChars] = *s; - } - - fracBuf[fracChars] = 0; - - // Check to see if we need to round - if (isdigit(*s) && *s > '4') - { - bRound = true; - } + value = dataconvert::strtoll128(firstInt, dummy, &endptr); } - frac = strtoll(fracBuf, &endptr, 10); - value += frac + (bRound ? 1 : 0); - value *= negate; + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, decimals); + + if (!dummy && endptr) + { + // Scale the integer portion according to the DECIMAL description + value *= scaleDivisor; + + // Get the fractional part. + if (endptr && (*endptr == *convData->decimal_point || *endptr == '.')) + { + s = endptr + 1; + + // Get the digits to the right of the decimal + // Only retrieve those that matter based on scale. + for (fracChars = 0; + *s && isdigit(*s) && fracChars < decimals; + ++fracChars, ++s) + { + // Save the frac characters to a side buffer. This way we can limit + // ourselves to the scale without modifying the original string. + fracBuf[fracChars] = *s; + } + + fracBuf[fracChars] = 0; + + // Check to see if we need to round + if (isdigit(*s) && *s > '4') + { + bRound = true; + } + } + + frac = dataconvert::strtoll128(fracBuf, dummy, &ep); + value += frac + (bRound ? 1 : 0); + value *= negate; + } + + decimal.s128Value = value; + + if (decimal.s128Value > max_number_decimal) + decimal.s128Value = max_number_decimal; + else if (decimal.s128Value < -max_number_decimal) + decimal.s128Value = -max_number_decimal; } + else + { + int64_t max_number_decimal = helpers::maxNumber_c[max_length]; - decimal.value = value; + int64_t value = 0, frac = 0; - if (decimal.value > max_number_decimal) - decimal.value = max_number_decimal; - else if (decimal.value < -max_number_decimal) - decimal.value = -max_number_decimal; + errno = 0; + + if (firstInt) // Checking to see if we have a decimal point, but no previous digits. + { + value = strtoll(firstInt, &endptr, 10); + } + + if (!errno && endptr) + { + // Scale the integer portion according to the DECIMAL description + value *= helpers::powerOf10_c[decimals]; + + // Get the fractional part. + if (endptr && (*endptr == *convData->decimal_point || *endptr == '.')) + { + s = endptr + 1; + + // Get the digits to the right of the decimal + // Only retrieve those that matter based on scale. + for (fracChars = 0; + *s && isdigit(*s) && fracChars < decimals; + ++fracChars, ++s) + { + // Save the frac characters to a side buffer. This way we can limit + // ourselves to the scale without modifying the original string. + fracBuf[fracChars] = *s; + } + + fracBuf[fracChars] = 0; + + // Check to see if we need to round + if (isdigit(*s) && *s > '4') + { + bRound = true; + } + } + + frac = strtoll(fracBuf, &endptr, 10); + value += frac + (bRound ? 1 : 0); + value *= negate; + } + + decimal.value = value; + + if (decimal.value > max_number_decimal) + decimal.value = max_number_decimal; + else if (decimal.value < -max_number_decimal) + decimal.value = -max_number_decimal; + } } break; @@ -1452,7 +1791,11 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, if (!isNull) { - decimal.value = x; + if (max_length > datatypes::INT64MAXPRECISION) + decimal.s128Value = x; + else + decimal.value = x; + decimal.scale = s; } } @@ -1471,7 +1814,11 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, if (!isNull) { - decimal.value = x; + if (max_length > datatypes::INT64MAXPRECISION) + decimal.s128Value = x; + else + decimal.value = x; + decimal.scale = s; } } @@ -1490,7 +1837,11 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, if (!isNull) { - decimal.value = x; + if (max_length > datatypes::INT64MAXPRECISION) + decimal.s128Value = x; + else + decimal.value = x; + decimal.scale = s; } } @@ -1509,7 +1860,11 @@ IDB_Decimal Func_cast_decimal::getDecimalVal(Row& row, if (!isNull) { - decimal.value = x; + if (max_length > datatypes::INT64MAXPRECISION) + decimal.s128Value = x; + else + decimal.value = x; + decimal.scale = s; } } @@ -1536,6 +1891,11 @@ double Func_cast_decimal::getDoubleVal(Row& row, isNull, operationColType); + if (decimal.precision > datatypes::INT64MAXPRECISION) + { + return datatypes::Decimal::getDoubleFromWideDecimal(decimal.s128Value, decimal.scale); + } + return (double) decimal.value / helpers::powerOf10_c[decimal.scale]; } @@ -1642,7 +2002,14 @@ double Func_cast_double::getDoubleVal(Row& row, { IDB_Decimal decimal = parm[0]->data()->getDecimalVal(row, isNull); - dblval = (double)(decimal.value / pow((double)10, decimal.scale)); + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + dblval = datatypes::Decimal::getDoubleFromWideDecimal(decimal.s128Value, decimal.scale); + } + else + { + dblval = (double)(decimal.value / pow((double)10, decimal.scale)); + } } break; diff --git a/utils/funcexp/func_ceil.cpp b/utils/funcexp/func_ceil.cpp index 967989c74..26200a94f 100644 --- a/utils/funcexp/func_ceil.cpp +++ b/utils/funcexp/func_ceil.cpp @@ -65,42 +65,58 @@ int64_t Func_ceil::getIntVal(Row& row, case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::TINYINT: case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: { - if (op_ct.scale == 0) - { - ret = parm[0]->data()->getIntVal(row, isNull); - break; - } + ret = parm[0]->data()->getIntVal(row, isNull); + } + break; - IDB_Decimal decimal = parm[0]->data()->getDecimalVal(row, isNull); + // ceil(decimal(38,38)) leads to this path + // otherwise Func_ceil::getDecimalVal() is called + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); if (isNull) break; - ret = decimal.value; - // negative scale is not supported by CNX yet - if (decimal.scale > 0) + if (d.scale > 0) { - - if (decimal.scale >= 19) + if (d.scale > datatypes::INT128MAXPRECISION) { std::ostringstream oss; - oss << "ceil: datatype of " << colDataTypeToString(op_ct.colDataType) - << " with scale " << (int) decimal.scale << " is beyond supported scale"; + oss << "ceil: datatype of " << execplan::colDataTypeToString(op_ct.colDataType) + << " with scale " << (int) d.scale << " is beyond supported scale"; throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); } - // Adjust to an int based on the scale. - int64_t tmp = ret; - ret /= helpers::powerOf10_c[decimal.scale]; + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t tmp = d.s128Value; + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + d.s128Value /= scaleDivisor; - // Add 1 if this is a positive number and there were values to the right of the - // decimal point so that we return the largest integer value not less than X. - if ((tmp - (ret * helpers::powerOf10_c[decimal.scale]) > 0)) - ret += 1; + // Add 1 if this is a positive number and there were values to the right of the + // decimal point so that we return the largest integer value not less than X. + if ((tmp - (d.s128Value * scaleDivisor)) > 0) + d.s128Value += 1; + + ret = datatypes::Decimal::getInt64FromWideDecimal(d.s128Value); + } + else + { + int64_t tmp = d.value; + d.value /= helpers::powerOf10_c[d.scale]; + + // Add 1 if this is a positive number and there were values to the right of the + // decimal point so that we return the largest integer value not less than X. + if ((tmp - (d.value * helpers::powerOf10_c[d.scale])) > 0) + d.value += 1; + + ret = d.value; + } } } break; @@ -308,6 +324,20 @@ double Func_ceil::getDoubleVal(Row& row, { ret = (double)ceill(parm[0]->data()->getLongDoubleVal(row, isNull)); } + else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || + op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) + { + IDB_Decimal tmp = getDecimalVal(row, parm, isNull, op_ct); + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + ret = datatypes::Decimal::getDoubleFromWideDecimal(tmp.s128Value); + } + else + { + ret = (double) tmp.value; + } + } else { if (isUnsigned(op_ct.colDataType)) @@ -350,6 +380,20 @@ long double Func_ceil::getLongDoubleVal(Row& row, if (!isNull) ret = ceil(strtod(str.c_str(), 0)); } + else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || + op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) + { + IDB_Decimal tmp = getDecimalVal(row, parm, isNull, op_ct); + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + ret = datatypes::Decimal::getLongDoubleFromWideDecimal(tmp.s128Value); + } + else + { + ret = (long double) tmp.value; + } + } else { if (isUnsigned(op_ct.colDataType)) @@ -403,6 +447,20 @@ string Func_ceil::getStrVal(Row& row, *d = '\0'; } + else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || + op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) + { + IDB_Decimal d = getDecimalVal(row, parm, isNull, op_ct); + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + dataconvert::DataConvert::decimalToString(&d.s128Value, d.scale, tmp, 511, op_ct.colDataType); + } + else + { + dataconvert::DataConvert::decimalToString(d.value, d.scale, tmp, 511, op_ct.colDataType); + } + } else if (isUnsigned(op_ct.colDataType)) { #ifndef __LP64__ @@ -424,5 +482,156 @@ string Func_ceil::getStrVal(Row& row, } +IDB_Decimal Func_ceil::getDecimalVal(Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& op_ct) +{ + IDB_Decimal ret; + + switch (op_ct.colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + { + ret.value = parm[0]->data()->getIntVal(row, isNull); + } + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + ret = parm[0]->data()->getDecimalVal(row, isNull); + + if (isNull) + break; + + // negative scale is not supported by CNX yet + if (ret.scale > 0) + { + if (ret.scale > datatypes::INT128MAXPRECISION) + { + std::ostringstream oss; + oss << "ceil: datatype of " << execplan::colDataTypeToString(op_ct.colDataType) + << " with scale " << (int) ret.scale << " is beyond supported scale"; + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t tmp = ret.s128Value; + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, ret.scale); + ret.s128Value /= scaleDivisor; + + // Add 1 if this is a positive number and there were values to the right of the + // decimal point so that we return the largest integer value not less than X. + if ((tmp - (ret.s128Value * scaleDivisor)) > 0) + ret.s128Value += 1; + } + else + { + int64_t tmp = ret.value; + ret.value /= helpers::powerOf10_c[ret.scale]; + + // Add 1 if this is a positive number and there were values to the right of the + // decimal point so that we return the largest integer value not less than X. + if ((tmp - (ret.value * helpers::powerOf10_c[ret.scale])) > 0) + ret.value += 1; + } + } + } + break; + + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + ret.value = (int64_t)parm[0]->data()->getUintVal(row, isNull); + } + break; + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + ret.value = (int64_t) ceil(parm[0]->data()->getDoubleVal(row, isNull)); + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + ret.value = (int64_t) ceill(parm[0]->data()->getLongDoubleVal(row, isNull)); + } + break; + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + const string& str = parm[0]->data()->getStrVal(row, isNull); + + if (!isNull) + ret.value = (int64_t) ceil(strtod(str.c_str(), 0)); + } + break; + + case CalpontSystemCatalog::DATE: + { + Date d (parm[0]->data()->getDateIntVal(row, isNull)); + + if (!isNull) + ret.value = d.convertToMySQLint(); + } + break; + + case CalpontSystemCatalog::DATETIME: + { + DateTime dt(parm[0]->data()->getDatetimeIntVal(row, isNull)); + + if (!isNull) + ret.value = dt.convertToMySQLint(); + } + break; + + case CalpontSystemCatalog::TIMESTAMP: + { + TimeStamp dt(parm[0]->data()->getTimestampIntVal(row, isNull)); + + if (!isNull) + ret.value = dt.convertToMySQLint(timeZone()); + } + break; + + case CalpontSystemCatalog::TIME: + { + Time dt(parm[0]->data()->getTimeIntVal(row, isNull)); + + if (!isNull) + ret.value = dt.convertToMySQLint(); + } + break; + + default: + { + std::ostringstream oss; + oss << "ceil: datatype of " << colDataTypeToString(op_ct.colDataType) + << " is not supported"; + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + ret.scale = 0; + + return ret; +} + + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_char.cpp b/utils/funcexp/func_char.cpp index c4a27730e..d9904ba63 100644 --- a/utils/funcexp/func_char.cpp +++ b/utils/funcexp/func_char.cpp @@ -134,15 +134,50 @@ string Func_char::getStrVal(Row& row, case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - IDB_Decimal d = rc->getDecimalVal(row, isNull); - double dscale = d.scale; - // get decimal and round up - value = d.value / pow(10.0, dscale); - int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1); + IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - if ( lefto > 4 ) - value++; - + if (ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + if (d.s128Value < 0) + return ""; + + int128_t scaleDivisor, scaleDivisor2; + + datatypes::getScaleDivisor(scaleDivisor, d.scale); + + scaleDivisor2 = (scaleDivisor <= 10) ? 1 : (scaleDivisor / 10); + + int128_t tmpval = d.s128Value / scaleDivisor; + int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; + + if (lefto > 4) + tmpval++; + + if (tmpval > static_cast(INT64_MAX)) + tmpval = INT64_MAX; + + if ( !getChar((int64_t)tmpval, buf) ) + { + isNull = true; + return ""; + } + } + else + { + double dscale = d.scale; + // get decimal and round up + int value = d.value / pow(10.0, dscale); + int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1); + + if ( lefto > 4 ) + value++; + + if ( !getChar((int64_t)value, buf) ) + { + isNull = true; + return ""; + } + } } break; diff --git a/utils/funcexp/func_elt.cpp b/utils/funcexp/func_elt.cpp index ccdca2cae..2fd377dd4 100644 --- a/utils/funcexp/func_elt.cpp +++ b/utils/funcexp/func_elt.cpp @@ -73,15 +73,38 @@ string Func_elt::getStrVal(rowgroup::Row& row, case CalpontSystemCatalog::DECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - double dscale = d.scale; - number = d.value / pow(10.0, dscale); - int lefto = (d.value - number * pow(10.0, dscale)) / pow(10.0, dscale - 1); - if ( utils::is_nonnegative(number) && lefto > 4 ) - number++; + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t scaleDivisor, scaleDivisor2; - if ( utils::is_negative(number) && lefto < -4 ) - number--; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + + scaleDivisor2 = (scaleDivisor <= 10) ? 1 : (scaleDivisor / 10); + + int128_t tmpval = d.s128Value / scaleDivisor; + int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; + + if (utils::is_nonnegative(tmpval) >= 0 && lefto > 4) + tmpval++; + + if (utils::is_negative(tmpval) < 0 && lefto < -4) + tmpval--; + + number = datatypes::Decimal::getInt64FromWideDecimal(tmpval); + } + else + { + double dscale = d.scale; + number = d.value / pow(10.0, dscale); + int lefto = (d.value - number * pow(10.0, dscale)) / pow(10.0, dscale - 1); + + if ( utils::is_nonnegative(number) && lefto > 4 ) + number++; + + if ( utils::is_negative(number) && lefto < -4 ) + number--; + } break; } diff --git a/utils/funcexp/func_floor.cpp b/utils/funcexp/func_floor.cpp index 1e0ee7f09..5116a9a54 100644 --- a/utils/funcexp/func_floor.cpp +++ b/utils/funcexp/func_floor.cpp @@ -62,41 +62,8 @@ int64_t Func_floor::getIntVal(Row& row, case execplan::CalpontSystemCatalog::MEDINT: case execplan::CalpontSystemCatalog::TINYINT: case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: { - if (op_ct.scale == 0) - { - ret = parm[0]->data()->getIntVal(row, isNull); - break; - } - - IDB_Decimal decimal = parm[0]->data()->getDecimalVal(row, isNull); - - if (isNull) - break; - - ret = decimal.value; - - // negative scale is not supported by CNX yet - if (decimal.scale > 0) - { - - if (decimal.scale >= 19) - { - std::ostringstream oss; - oss << "floor: datatype of " << execplan::colDataTypeToString(op_ct.colDataType) - << " with scale " << (int) decimal.scale << " is beyond supported scale"; - throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); - } - - int64_t tmp = ret; - ret /= helpers::powerOf10_c[decimal.scale]; - - // Largest integer value not greater than X. - if (tmp < 0 && tmp < ret) - ret -= 1; - } + ret = parm[0]->data()->getIntVal(row, isNull); } break; @@ -338,6 +305,20 @@ double Func_floor::getDoubleVal(Row& row, if (!isNull) ret = floor(strtod(str.c_str(), 0)); } + else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || + op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) + { + IDB_Decimal tmp = getDecimalVal(row, parm, isNull, op_ct); + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + ret = datatypes::Decimal::getDoubleFromWideDecimal(tmp.s128Value); + } + else + { + ret = (double) tmp.value; + } + } else { ret = (double) getIntVal(row, parm, isNull, op_ct); @@ -371,6 +352,20 @@ long double Func_floor::getLongDoubleVal(Row& row, if (!isNull) ret = floor(strtod(str.c_str(), 0)); } + else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || + op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) + { + IDB_Decimal tmp = getDecimalVal(row, parm, isNull, op_ct); + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + ret = datatypes::Decimal::getLongDoubleFromWideDecimal(tmp.s128Value); + } + else + { + ret = (long double) tmp.value; + } + } else { ret = (long double) getIntVal(row, parm, isNull, op_ct); @@ -416,6 +411,20 @@ string Func_floor::getStrVal(Row& row, *d = '\0'; } + else if (op_ct.colDataType == CalpontSystemCatalog::DECIMAL || + op_ct.colDataType == CalpontSystemCatalog::UDECIMAL) + { + IDB_Decimal d = getDecimalVal(row, parm, isNull, op_ct); + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + dataconvert::DataConvert::decimalToString(&d.s128Value, d.scale, tmp, 511, op_ct.colDataType); + } + else + { + dataconvert::DataConvert::decimalToString(d.value, d.scale, tmp, 511, op_ct.colDataType); + } + } else if (isUnsigned(op_ct.colDataType)) { #ifndef __LP64__ @@ -436,6 +445,165 @@ string Func_floor::getStrVal(Row& row, return string(tmp); } +IDB_Decimal Func_floor::getDecimalVal(Row& row, + FunctionParm& parm, + bool& isNull, + CalpontSystemCatalog::ColType& op_ct) +{ + IDB_Decimal ret; + + switch (op_ct.colDataType) + { + case execplan::CalpontSystemCatalog::BIGINT: + case execplan::CalpontSystemCatalog::INT: + case execplan::CalpontSystemCatalog::MEDINT: + case execplan::CalpontSystemCatalog::TINYINT: + case execplan::CalpontSystemCatalog::SMALLINT: + { + ret.value = parm[0]->data()->getIntVal(row, isNull); + } + break; + + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + ret = parm[0]->data()->getDecimalVal(row, isNull); + + if (isNull) + break; + + // negative scale is not supported by CNX yet + if (ret.scale > 0) + { + if (ret.scale > datatypes::INT128MAXPRECISION) + { + std::ostringstream oss; + oss << "floor: datatype of " << execplan::colDataTypeToString(op_ct.colDataType) + << " with scale " << (int) ret.scale << " is beyond supported scale"; + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t tmp = ret.s128Value; + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, ret.scale); + ret.s128Value /= scaleDivisor; + + // Largest integer value not greater than X. + if (tmp < 0 && tmp < ret.s128Value) + ret.s128Value -= 1; + } + else + { + int64_t tmp = ret.value; + ret.value /= helpers::powerOf10_c[ret.scale]; + + // Largest integer value not greater than X. + if (tmp < 0 && tmp < ret.value) + ret.value -= 1; + } + } + } + break; + + case execplan::CalpontSystemCatalog::UBIGINT: + case execplan::CalpontSystemCatalog::UINT: + case execplan::CalpontSystemCatalog::UMEDINT: + case execplan::CalpontSystemCatalog::UTINYINT: + case execplan::CalpontSystemCatalog::USMALLINT: + { + ret.value = (int64_t)parm[0]->data()->getUintVal(row, isNull); + } + break; + + case execplan::CalpontSystemCatalog::DOUBLE: + case execplan::CalpontSystemCatalog::UDOUBLE: + case execplan::CalpontSystemCatalog::FLOAT: + case execplan::CalpontSystemCatalog::UFLOAT: + { + ret.value = (int64_t) floor(parm[0]->data()->getDoubleVal(row, isNull)); + } + break; + + case execplan::CalpontSystemCatalog::LONGDOUBLE: + { + ret.value = (int64_t) floorl(parm[0]->data()->getLongDoubleVal(row, isNull)); + } + break; + + case execplan::CalpontSystemCatalog::VARCHAR: + case execplan::CalpontSystemCatalog::CHAR: + case execplan::CalpontSystemCatalog::TEXT: + { + const string& str = parm[0]->data()->getStrVal(row, isNull); + + if (!isNull) + ret.value = (int64_t) floor(strtod(str.c_str(), 0)); + } + break; + + case execplan::CalpontSystemCatalog::DATE: + { + string str = DataConvert::dateToString1(parm[0]->data()->getDateIntVal(row, isNull)); + + if (!isNull) + ret.value = atoll(str.c_str()); + } + break; + + case execplan::CalpontSystemCatalog::DATETIME: + { + string str = + DataConvert::datetimeToString1(parm[0]->data()->getDatetimeIntVal(row, isNull)); + + // strip off micro seconds + str = str.substr(0, 14); + + if (!isNull) + ret.value = atoll(str.c_str()); + } + break; + + case execplan::CalpontSystemCatalog::TIMESTAMP: + { + string str = + DataConvert::timestampToString1(parm[0]->data()->getTimestampIntVal(row, isNull), timeZone()); + + // strip off micro seconds + str = str.substr(0, 14); + + if (!isNull) + ret.value = atoll(str.c_str()); + } + break; + + case execplan::CalpontSystemCatalog::TIME: + { + string str = + DataConvert::timeToString1(parm[0]->data()->getTimeIntVal(row, isNull)); + + // strip off micro seconds + str = str.substr(0, 14); + + if (!isNull) + ret.value = atoll(str.c_str()); + } + break; + + default: + { + std::ostringstream oss; + oss << "floor: datatype of " << execplan::colDataTypeToString(op_ct.colDataType); + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + } + + ret.scale = 0; + + return ret; +} + } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_inet_aton.cpp b/utils/funcexp/func_inet_aton.cpp index 4bcdc3115..2f0e5295e 100644 --- a/utils/funcexp/func_inet_aton.cpp +++ b/utils/funcexp/func_inet_aton.cpp @@ -154,21 +154,34 @@ execplan::IDB_Decimal Func_inet_aton::getDecimalVal(rowgroup::Row& row, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { -// std::cout << "In Func_inet_aton::getDecimalVal" << std::endl; - - execplan::IDB_Decimal dValue ( joblist::NULL_INT64, 0, 0 ); + execplan::CalpontSystemCatalog::ColType colType = fp[0]->data()->resultType(); const std::string& sValue = fp[0]->data()->getStrVal(row, isNull); - if (!isNull) + if (colType.precision <= datatypes::INT64MAXPRECISION) { - int64_t iValue = convertAton( sValue, isNull ); - if (!isNull) - return execplan::IDB_Decimal( iValue, 0, 0 ); - } + { + int64_t iValue = convertAton( sValue, isNull ); - return dValue; + if (!isNull) + return execplan::IDB_Decimal( iValue, colType.scale, colType.precision ); + } + + return execplan::IDB_Decimal( joblist::NULL_INT64, colType.scale, colType.precision ); + } + else + { + if (!isNull) + { + int64_t iValue = convertAton( sValue, isNull ); + + if (!isNull) + return execplan::IDB_Decimal( 0, colType.scale, colType.precision, (int128_t) iValue ); + } + + return execplan::IDB_Decimal( 0, colType.scale, colType.precision, datatypes::Decimal128Null ); + } } //------------------------------------------------------------------------------ diff --git a/utils/funcexp/func_inet_ntoa.cpp b/utils/funcexp/func_inet_ntoa.cpp index b9a31b08e..0d33d5ec1 100644 --- a/utils/funcexp/func_inet_ntoa.cpp +++ b/utils/funcexp/func_inet_ntoa.cpp @@ -211,8 +211,6 @@ execplan::IDB_Decimal Func_inet_ntoa::getDecimalVal(rowgroup::Row& row, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct) { -// std::cout << "In Func_inet_ntoa::getDecimalVal" << std::endl; - // IDB_Decimal dValue = fp[0]->data()->getDecimalVal(row, isNull); execplan::IDB_Decimal dValue ( joblist::NULL_INT64, 0, 0 ); isNull = true; diff --git a/utils/funcexp/func_makedate.cpp b/utils/funcexp/func_makedate.cpp index 8c8c50abf..8efc3f171 100644 --- a/utils/funcexp/func_makedate.cpp +++ b/utils/funcexp/func_makedate.cpp @@ -68,15 +68,38 @@ uint64_t makedate(rowgroup::Row& row, case CalpontSystemCatalog::DECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - double dscale = d.scale; - year = d.value / pow(10.0, dscale); - int lefto = (d.value - year * pow(10.0, dscale)) / pow(10.0, dscale - 1); - if ( year >= 0 && lefto > 4 ) - year++; + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t scaleDivisor, scaleDivisor2; - if ( year < 0 && lefto < -4 ) - year--; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + + scaleDivisor2 = (scaleDivisor <= 10) ? 1 : (scaleDivisor / 10); + + int128_t tmpval = d.s128Value / scaleDivisor; + int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; + + if (tmpval >= 0 && lefto > 4) + tmpval++; + + if (tmpval < 0 && lefto < -4) + tmpval--; + + year = datatypes::Decimal::getInt64FromWideDecimal(tmpval); + } + else + { + double dscale = d.scale; + year = d.value / pow(10.0, dscale); + int lefto = (d.value - year * pow(10.0, dscale)) / pow(10.0, dscale - 1); + + if ( year >= 0 && lefto > 4 ) + year++; + + if ( year < 0 && lefto < -4 ) + year--; + } break; } @@ -128,23 +151,53 @@ uint64_t makedate(rowgroup::Row& row, case CalpontSystemCatalog::DECIMAL: { IDB_Decimal d = parm[1]->data()->getDecimalVal(row, isNull); - double dscale = d.scale; - int64_t tmp = d.value / pow(10.0, dscale); - int lefto = (d.value - tmp * pow(10.0, dscale)) / pow(10.0, dscale - 1); - if ( tmp >= 0 && lefto > 4 ) - tmp++; - - if ( tmp < 0 && lefto < -4 ) - tmp--; - - if (tmp < 1) + if (parm[1]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) { - isNull = true; - return 0; - } + int128_t scaleDivisor, scaleDivisor2; - dayofyear = helpers::intToString(tmp); + datatypes::getScaleDivisor(scaleDivisor, d.scale); + + scaleDivisor2 = (scaleDivisor <= 10) ? 1 : (scaleDivisor / 10); + + int128_t tmpval = d.s128Value / scaleDivisor; + int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; + + if (tmpval >= 0 && lefto > 4) + tmpval++; + + if (tmpval < 0 && lefto < -4) + tmpval--; + + if (tmpval < 1) + { + isNull = true; + return 0; + } + + int64_t tmpval64 = datatypes::Decimal::getInt64FromWideDecimal(tmpval); + dayofyear = helpers::intToString(tmpval64); + } + else + { + double dscale = d.scale; + int64_t tmp = d.value / pow(10.0, dscale); + int lefto = (d.value - tmp * pow(10.0, dscale)) / pow(10.0, dscale - 1); + + if (tmp >= 0 && lefto > 4) + tmp++; + + if (tmp < 0 && lefto < -4) + tmp--; + + if (tmp < 1) + { + isNull = true; + return 0; + } + + dayofyear = helpers::intToString(tmp); + } break; } diff --git a/utils/funcexp/func_maketime.cpp b/utils/funcexp/func_maketime.cpp index b43fed4ab..3bc7b8c27 100644 --- a/utils/funcexp/func_maketime.cpp +++ b/utils/funcexp/func_maketime.cpp @@ -74,15 +74,38 @@ string Func_maketime::getStrVal(rowgroup::Row& row, case CalpontSystemCatalog::DECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - double dscale = d.scale; - hour = d.value / pow(10.0, dscale); - int lefto = (d.value - hour * pow(10.0, dscale)) / pow(10.0, dscale - 1); - if ( hour >= 0 && lefto > 4 ) - hour++; + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t scaleDivisor, scaleDivisor2; - if ( hour < 0 && lefto < -4 ) - hour--; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + + scaleDivisor2 = (scaleDivisor <= 10) ? 1 : (scaleDivisor / 10); + + int128_t tmpval = d.s128Value / scaleDivisor; + int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; + + if (tmpval >= 0 && lefto > 4) + tmpval++; + + if (tmpval < 0 && lefto < -4) + tmpval--; + + hour = datatypes::Decimal::getInt64FromWideDecimal(tmpval); + } + else + { + double dscale = d.scale; + hour = d.value / pow(10.0, dscale); + int lefto = (d.value - hour * pow(10.0, dscale)) / pow(10.0, dscale - 1); + + if ( hour >= 0 && lefto > 4 ) + hour++; + + if ( hour < 0 && lefto < -4 ) + hour--; + } break; } @@ -114,15 +137,38 @@ string Func_maketime::getStrVal(rowgroup::Row& row, case CalpontSystemCatalog::DECIMAL: { IDB_Decimal d = parm[1]->data()->getDecimalVal(row, isNull); - double dscale = d.scale; - min = d.value / pow(10.0, dscale); - int lefto = (d.value - min * pow(10.0, dscale)) / pow(10.0, dscale - 1); - if ( min >= 0 && lefto > 4 ) - min++; + if (parm[1]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t scaleDivisor, scaleDivisor2; - if ( min < 0 && lefto < -4 ) - min--; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + + scaleDivisor2 = (scaleDivisor <= 10) ? 1 : (scaleDivisor / 10); + + int128_t tmpval = d.s128Value / scaleDivisor; + int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; + + if (tmpval >= 0 && lefto > 4) + tmpval++; + + if (tmpval < 0 && lefto < -4) + tmpval--; + + min = datatypes::Decimal::getInt64FromWideDecimal(tmpval); + } + else + { + double dscale = d.scale; + min = d.value / pow(10.0, dscale); + int lefto = (d.value - min * pow(10.0, dscale)) / pow(10.0, dscale - 1); + + if ( min >= 0 && lefto > 4 ) + min++; + + if ( min < 0 && lefto < -4 ) + min--; + } break; } @@ -160,15 +206,38 @@ string Func_maketime::getStrVal(rowgroup::Row& row, case CalpontSystemCatalog::DECIMAL: { IDB_Decimal d = parm[2]->data()->getDecimalVal(row, isNull); - double dscale = d.scale; - sec = d.value / pow(10.0, dscale); - int lefto = (d.value - sec * pow(10.0, dscale)) / pow(10.0, dscale - 1); - if ( sec >= 0 && lefto > 4 ) - sec++; + if (parm[2]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t scaleDivisor, scaleDivisor2; - if ( sec < 0 && lefto < -4 ) - sec--; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + + scaleDivisor2 = (scaleDivisor <= 10) ? 1 : (scaleDivisor / 10); + + int128_t tmpval = d.s128Value / scaleDivisor; + int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; + + if (tmpval >= 0 && lefto > 4) + tmpval++; + + if (tmpval < 0 && lefto < -4) + tmpval--; + + sec = datatypes::Decimal::getInt64FromWideDecimal(tmpval); + } + else + { + double dscale = d.scale; + sec = d.value / pow(10.0, dscale); + int lefto = (d.value - sec * pow(10.0, dscale)) / pow(10.0, dscale - 1); + + if ( sec >= 0 && lefto > 4 ) + sec++; + + if ( sec < 0 && lefto < -4 ) + sec--; + } break; } diff --git a/utils/funcexp/func_mod.cpp b/utils/funcexp/func_mod.cpp index 214fd2b6c..1afc308f5 100644 --- a/utils/funcexp/func_mod.cpp +++ b/utils/funcexp/func_mod.cpp @@ -25,6 +25,8 @@ #include using namespace std; +#include + #include "functor_real.h" #include "funchelpers.h" #include "functioncolumn.h" @@ -54,7 +56,6 @@ IDB_Decimal Func_mod::getDecimalVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType& operationColType) { - IDB_Decimal retValue; retValue.value = 0; retValue.scale = 0; @@ -65,22 +66,85 @@ IDB_Decimal Func_mod::getDecimalVal(Row& row, return retValue; } - int64_t div = parm[1]->data()->getIntVal(row, isNull); - - if ( div == 0 ) + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH || + parm[1]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) { - isNull = true; - return retValue; + IDB_Decimal div = parm[1]->data()->getDecimalVal(row, isNull); + + int128_t divInt, dividendInt; + + if (parm[1]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + divInt = div.s128Value; + else + divInt = div.value; + + if (divInt == 0) + { + isNull = true; + return retValue; + } + + IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); + + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + dividendInt = d.s128Value; + else + dividendInt = d.value; + + // integer division + if (d.scale == 0 && div.scale == 0) + { + retValue.s128Value = dividendInt % divInt; + } + // special case integer division + else if (div.scale == 0) + { + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + int128_t value = dividendInt / scaleDivisor; + int128_t lefto = dividendInt % scaleDivisor; + int128_t mod = (value % divInt) * scaleDivisor + lefto; + retValue.s128Value = mod; + } + // float division + else + { + __float128 divF, dividendF; + + int128_t scaleDivisor; + + datatypes::getScaleDivisor(scaleDivisor, div.scale); + divF = (__float128) divInt / scaleDivisor; + + datatypes::getScaleDivisor(scaleDivisor, d.scale); + dividendF = (__float128) dividendInt / scaleDivisor; + + __float128 mod = fmodq(dividendF, divF) * scaleDivisor; + + retValue.s128Value = (int128_t) mod; + } + retValue.scale = d.scale; + retValue.precision = datatypes::INT128MAXPRECISION; } + else + { + int64_t div = parm[1]->data()->getIntVal(row, isNull); - IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - int64_t value = d.value / pow(10.0, d.scale); - int lefto = d.value % (int)pow(10.0, d.scale); + if ( div == 0 ) + { + isNull = true; + return retValue; + } - int64_t mod = (value % div) * pow(10.0, d.scale) + lefto; + IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); + int64_t value = d.value / pow(10.0, d.scale); + int lefto = d.value % (int)pow(10.0, d.scale); - retValue.value = mod; - retValue.scale = d.scale; + int64_t mod = (value % div) * pow(10.0, d.scale) + lefto; + + retValue.value = mod; + retValue.scale = d.scale; + } return retValue; } @@ -164,9 +228,28 @@ double Func_mod::getDoubleVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - int64_t value = d.value / pow(10.0, d.scale); - mod = value % div; + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + if (d.scale == 0) + { + mod = d.s128Value % div; + } + else + { + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + int128_t value = d.s128Value / scaleDivisor; + int128_t lefto = d.s128Value % scaleDivisor; + __float128 tmp = (__float128) (value % div) + (__float128) lefto / scaleDivisor; + mod = datatypes::Decimal::getDoubleFromFloat128(tmp); + } + } + else + { + int64_t value = d.value / pow(10.0, d.scale); + mod = value % div; + } } break; @@ -268,9 +351,28 @@ long double Func_mod::getLongDoubleVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - int64_t value = d.value / pow(10.0, d.scale); - mod = value % div; + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + if (d.scale == 0) + { + mod = d.s128Value % div; + } + else + { + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + int128_t value = d.s128Value / scaleDivisor; + int128_t lefto = d.s128Value % scaleDivisor; + __float128 tmp = (__float128) (value % div) + (__float128) lefto / scaleDivisor; + mod = datatypes::Decimal::getLongDoubleFromFloat128(tmp); + } + } + else + { + int64_t value = d.value / pow(10.0, d.scale); + mod = value % div; + } } break; @@ -375,9 +477,28 @@ int64_t Func_mod::getIntVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - int64_t value = d.value / pow(10.0, d.scale); - mod = value % div; + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + if (d.scale == 0) + { + mod = d.s128Value % div; + } + else + { + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + int128_t value = d.s128Value / scaleDivisor; + int128_t lefto = d.s128Value % scaleDivisor; + __float128 tmp = (__float128) (value % div) + (__float128) lefto / scaleDivisor; + mod = datatypes::Decimal::getInt64FromFloat128(tmp); + } + } + else + { + int64_t value = d.value / pow(10.0, d.scale); + mod = value % div; + } } break; @@ -473,9 +594,28 @@ uint64_t Func_mod::getUIntVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - int64_t value = d.value / pow(10.0, d.scale); - mod = value % div; + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + if (d.scale == 0) + { + mod = d.s128Value % div; + } + else + { + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + int128_t value = d.s128Value / scaleDivisor; + int128_t lefto = d.s128Value % scaleDivisor; + __float128 tmp = (__float128) (value % div) + (__float128) lefto / scaleDivisor; + mod = datatypes::Decimal::getUInt64FromFloat128(tmp); + } + } + else + { + int64_t value = d.value / pow(10.0, d.scale); + mod = value % div; + } } break; diff --git a/utils/funcexp/func_monthname.cpp b/utils/funcexp/func_monthname.cpp index 7c27c9581..f04c37158 100644 --- a/utils/funcexp/func_monthname.cpp +++ b/utils/funcexp/func_monthname.cpp @@ -197,7 +197,11 @@ execplan::IDB_Decimal Func_monthname::getDecimalVal(rowgroup::Row& row, execplan::CalpontSystemCatalog::ColType& op_ct) { IDB_Decimal d; - d.value = getIntVal(row, fp, isNull, op_ct); + + if (fp[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + d.s128Value = getIntVal(row, fp, isNull, op_ct); + else + d.value = getIntVal(row, fp, isNull, op_ct); d.scale = 0; return d; } diff --git a/utils/funcexp/func_period_diff.cpp b/utils/funcexp/func_period_diff.cpp index 0f1b923c9..3f344fd8f 100644 --- a/utils/funcexp/func_period_diff.cpp +++ b/utils/funcexp/func_period_diff.cpp @@ -86,7 +86,24 @@ int64_t Func_period_diff::getIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - period1 = d.value / pow(10.0, d.scale); + + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + int128_t tmpval = d.s128Value / scaleDivisor; + + if (tmpval > static_cast(INT64_MAX)) + tmpval = INT64_MAX; + else if (tmpval < static_cast(INT64_MIN)) + tmpval = INT64_MIN; + + period1 = tmpval; + } + else + { + period1 = d.value / pow(10.0, d.scale); + } break; } @@ -135,7 +152,24 @@ int64_t Func_period_diff::getIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[1]->data()->getDecimalVal(row, isNull); - period2 = d.value / pow(10.0, d.scale); + + if (parm[1]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + int128_t tmpval = d.s128Value / scaleDivisor; + + if (tmpval > static_cast(INT64_MAX)) + tmpval = INT64_MAX; + else if (tmpval < static_cast(INT64_MIN)) + tmpval = INT64_MIN; + + period2 = tmpval; + } + else + { + period2 = d.value / pow(10.0, d.scale); + } break; } diff --git a/utils/funcexp/func_round.cpp b/utils/funcexp/func_round.cpp index 0064ae8df..780b499b0 100644 --- a/utils/funcexp/func_round.cpp +++ b/utils/funcexp/func_round.cpp @@ -116,18 +116,36 @@ int64_t Func_round::getIntVal(Row& row, { IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); - if (x.scale > 0) + if (!datatypes::Decimal::isWideDecimalType(op_ct)) { - while (x.scale-- > 0) - x.value /= 10; + if (x.scale > 0) + { + while (x.scale-- > 0) + x.value /= 10; + } + else + { + while (x.scale++ < 0) + x.value *= 10; + } + + return x.value; } else { - while (x.scale++ < 0) - x.value *= 10; - } + if (x.scale > 0) + { + while (x.scale-- > 0) + x.s128Value /= 10; + } + else + { + while (x.scale++ < 0) + x.s128Value *= 10; + } - return x.value; + return datatypes::Decimal::getInt64FromWideDecimal(x.s128Value); + } } @@ -187,7 +205,12 @@ double Func_round::getDoubleVal(Row& row, if (isNull) return 0.0; - double d = x.value; + double d; + + if (!datatypes::Decimal::isWideDecimalType(op_ct)) + d = x.value; + else + d = datatypes::Decimal::getDoubleFromWideDecimal(x.s128Value); if (x.scale > 0) { @@ -249,7 +272,12 @@ long double Func_round::getLongDoubleVal(Row& row, if (isNull) return 0.0; - double d = x.value; + double d; + + if (!datatypes::Decimal::isWideDecimalType(op_ct)) + d = x.value; + else + d = datatypes::Decimal::getDoubleFromWideDecimal(x.s128Value); if (x.scale > 0) { @@ -283,62 +311,130 @@ IDB_Decimal Func_round::getDecimalVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { int64_t d = 0; - //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. - volatile int64_t p = 1; decimal = parm[0]->data()->getDecimalVal(row, isNull); - if (!isNull && parm.size() > 1) // round(X, D) + if (!datatypes::Decimal::isWideDecimalType(op_ct)) { - int64_t nvp = p; - d = parm[1]->data()->getIntVal(row, isNull); + //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. + volatile int64_t p = 1; - if (!isNull) - helpers::decimalPlaceDec(d, nvp, decimal.scale); - - p = nvp; - } - - if (isNull) - break; - - int64_t x = decimal.value; - - if (d > 0) - { - x = x * p; - } - else if (d < 0) - { - int64_t h = p / 2; // 0.5 - - if ((x >= h) || (x <= -h)) + if (!isNull && parm.size() > 1) // round(X, D) { - if (x >= 0) - x += h; - else - x -= h; + int64_t nvp = p; + d = parm[1]->data()->getIntVal(row, isNull); - if (p != 0) - x = x / p; + if (!isNull) + helpers::decimalPlaceDec(d, nvp, decimal.scale); + + p = nvp; + } + + if (isNull) + break; + + int64_t x = decimal.value; + + if (d > 0) + { + x = x * p; + } + else if (d < 0) + { + int64_t h = p / 2; // 0.5 + + if ((x >= h) || (x <= -h)) + { + if (x >= 0) + x += h; + else + x -= h; + + if (p != 0) + x = x / p; + else + x = 0; + } else + { x = 0; + } } - else + + // negative scale is not supported by CNX yet, set d to 0. + if (decimal.scale < 0) { - x = 0; + do + x *= 10; + + while (++decimal.scale < 0); } - } - // negative scale is not supported by CNX yet, set d to 0. - if (decimal.scale < 0) + decimal.value = x; + } + else { - do - x *= 10; + //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. + volatile int128_t p = 1; - while (++decimal.scale < 0); + if (!isNull && parm.size() > 1) // round(X, D) + { + int128_t nvp = p; + d = parm[1]->data()->getIntVal(row, isNull); + + if (!isNull) + helpers::decimalPlaceDec(d, nvp, decimal.scale); + + p = nvp; + } + + if (isNull) + break; + + if (d < -datatypes::INT128MAXPRECISION) + { + decimal.s128Value = 0; + break; + } + + int128_t x = decimal.s128Value; + + if (d > 0) + { + x = x * p; + } + else if (d < 0) + { + int128_t h = p / 2; // 0.5 + + if ((x >= h) || (x <= -h)) + { + if (x >= 0) + x += h; + else + x -= h; + + if (p != 0) + x = x / p; + else + x = 0; + } + else + { + x = 0; + } + } + + // negative scale is not supported by CNX yet, set d to 0. + if (decimal.scale < 0) + { + do + x *= 10; + + while (++decimal.scale < 0); + } + + decimal.s128Value = x; } - - decimal.value = x; } break; @@ -624,7 +720,14 @@ string Func_round::getStrVal(Row& row, break; } - return dataconvert::DataConvert::decimalToString(x.value, x.scale, op_ct.colDataType); + if (!datatypes::Decimal::isWideDecimalType(op_ct)) + return dataconvert::DataConvert::decimalToString(x.value, x.scale, op_ct.colDataType); + else + { + char buf[utils::MAXLENGTH16BYTES]; + dataconvert::DataConvert::decimalToString( &x.s128Value, x.scale, buf, utils::MAXLENGTH16BYTES, op_ct.colDataType); + return string(buf); + } } diff --git a/utils/funcexp/func_sec_to_time.cpp b/utils/funcexp/func_sec_to_time.cpp index 3fc94b5ef..afb532ec1 100644 --- a/utils/funcexp/func_sec_to_time.cpp +++ b/utils/funcexp/func_sec_to_time.cpp @@ -257,12 +257,15 @@ execplan::IDB_Decimal Func_sec_to_time::getDecimalVal(rowgroup::Row& row, execplan::CalpontSystemCatalog::ColType& op_ct) { IDB_Decimal d; + int64_t val = parm[0]->data()->getIntVal(row, isNull); + int64_t tmpVal; + if (val > 3020399) - d.value = 8385959; + tmpVal = 8385959; else if (val < -3020399) - d.value = 4286581337LL; + tmpVal = 4286581337LL; else { string time = getStrVal(row, parm, isNull, op_ct); @@ -277,15 +280,17 @@ execplan::IDB_Decimal Func_sec_to_time::getDecimalVal(rowgroup::Row& row, char* ep = NULL; const char* str = time.c_str(); errno = 0; - d.value = strtoll(str, &ep, 10); + tmpVal = strtoll(str, &ep, 10); } + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + d.s128Value = tmpVal; + else + d.value = tmpVal; + d.scale = 0; return d; } - - - } // namespace funcexp // vim:ts=4 sw=4: diff --git a/utils/funcexp/func_truncate.cpp b/utils/funcexp/func_truncate.cpp index 057a1c299..d60fa6e70 100644 --- a/utils/funcexp/func_truncate.cpp +++ b/utils/funcexp/func_truncate.cpp @@ -119,18 +119,36 @@ int64_t Func_truncate::getIntVal(Row& row, { IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); - if (x.scale > 0) + if (!datatypes::Decimal::isWideDecimalType(op_ct)) { - while (x.scale-- > 0) - x.value /= 10; + if (x.scale > 0) + { + while (x.scale-- > 0) + x.value /= 10; + } + else + { + while (x.scale++ < 0) + x.value *= 10; + } + + return x.value; } else { - while (x.scale++ < 0) - x.value *= 10; - } + if (x.scale > 0) + { + while (x.scale-- > 0) + x.s128Value /= 10; + } + else + { + while (x.scale++ < 0) + x.s128Value *= 10; + } - return x.value; + return datatypes::Decimal::getInt64FromWideDecimal(x.s128Value); + } } @@ -210,7 +228,12 @@ double Func_truncate::getDoubleVal(Row& row, if (isNull) return 0.0; - double d = x.value; + double d; + + if (!datatypes::Decimal::isWideDecimalType(op_ct)) + d = x.value; + else + d = datatypes::Decimal::getDoubleFromWideDecimal(x.s128Value); if (x.scale > 0) { @@ -265,7 +288,12 @@ long double Func_truncate::getLongDoubleVal(Row& row, if (isNull) return 0.0; - double d = x.value; + double d; + + if (!datatypes::Decimal::isWideDecimalType(op_ct)) + d = x.value; + else + d = datatypes::Decimal::getDoubleFromWideDecimal(x.s128Value); if (x.scale > 0) { @@ -304,55 +332,116 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, case execplan::CalpontSystemCatalog::UDECIMAL: { int64_t d = 0; - //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. - volatile int64_t p = 1; decimal = parm[0]->data()->getDecimalVal(row, isNull); - if (!isNull) + if (!datatypes::Decimal::isWideDecimalType(op_ct)) { - int64_t nvp = p; - d = parm[1]->data()->getIntVal(row, isNull); + //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. + volatile int64_t p = 1; if (!isNull) - helpers::decimalPlaceDec(d, nvp, decimal.scale); - - p = nvp; - } - - if (isNull) - break; - - int64_t x = decimal.value; - - if (d > 0) - { - x = x * p; - } - else if (d < 0) - { - if ((x >= p) || (x <= -p)) { - if (p != 0) - x = x / p; + int64_t nvp = p; + d = parm[1]->data()->getIntVal(row, isNull); + + if (!isNull) + helpers::decimalPlaceDec(d, nvp, decimal.scale); + + p = nvp; + } + + if (isNull) + break; + + int64_t x = decimal.value; + + if (d > 0) + { + x = x * p; + } + else if (d < 0) + { + if ((x >= p) || (x <= -p)) + { + if (p != 0) + x = x / p; + else + x = 0; + } else + { x = 0; + } } - else + + // negative scale is not supported by CNX yet, set d to 0. + if (decimal.scale < 0) { - x = 0; + do + x *= 10; + + while (++decimal.scale < 0); } - } - // negative scale is not supported by CNX yet, set d to 0. - if (decimal.scale < 0) + decimal.value = x; + } + else { - do - x *= 10; + //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. + volatile int128_t p = 1; - while (++decimal.scale < 0); + if (!isNull) + { + int128_t nvp = p; + d = parm[1]->data()->getIntVal(row, isNull); + + if (!isNull) + helpers::decimalPlaceDec(d, nvp, decimal.scale); + + p = nvp; + } + + if (isNull) + break; + + if (d < -datatypes::INT128MAXPRECISION) + { + decimal.s128Value = 0; + break; + } + + int128_t x = decimal.s128Value; + + if (d > 0) + { + x = x * p; + } + else if (d < 0) + { + if ((x >= p) || (x <= -p)) + { + if (p != 0) + x = x / p; + else + x = 0; + } + else + { + x = 0; + } + } + + // negative scale is not supported by CNX yet, set d to 0. + if (decimal.scale < 0) + { + do + x *= 10; + + while (++decimal.scale < 0); + } + + decimal.s128Value = x; } - - decimal.value = x; } break; @@ -650,7 +739,14 @@ string Func_truncate::getStrVal(Row& row, break; } - return dataconvert::DataConvert::decimalToString(x.value, x.scale, op_ct.colDataType); + if (!datatypes::Decimal::isWideDecimalType(op_ct)) + return dataconvert::DataConvert::decimalToString(x.value, x.scale, op_ct.colDataType); + else + { + char buf[utils::MAXLENGTH16BYTES]; + dataconvert::DataConvert::decimalToString( &x.s128Value, x.scale, buf, utils::MAXLENGTH16BYTES, op_ct.colDataType); + return string(buf); + } } diff --git a/utils/funcexp/funchelpers.h b/utils/funcexp/funchelpers.h index 8f017d544..709d441bc 100644 --- a/utils/funcexp/funchelpers.h +++ b/utils/funcexp/funchelpers.h @@ -470,7 +470,8 @@ inline int power ( int16_t a ) return b; } -inline void decimalPlaceDec(int64_t& d, int64_t& p, int8_t& s) +template +inline void decimalPlaceDec(int64_t& d, T& p, int8_t& s) { // find new scale if D < s if (d < s) diff --git a/utils/funcexp/functor_real.h b/utils/funcexp/functor_real.h index 0f5137891..d31a55686 100644 --- a/utils/funcexp/functor_real.h +++ b/utils/funcexp/functor_real.h @@ -281,6 +281,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; @@ -318,6 +323,11 @@ public: FunctionParm& fp, bool& isNull, execplan::CalpontSystemCatalog::ColType& op_ct); + + execplan::IDB_Decimal getDecimalVal(rowgroup::Row& row, + FunctionParm& fp, + bool& isNull, + execplan::CalpontSystemCatalog::ColType& op_ct); }; diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index edd552c93..17d955a1f 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -1947,6 +1947,10 @@ inline void copyRow(const Row& in, Row* out, uint32_t colCount) out->setUintField(in.getUintField(i), i); else if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::LONGDOUBLE)) out->setLongDoubleField(in.getLongDoubleField(i), i); + else if (UNLIKELY((in.getColType(i) == execplan::CalpontSystemCatalog::DECIMAL || + in.getColType(i) == execplan::CalpontSystemCatalog::UDECIMAL) && + in.getColumnWidth(i) == datatypes::MAXDECIMALWIDTH)) + in.copyBinaryField(*out, i, i); else out->setIntField(in.getIntField(i), i); } From 3d94ec156896f07b0c4c85ba1865f445fe8b6b08 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Fri, 8 May 2020 09:50:15 +0000 Subject: [PATCH 35/78] MCOL-641 Followup on functions commit. --- dbcon/mysql/ha_mcs_impl.cpp | 1 - utils/funcexp/func_ceil.cpp | 4 ++-- utils/funcexp/func_floor.cpp | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index f80f9738b..f334a3375 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -820,7 +820,6 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h dataconvert::DataConvert::decimalToString(dec, (unsigned)colType.scale, buf, sizeof(buf), colType.colDataType); - std::cout << buf << std::endl; Field_new_decimal* f2 = (Field_new_decimal*)*f; f2->store(buf, strlen(buf), f2->charset()); diff --git a/utils/funcexp/func_ceil.cpp b/utils/funcexp/func_ceil.cpp index 26200a94f..5714b2349 100644 --- a/utils/funcexp/func_ceil.cpp +++ b/utils/funcexp/func_ceil.cpp @@ -70,8 +70,8 @@ int64_t Func_ceil::getIntVal(Row& row, } break; - // ceil(decimal(38,38)) leads to this path - // otherwise Func_ceil::getDecimalVal() is called + // ceil(decimal(X,Y)) leads to this path if X, Y allows to + // downcast to INT otherwise Func_ceil::getDecimalVal() is called case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { diff --git a/utils/funcexp/func_floor.cpp b/utils/funcexp/func_floor.cpp index 5116a9a54..a8afa77f1 100644 --- a/utils/funcexp/func_floor.cpp +++ b/utils/funcexp/func_floor.cpp @@ -151,6 +151,24 @@ int64_t Func_floor::getIntVal(Row& row, } break; + // floor(decimal(X,Y)) leads to this path if X, Y allows to + // downcast to INT otherwise Func_floor::getDecimalVal() is called + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + IDB_Decimal tmp = getDecimalVal(row, parm, isNull, op_ct); + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + ret = datatypes::Decimal::getInt64FromWideDecimal(tmp.s128Value); + } + else + { + ret = tmp.value; + } + break; + } + default: { std::ostringstream oss; From e88cbe9bc1350bf66101c5c51e944a7caabb6a39 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Fri, 8 May 2020 10:17:17 +0000 Subject: [PATCH 36/78] MCOL-641 Simple aggregates support: min, max, sum, avg for wide-DECIMALs. --- datatypes/mcs_decimal.cpp | 5 + datatypes/mcs_decimal.h | 25 ++- dbcon/joblist/tupleaggregatestep.cpp | 134 +++++++----- dbcon/mysql/ha_mcs_execplan.cpp | 63 +++--- dbcon/mysql/ha_mcs_partition.cpp | 2 +- utils/joiner/tuplejoiner.cpp | 2 +- utils/rowgroup/rowaggregation.cpp | 304 ++++++++++++++++++--------- utils/rowgroup/rowaggregation.h | 1 + utils/rowgroup/rowgroup.h | 19 +- 9 files changed, 343 insertions(+), 212 deletions(-) diff --git a/datatypes/mcs_decimal.cpp b/datatypes/mcs_decimal.cpp index 325f67721..c5b61b5fa 100644 --- a/datatypes/mcs_decimal.cpp +++ b/datatypes/mcs_decimal.cpp @@ -203,6 +203,11 @@ namespace datatypes return std::string(buf); } + std::string Decimal::toString(const execplan::IDB_Decimal& value) + { + return toString(const_cast(value)); + } + int Decimal::compare(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r) { int128_t divisorL, divisorR; diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index cf70a357d..346dd2dda 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -24,6 +24,7 @@ #include "calpontsystemcatalog.h" using int128_t = __int128; +using ColTypeAlias = execplan::CalpontSystemCatalog::ColType; namespace execplan { @@ -37,6 +38,8 @@ constexpr uint32_t MAXDECIMALWIDTH = 16U; constexpr uint8_t INT64MAXPRECISION = 18U; constexpr uint8_t INT128MAXPRECISION = 38U; constexpr uint8_t MAXLEGACYWIDTH = 8U; +constexpr uint8_t MAXSCALEINC4AVG = 4U; +constexpr int8_t IGNOREPRECISION = -1; const uint64_t mcs_pow_10[20] = { @@ -159,12 +162,13 @@ class Decimal @brief Convenience method to put decimal into a std::string. */ static std::string toString(execplan::IDB_Decimal& value); + static std::string toString(const execplan::IDB_Decimal& value); /** @brief The method detects whether decimal type is wide using csc data type. */ - static constexpr inline bool isWideDecimalType(const execplan::CalpontSystemCatalog::ColType& ct) + static constexpr inline bool isWideDecimalType(const ColTypeAlias& ct) { return ((ct.colDataType == execplan::CalpontSystemCatalog::DECIMAL || ct.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && @@ -185,7 +189,7 @@ class Decimal @brief The method sets the legacy scale and precision of a wide decimal column which is the result of an arithmetic operation. */ - static inline void setDecimalScalePrecisionLegacy(execplan::CalpontSystemCatalog::ColType& ct, + static inline void setDecimalScalePrecisionLegacy(ColTypeAlias& ct, unsigned int precision, unsigned int scale) { ct.scale = scale; @@ -200,7 +204,7 @@ class Decimal @brief The method sets the scale and precision of a wide decimal column which is the result of an arithmetic operation. */ - static inline void setDecimalScalePrecision(execplan::CalpontSystemCatalog::ColType& ct, + static inline void setDecimalScalePrecision(ColTypeAlias& ct, unsigned int precision, unsigned int scale) { ct.colWidth = (precision > INT64MAXPRECISION) @@ -216,7 +220,7 @@ class Decimal @brief The method sets the scale and precision of a wide decimal column which is the result of an arithmetic operation, based on a heuristic. */ - static inline void setDecimalScalePrecisionHeuristic(execplan::CalpontSystemCatalog::ColType& ct, + static inline void setDecimalScalePrecisionHeuristic(ColTypeAlias& ct, unsigned int precision, unsigned int scale) { unsigned int diff = 0; @@ -337,6 +341,19 @@ class Decimal return static_cast(value); } + + /** + @brief MDB increases scale by up to 4 digits calculating avg() + */ + static inline void setScalePrecision4Avg( + unsigned int& precision, + unsigned int& scale) + { + uint32_t scaleAvailable = INT128MAXPRECISION - scale; + uint32_t precisionAvailable = INT128MAXPRECISION - precision; + scale += (scaleAvailable >= MAXSCALEINC4AVG) ? MAXSCALEINC4AVG : scaleAvailable; + precision += (precisionAvailable >= MAXSCALEINC4AVG) ? MAXSCALEINC4AVG : precisionAvailable; + } }; /** diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index 789932ff7..6a0b0fe11 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -343,6 +343,41 @@ string keyName(uint64_t i, uint32_t key, const joblist::JobInfo& jobInfo) namespace joblist { +void wideDecimalOrLongDouble(const uint64_t colProj, + const CalpontSystemCatalog::ColDataType type, + const vector& precisionProj, + const vector& oidsProj, + const uint32_t aggKey, + const vector& scaleProj, + const vector& width, + vector& oidsAgg, + vector& keysAgg, + vector& typeAgg, + vector& scaleAgg, + vector& precisionAgg, + vector& widthAgg) +{ + if ((type == CalpontSystemCatalog::DECIMAL + || type == CalpontSystemCatalog::UDECIMAL) + && datatypes::Decimal::isWideDecimalType(precisionProj[colProj])) + { + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(aggKey); + typeAgg.push_back(type); + scaleAgg.push_back(scaleProj[colProj]); + precisionAgg.push_back(precisionProj[colProj]); + widthAgg.push_back(width[colProj]); + } + else + { + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(aggKey); + typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); + scaleAgg.push_back(0); + precisionAgg.push_back(-1); + widthAgg.push_back(sizeof(long double)); + } +} TupleAggregateStep::TupleAggregateStep( const SP_ROWAGG_UM_t& agg, @@ -717,25 +752,47 @@ void TupleAggregateStep::configDeliveredRowGroup(const JobInfo& jobInfo) // correct the scale vector scale = fRowGroupOut.getScale(); + vector precision = fRowGroupOut.getPrecision(); -// for (uint64_t i = 0; i < scale.size(); i++) -// { - // to support CNX_DECIMAL_SCALE the avg column's scale is coded with two scales: - // fe's avg column scale << 8 + original column scale - //if ((scale[i] & 0x0000FF00) > 0) -// scale[i] = scale[i] & 0x000000FF; -// } - - size_t retColCount = jobInfo.nonConstDelCols.size(); + size_t retColCount = 0; + auto scaleIter = scale.begin(); + auto precisionIter = precision.begin(); if (jobInfo.havingStep) + { retColCount = jobInfo.returnedColVec.size(); + idbassert(jobInfo.returnedColVec.size() == jobInfo.nonConstCols.size()); + for (auto& rc : jobInfo.nonConstCols) + { + auto& colType = rc->resultType(); + if (datatypes::Decimal::isWideDecimalType(colType)) + { + *scaleIter = colType.scale; + *precisionIter = colType.precision; + } + scaleIter++; precisionIter++; + } + } + else + { + retColCount = jobInfo.nonConstDelCols.size(); + for (auto& rc : jobInfo.nonConstDelCols) + { + auto& colType = rc->resultType(); + if (datatypes::Decimal::isWideDecimalType(colType)) + { + *scaleIter = colType.scale; + *precisionIter = colType.precision; + } + scaleIter++; precisionIter++; + } + } vector::const_iterator offsets0 = fRowGroupOut.getOffsets().begin(); vector::const_iterator types0 = fRowGroupOut.getColTypes().begin(); vector csNums = fRowGroupOut.getCharsetNumbers(); - vector::const_iterator precision0 = fRowGroupOut.getPrecision().begin(); + vector::const_iterator precision0 = precision.begin(); fRowGroupDelivered = RowGroup(retColCount, vector(offsets0, offsets0 + retColCount + 1), vector(oids.begin(), oids.begin() + retColCount), @@ -896,7 +953,6 @@ SJSTEP TupleAggregateStep::prepAggregate(SJSTEP& step, JobInfo& jobInfo) // preprocess the columns used by group_concat jobInfo.groupConcatInfo.prepGroupConcat(jobInfo); bool doUMOnly = jobInfo.groupConcatInfo.columns().size() > 0 -// || jobInfo.windowSet.size() > 0 || sas || ces; @@ -1303,14 +1359,11 @@ void TupleAggregateStep::prep1PhaseAggregate( cerr << "prep1PhaseAggregate: " << emsg << endl; throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); } - - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(key); - typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); + wideDecimalOrLongDouble(colProj, typeProj[colProj], + precisionProj, oidsProj, key, scaleProj, width, + oidsAgg, keysAgg, typeAgg, scaleAgg, + precisionAgg, widthAgg); csNumAgg.push_back(csNumProj[colProj]); - precisionAgg.push_back(-1); - widthAgg.push_back(sizeof(long double)); - scaleAgg.push_back(0); } break; @@ -1755,11 +1808,6 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( throw logic_error(emsg.str()); } - // skip sum / count(column) if avg is also selected -// if ((aggOp == ROWAGG_SUM || aggOp == ROWAGG_COUNT_COL_NAME) && -// (avgSet.find(aggKey) != avgSet.end())) -// continue; - if (aggOp == ROWAGG_DISTINCT_SUM || aggOp == ROWAGG_DISTINCT_AVG || aggOp == ROWAGG_COUNT_DISTINCT_COL_NAME) @@ -3128,31 +3176,11 @@ void TupleAggregateStep::prep2PhasesAggregate( cerr << "prep2PhasesAggregate: " << emsg << endl; throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); } - - // WIP MCOL-641 Replace condition with a - // dynamic one - if (typeProj[colProj] == CalpontSystemCatalog::DECIMAL - && width[colProj] == 16) - { - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - typeAggPm.push_back(CalpontSystemCatalog::DECIMAL); - scaleAggPm.push_back(0); - // WIP makes this dynamic - precisionAggPm.push_back(38); - widthAggPm.push_back(width[colProj]); - csNumAggPm.push_back(8); - } - else - { - oidsAggPm.push_back(oidsProj[colProj]); - keysAggPm.push_back(aggKey); - typeAggPm.push_back(CalpontSystemCatalog::LONGDOUBLE); - scaleAggPm.push_back(0); - csNumAggPm.push_back(8); - precisionAggPm.push_back(-1); - widthAggPm.push_back(sizeof(long double)); - } + wideDecimalOrLongDouble(colProj, typeProj[colProj], + precisionProj, oidsProj, aggKey, scaleProj, width, + oidsAggPm, keysAggPm, typeAggPm, scaleAggPm, + precisionAggPm, widthAggPm); + scaleAggPm.push_back(0); colAggPm++; } @@ -3435,13 +3463,11 @@ void TupleAggregateStep::prep2PhasesAggregate( if (aggOp == ROWAGG_SUM) { - oidsAggUm.push_back(oidsAggPm[colPm]); - keysAggUm.push_back(retKey); - scaleAggUm.push_back(0); - typeAggUm.push_back(CalpontSystemCatalog::LONGDOUBLE); + wideDecimalOrLongDouble(colPm, typeProj[colPm], + precisionProj, oidsProj, retKey, scaleProj, widthAggPm, + oidsAggUm, keysAggUm, typeAggUm, scaleAggUm, + precisionAggUm, widthAggUm); csNumAggUm.push_back(8); - precisionAggUm.push_back(-1); - widthAggUm.push_back(sizeof(long double)); } else { diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index a3a72428f..6bc80e42c 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -3621,7 +3621,8 @@ ArithmeticColumn* buildArithmeticColumn( unsigned int precision = idp->max_length; unsigned int scale = idp->decimals; - datatypes::Decimal::setDecimalScalePrecisionLegacy(mysql_type, precision, scale); + datatypes::Decimal::setDecimalScalePrecisionLegacy(mysql_type, + precision, scale); } else { @@ -3638,7 +3639,8 @@ ArithmeticColumn* buildArithmeticColumn( int32_t scale2 = pt->right()->data()->resultType().scale; if (funcName == "/" && - (mysql_type.scale - (scale1 - scale2)) > datatypes::INT128MAXPRECISION) + (mysql_type.scale - (scale1 - scale2)) > + datatypes::INT128MAXPRECISION) { Item_decimal* idp = (Item_decimal*)item; @@ -4980,15 +4982,33 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) // use the first parm for result type. parm = ac->aggParms()[0]; - // WIP why do we use LONGDOUBLE for AVG? - if (isp->sum_func() == Item_sum::AVG_FUNC || - isp->sum_func() == Item_sum::AVG_DISTINCT_FUNC) + bool isAvg = (isp->sum_func() == Item_sum::AVG_FUNC || + isp->sum_func() == Item_sum::AVG_DISTINCT_FUNC); + if (isAvg || isp->sum_func() == Item_sum::SUM_FUNC || + isp->sum_func() == Item_sum::SUM_DISTINCT_FUNC) { CalpontSystemCatalog::ColType ct = parm->resultType(); - ct.colDataType = CalpontSystemCatalog::LONGDOUBLE; - ct.colWidth = sizeof(long double); - ct.scale += 4; - ct.precision = -1; + if (datatypes::Decimal::isWideDecimalType(ct)) + { + uint32_t precision = ct.precision; + uint32_t scale = ct.scale; + if (isAvg) + { + datatypes::Decimal::setScalePrecision4Avg(precision, scale); + } + ct.precision = precision; + ct.scale = scale; + } + else + { + ct.colDataType = CalpontSystemCatalog::LONGDOUBLE; + ct.colWidth = sizeof(long double); + if (isAvg) + { + ct.scale += datatypes::MAXSCALEINC4AVG; + } + ct.precision = datatypes::IGNOREPRECISION; + } ac->resultType(ct); } else if (isp->sum_func() == Item_sum::COUNT_FUNC || @@ -5000,25 +5020,6 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) ct.scale = parm->resultType().scale; ac->resultType(ct); } - else if (isp->sum_func() == Item_sum::SUM_FUNC || - isp->sum_func() == Item_sum::SUM_DISTINCT_FUNC) - { - // WIP MCOL-641 This fast hack breaks aggregates for - // all float DT's - // UPD it doesn't break b/c actual DT for result type - // is set during JobList creation. - /*CalpontSystemCatalog::ColType ct = parm->resultType(); - ct.colDataType = CalpontSystemCatalog::LONGDOUBLE; - ct.colWidth = sizeof(long double); - ct.precision = -1;*/ - CalpontSystemCatalog::ColType ct = parm->resultType(); - ct.colDataType = CalpontSystemCatalog::DECIMAL; - ct.colWidth = 16; - ct.precision = 38; - // WIP set the scale if argument is a float-based DT - ct.scale = 0; - ac->resultType(ct); - } else if (isp->sum_func() == Item_sum::STD_FUNC || isp->sum_func() == Item_sum::VARIANCE_FUNC) { @@ -5058,7 +5059,11 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) } // adjust decimal result type according to internalDecimalScale - if (gwi.internalDecimalScale >= 0 && ac->resultType().colDataType == CalpontSystemCatalog::DECIMAL) + bool isWideDecimal = + datatypes::Decimal::isWideDecimalType(ac->resultType()); + // This must be also valid for UDECIMAL + if (!isWideDecimal && gwi.internalDecimalScale >= 0 + && ac->resultType().colDataType == CalpontSystemCatalog::DECIMAL) { CalpontSystemCatalog::ColType ct = ac->resultType(); ct.scale = gwi.internalDecimalScale; diff --git a/dbcon/mysql/ha_mcs_partition.cpp b/dbcon/mysql/ha_mcs_partition.cpp index edf357df2..7271fba51 100644 --- a/dbcon/mysql/ha_mcs_partition.cpp +++ b/dbcon/mysql/ha_mcs_partition.cpp @@ -1100,7 +1100,7 @@ extern "C" mapit = partMap.find(logicalPartNum); - int state; + int state = CP_INVALID; if (ct.colWidth <= 8) state = em.getExtentMaxMin(iter->range.start, partInfo.max, partInfo.min, seqNum); diff --git a/utils/joiner/tuplejoiner.cpp b/utils/joiner/tuplejoiner.cpp index 9282e7253..a8e853553 100644 --- a/utils/joiner/tuplejoiner.cpp +++ b/utils/joiner/tuplejoiner.cpp @@ -1127,7 +1127,7 @@ void TupleJoiner::updateCPData(const Row& r) } else { - int64_t val; + int64_t val = 0; if (r.getColType(colIdx) == CalpontSystemCatalog::LONGDOUBLE) { double dval = (double)roundl(r.getLongDoubleField(colIdx)); diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index ce52b8d2e..ad5d02233 100755 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -1,6 +1,6 @@ /* Copyright (C) 2014 InfiniDB, Inc. - Copyright (c) 2019 MariaDB Corporation + Copyright (c) 2019-2020 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -57,6 +57,8 @@ //..comment out NDEBUG to enable assertions, uncomment NDEBUG to disable //#define NDEBUG +#include "funcexp/utils_utf8.h" +#include "mcs_decimal.h" using namespace std; using namespace boost; @@ -70,12 +72,16 @@ namespace const int64_t AGG_ROWGROUP_SIZE = 256; template -bool minMax(T d1, T d2, int type) +inline bool minMax(T d1, T d2, int type) { if (type == rowgroup::ROWAGG_MIN) return d1 < d2; else return d1 > d2; } +inline bool minMax(int128_t* d1, int128_t* d2, int type) +{ + return (type == rowgroup::ROWAGG_MIN) ? *d1 < *d2 : *d1 > *d2; +} inline int64_t getIntNullValue(int colType) { @@ -334,6 +340,16 @@ inline bool ExternalKeyEq::operator()(const RowPosition& pos1, const RowPosition static const string overflowMsg("Aggregation overflow."); +inline void RowAggregation::updateIntMinMax(int128_t* val1, int128_t* val2, int64_t col, int func) +{ + int32_t colOutOffset = fRow.getOffset(col); + if (isNull(fRowGroupOut, fRow, col)) + fRow.setBinaryField_offset(val1, sizeof(int128_t), colOutOffset); + else if (minMax(val1, val2, func)) + fRow.setBinaryField_offset(val1, sizeof(int128_t), colOutOffset); +} + + inline void RowAggregation::updateIntMinMax(int64_t val1, int64_t val2, int64_t col, int func) { if (isNull(fRowGroupOut, fRow, col)) @@ -1010,13 +1026,31 @@ void RowAggregation::initMapData(const Row& rowIn) case execplan::CalpontSystemCatalog::MEDINT: case execplan::CalpontSystemCatalog::INT: case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: { fRow.setIntField(rowIn.getIntField(colIn), colOut); break; } + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (LIKELY(fRow.getColumnWidth(colIn) == datatypes::MAXDECIMALWIDTH)) + { + uint32_t colOutOffset = fRow.getOffset(colOut); + fRow.setBinaryField_offset( + rowIn.getBinaryField(colIn), + sizeof(int128_t), + colOutOffset); + } + else + { + fRow.setIntField(rowIn.getIntField(colIn), colOut); + } + + break; + } + + case execplan::CalpontSystemCatalog::UTINYINT: case execplan::CalpontSystemCatalog::USMALLINT: case execplan::CalpontSystemCatalog::UMEDINT: @@ -1113,8 +1147,6 @@ void RowAggregation::makeAggFieldsNull(Row& row) fFunctionCols[i]->fAggFunction == ROWAGG_GROUP_CONCAT || fFunctionCols[i]->fAggFunction == ROWAGG_STATS) { -// done by memset -// row.setIntField(0, colOut); continue; } @@ -1160,18 +1192,18 @@ void RowAggregation::makeAggFieldsNull(Row& row) case execplan::CalpontSystemCatalog::UDECIMAL: { int colWidth = fRowGroupOut->getColumnWidth(colOut); - if (colWidth <= 8) + if (LIKELY(colWidth == datatypes::MAXDECIMALWIDTH)) + { + uint32_t offset = row.getOffset(colOut); + row.setBinaryField_offset( + const_cast(&datatypes::Decimal128Null), + colWidth, + offset); + } + else if (colWidth == datatypes::MAXLEGACYWIDTH) { row.setIntField(getUintNullValue(colDataType, colWidth), colOut); } - else - { - int128_t nullValue = 0; - utils::setWideDecimalNullValue(nullValue); - uint32_t offset = row.getOffset(colOut); - row.setBinaryField_offset(&nullValue, sizeof(nullValue), - offset); - } break; } @@ -1183,7 +1215,7 @@ void RowAggregation::makeAggFieldsNull(Row& row) { int colWidth = fRowGroupOut->getColumnWidth(colOut); - if (colWidth <= 8) + if (colWidth <= datatypes::MAXLEGACYWIDTH) { row.setUintField(getUintNullValue(colDataType, colWidth), colOut); } @@ -1256,8 +1288,6 @@ void RowAggregation::doMinMax(const Row& rowIn, int64_t colIn, int64_t colOut, i case execplan::CalpontSystemCatalog::MEDINT: case execplan::CalpontSystemCatalog::INT: case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: { int64_t valIn = rowIn.getIntField(colIn); int64_t valOut = fRow.getIntField(colOut); @@ -1265,6 +1295,24 @@ void RowAggregation::doMinMax(const Row& rowIn, int64_t colIn, int64_t colOut, i break; } + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (LIKELY(fRow.getColumnWidth(colIn) == datatypes::MAXDECIMALWIDTH)) + { + updateIntMinMax(rowIn.getBinaryField(colIn), + fRow.getBinaryField(colOut), + colOut, funcType); + } + else + { + int64_t valIn = rowIn.getIntField(colIn); + int64_t valOut = fRow.getIntField(colOut); + updateIntMinMax(valIn, valOut, colOut, funcType); + } + break; + } + case execplan::CalpontSystemCatalog::UTINYINT: case execplan::CalpontSystemCatalog::USMALLINT: case execplan::CalpontSystemCatalog::UMEDINT: @@ -1340,17 +1388,12 @@ void RowAggregation::doMinMax(const Row& rowIn, int64_t colIn, int64_t colOut, i // Note: NULL value check must be done on UM & PM // UM may receive NULL values, too. //------------------------------------------------------------------------------ -// WIP MCOL-641. This and other methods must be type based to avoid needless mem -// allocation for wide DTs void RowAggregation::doSum(const Row& rowIn, int64_t colIn, int64_t colOut, int funcType) { int colDataType = (fRowGroupIn.getColTypes())[colIn]; long double valIn = 0; bool isWideDataType = false; - void *wideValInPtr = NULL; - // WIP MCOL-641 Probably the width must be taken - // from colOut - uint32_t width = fRowGroupOut->getColumnWidth(colOut); + void *wideValInPtr = nullptr; if (isNull(&fRowGroupIn, rowIn, colIn) == true) return; @@ -1380,9 +1423,14 @@ void RowAggregation::doSum(const Row& rowIn, int64_t colIn, int64_t colOut, int case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - // WIP MCOL-641 make the size dynamic and use branch prediction cond - isWideDataType = (width) > 8 ? true : false; - if (!isWideDataType) + uint32_t width = fRowGroupOut->getColumnWidth(colOut); + isWideDataType = width == datatypes::MAXDECIMALWIDTH; + if(LIKELY(isWideDataType)) + { + int128_t *dec = rowIn.getBinaryField(colIn); + wideValInPtr = reinterpret_cast(dec); + } + else { valIn = rowIn.getIntField(colIn); double scale = (double)(fRowGroupIn.getScale())[colIn]; @@ -1391,19 +1439,6 @@ void RowAggregation::doSum(const Row& rowIn, int64_t colIn, int64_t colOut, int valIn /= pow(10.0, scale); } } - else - { - if (colDataType == execplan::CalpontSystemCatalog::DECIMAL) - { - int128_t *dec = rowIn.getBinaryField(colIn); - wideValInPtr = reinterpret_cast(dec); - } - else - { - uint128_t *dec = rowIn.getBinaryField(colIn); - wideValInPtr = reinterpret_cast(dec); - } - } break; } @@ -1455,49 +1490,31 @@ void RowAggregation::doSum(const Row& rowIn, int64_t colIn, int64_t colOut, int break; } } - // WIP MCOL-641 - if (!isWideDataType) + if (LIKELY(!isWideDataType)) { - if (isNull(fRowGroupOut, fRow, colOut)) - { - fRow.setLongDoubleField(valIn, colOut); - } - else + if (LIKELY(!isNull(fRowGroupOut, fRow, colOut))) { long double valOut = fRow.getLongDoubleField(colOut); fRow.setLongDoubleField(valIn+valOut, colOut); } + else + { + fRow.setLongDoubleField(valIn, colOut); + } } else { uint32_t offset = fRow.getOffset(colOut); - if (colDataType == execplan::CalpontSystemCatalog::DECIMAL) + int128_t* dec = reinterpret_cast(wideValInPtr); + if (LIKELY(!isNull(fRowGroupOut, fRow, colOut))) { - int128_t *dec = reinterpret_cast(wideValInPtr); - if (isNull(fRowGroupOut, fRow, colOut)) - { - fRow.setBinaryField_offset(dec, sizeof(*dec), offset); - } - else - { - int128_t *valOutPtr = fRow.getBinaryField(valOutPtr, colOut); - int128_t sum = *valOutPtr + *dec; - fRow.setBinaryField_offset(&sum, sizeof(sum), offset); - } + int128_t *valOutPtr = fRow.getBinaryField(valOutPtr, colOut); + int128_t sum = *valOutPtr + *dec; + fRow.setBinaryField_offset(&sum, sizeof(sum), offset); } else { - uint128_t *dec = reinterpret_cast(wideValInPtr); - if (isNull(fRowGroupOut, fRow, colOut)) - { - fRow.setBinaryField_offset(dec, sizeof(*dec), offset); - } - else - { - uint128_t *valOutPtr = fRow.getBinaryField(valOutPtr, colOut); - uint128_t sum = *valOutPtr + *dec; - fRow.setBinaryField_offset(&sum, sizeof(sum), offset); - } + fRow.setBinaryField_offset(dec, sizeof(*dec), offset); } } // end-of isWideDataType block } @@ -1791,7 +1808,7 @@ void RowAggregation::updateEntry(const Row& rowIn) case ROWAGG_AVG: // count(column) for average is inserted after the sum, - // colOut+1 is the position of the count column. + // colOut+1 is the position of the aux count column. doAvg(rowIn, colIn, colOut, colOut + 1); break; @@ -1851,6 +1868,8 @@ void RowAggregation::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, int6 int colDataType = (fRowGroupIn.getColTypes())[colIn]; long double valIn = 0; long double valOut = fRow.getLongDoubleField(colOut); + bool isWideDataType = false; + void *wideValInPtr = nullptr; switch (colDataType) { @@ -1862,7 +1881,6 @@ void RowAggregation::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, int6 { valIn = rowIn.getIntField(colIn); break; - break; } case execplan::CalpontSystemCatalog::UTINYINT: @@ -1878,11 +1896,21 @@ void RowAggregation::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, int6 case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - valIn = rowIn.getIntField(colIn); - double scale = (double)(fRowGroupIn.getScale())[colIn]; - if (valIn != 0 && scale > 0) + uint32_t width = fRowGroupOut->getColumnWidth(colOut); + isWideDataType = width == datatypes::MAXDECIMALWIDTH; + if(LIKELY(isWideDataType)) { - valIn /= pow(10.0, scale); + int128_t* dec = rowIn.getBinaryField(colIn); + wideValInPtr = reinterpret_cast(dec); + } + else + { + valIn = rowIn.getIntField(colIn); + double scale = (double)(fRowGroupIn.getScale())[colIn]; + if (valIn != 0 && scale > 0) + { + valIn /= pow(10.0, scale); + } } break; } @@ -1917,16 +1945,32 @@ void RowAggregation::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, int6 } } - if (fRow.getUintField(colAux) == 0) + // min(count) = 0 + uint64_t count = fRow.getUintField(colAux) + 1; + fRow.setUintField<8>(count, colAux); + bool notFirstValue = count > 1; + + if (LIKELY(!isWideDataType)) { - // This is the first value - fRow.setLongDoubleField(valIn, colOut); - fRow.setUintField(1, colAux); + if (LIKELY(notFirstValue)) + fRow.setLongDoubleField(valIn + valOut, colOut); + else // This is the first value + fRow.setLongDoubleField(valIn, colOut); } else { - fRow.setLongDoubleField(valIn + valOut, colOut); - fRow.setUintField(fRow.getUintField(colAux) + 1, colAux); + uint32_t offset = fRow.getOffset(colOut); + int128_t* dec = reinterpret_cast(wideValInPtr); + if (LIKELY(notFirstValue)) + { + int128_t *valOutPtr = fRow.getBinaryField(valOutPtr, colOut); + int128_t sum = *valOutPtr + *dec; + fRow.setBinaryField_offset(&sum, sizeof(sum), offset); + } + else + { + fRow.setBinaryField_offset(dec, sizeof(*dec), offset); + } } } @@ -2632,11 +2676,6 @@ void RowAggregationUM::calculateAvgColumns() int64_t colOut = fFunctionCols[i]->fOutputColumnIndex; int64_t colAux = fFunctionCols[i]->fAuxColumnIndex; -// int scale = fRowGroupOut->getScale()[colOut]; -// int scale1 = scale >> 8; -// int scale2 = scale & 0x000000FF; -// long double factor = pow(10.0, scale2 - scale1); - for (uint64_t j = 0; j < fRowGroupOut->getRowCount(); j++) { fRowGroupOut->getRow(j, &fRow); @@ -2645,14 +2684,38 @@ void RowAggregationUM::calculateAvgColumns() if (cnt == 0) // empty set, value is initialized to null. continue; - long double sum = 0.0; - long double avg = 0.0; + uint32_t precision = fRow.getPrecision(colOut); + bool isWideDecimal = + datatypes::Decimal::isWideDecimalType(precision); - // MCOL-1822 Always long double - sum = fRow.getLongDoubleField(colOut); - avg = sum / cnt; -// avg *= factor; - fRow.setLongDoubleField(avg, colOut); + if (LIKELY(!isWideDecimal)) + { + long double sum = 0.0; + long double avg = 0.0; + sum = fRow.getLongDoubleField(colOut); + avg = sum / cnt; + fRow.setLongDoubleField(avg, colOut); + } + else + { + uint32_t offset = fRow.getOffset(colOut); + uint32_t scale = fRow.getScale(colOut); + // Get multiplied to deliver AVG with the scale closest + // to the expected original scale + 4. + // There is a counterpart in buildAggregateColumn. + datatypes::Decimal::setScalePrecision4Avg(precision, scale); + int128_t* sumPnt = fRow.getBinaryField_offset(offset); + uint32_t scaleDiff = scale - fRow.getScale(colOut); + // multiplication overflow check + datatypes::MultiplicationOverflowCheck multOp; + int128_t sum = 0; + if (scaleDiff > 0) + multOp(*sumPnt, datatypes::mcs_pow_10[scaleDiff], sum); + else + sum = *sumPnt; + int128_t avg = sum / cnt; + fRow.setBinaryField_offset(&avg, sizeof(avg), offset); + } } } } @@ -4174,6 +4237,8 @@ void RowAggregationUMP2::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, int colDataType = (fRowGroupIn.getColTypes())[colIn]; long double valIn = 0; long double valOut = fRow.getLongDoubleField(colOut); + bool isWideDataType = false; + void *wideValInPtr = nullptr; switch (colDataType) { @@ -4200,12 +4265,21 @@ void RowAggregationUMP2::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - valIn = rowIn.getIntField(colIn); - break; - double scale = (double)(fRowGroupIn.getScale())[colIn]; - if (valIn != 0 && scale > 0) + uint32_t width = fRowGroupOut->getColumnWidth(colOut); + isWideDataType = width == datatypes::MAXDECIMALWIDTH; + if(LIKELY(isWideDataType)) { - valIn /= pow(10.0, scale); + int128_t* dec = rowIn.getBinaryField(colIn); + wideValInPtr = reinterpret_cast(dec); + } + else + { + valIn = rowIn.getIntField(colIn); + double scale = (double)(fRowGroupIn.getScale())[colIn]; + if (valIn != 0 && scale > 0) + { + valIn /= pow(10.0, scale); + } } break; } @@ -4240,16 +4314,36 @@ void RowAggregationUMP2::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, } } - int64_t cnt = fRow.getUintField(colAux); - if (cnt == 0) + uint64_t cnt = fRow.getUintField(colAux); + if (LIKELY(!isWideDataType)) { - fRow.setLongDoubleField(valIn, colOut); - fRow.setUintField(rowIn.getUintField(colIn + 1), colAux); + if (LIKELY(cnt > 0)) + { + fRow.setLongDoubleField(valIn + valOut, colOut); + fRow.setUintField(rowIn.getUintField(colIn + 1) + cnt, colAux); + } + else + { + fRow.setLongDoubleField(valIn, colOut); + fRow.setUintField(rowIn.getUintField(colIn + 1), colAux); + } } else { - fRow.setLongDoubleField(valIn + valOut, colOut); - fRow.setUintField(rowIn.getUintField(colIn + 1) + cnt, colAux); + uint32_t offset = fRow.getOffset(colOut); + int128_t* dec = reinterpret_cast(wideValInPtr); + if (LIKELY(cnt > 0)) + { + int128_t *valOutPtr = fRow.getBinaryField(valOutPtr, colOut); + int128_t sum = *valOutPtr + *dec; + fRow.setBinaryField_offset(&sum, sizeof(sum), offset); + fRow.setUintField(rowIn.getUintField(colIn + 1) + cnt, colAux); + } + else + { + fRow.setBinaryField_offset(dec, sizeof(*dec), offset); + fRow.setUintField(rowIn.getUintField(colIn + 1), colAux); + } } } diff --git a/utils/rowgroup/rowaggregation.h b/utils/rowgroup/rowaggregation.h index f0d50381f..3c138b1cf 100644 --- a/utils/rowgroup/rowaggregation.h +++ b/utils/rowgroup/rowaggregation.h @@ -653,6 +653,7 @@ protected: copyRow(fNullRow, &row); } + inline void updateIntMinMax(int128_t* val1, int128_t* val2, int64_t col, int func); inline void updateIntMinMax(int64_t val1, int64_t val2, int64_t col, int func); inline void updateUintMinMax(uint64_t val1, uint64_t val2, int64_t col, int func); inline void updateCharMinMax(uint64_t val1, uint64_t val2, int64_t col, int func); diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 17d955a1f..7cda3a60a 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -810,25 +810,16 @@ inline uint32_t Row::getStringLength(uint32_t colIndex) const return strnlen((char*) &data[offsets[colIndex]], getColumnWidth(colIndex)); } -// WIP Remove this -// Check whether memcpy affects perf here -/*inline void Row::setBinaryField(const uint8_t* strdata, uint32_t length, uint32_t offset) -{ - memcpy(&data[offset], strdata, length); -}*/ - -// MCOL-641. This method can be applied to uint8_t* buffers. template inline void Row::setBinaryField(T* value, uint32_t width, uint32_t colIndex) { memcpy(&data[offsets[colIndex]], value, width); } -// MCOL-641. This method !cannot! be applied to uint8_t* buffers. +// This method !cannot! be applied to uint8_t* buffers. template inline void Row::setBinaryField_offset(T* value, uint32_t width, uint32_t offset) { - // WIP Compare performance. *reinterpret_cast(&data[offset]) = *value; } @@ -871,23 +862,15 @@ inline std::string Row::getStringField(uint32_t colIndex) const strnlen((char*) &data[offsets[colIndex]], getColumnWidth(colIndex))); } -/*inline std::string Row::getBinaryField(uint32_t colIndex) const -{ - return std::string((char*) &data[offsets[colIndex]], getColumnWidth(colIndex)); -}*/ - -// WIP MCOL-641 template inline T* Row::getBinaryField(uint32_t colIndex) const { - //return reinterpret_cast(&data[offsets[colIndex]]); return getBinaryField_offset(offsets[colIndex]); } template inline T* Row::getBinaryField(T* argtype, uint32_t colIndex) const { - //return reinterpret_cast(&data[offsets[colIndex]]); return getBinaryField_offset(offsets[colIndex]); } From 21a41738e1a5bc0dded1f76efa22e28713e4795b Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Fri, 8 May 2020 15:57:23 +0000 Subject: [PATCH 37/78] MCOL-641 Simple aggregates works with GROUP BY column keys. Fixed constant colump copy for binary columns in TNS. --- datatypes/mcs_decimal.h | 15 ++++++++++++- dbcon/joblist/tupleconstantstep.cpp | 15 +------------ utils/rowgroup/rowaggregation.cpp | 2 +- utils/rowgroup/rowgroup.h | 35 ++++++++++++++++++++++++----- 4 files changed, 45 insertions(+), 22 deletions(-) diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index 346dd2dda..1a905bd72 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -25,6 +25,7 @@ using int128_t = __int128; using ColTypeAlias = execplan::CalpontSystemCatalog::ColType; +using ColDataTypeAlias = execplan::CalpontSystemCatalog::ColDataType; namespace execplan { @@ -166,7 +167,7 @@ class Decimal /** @brief The method detects whether decimal type is wide - using csc data type. + using csc colType. */ static constexpr inline bool isWideDecimalType(const ColTypeAlias& ct) { @@ -185,6 +186,18 @@ class Decimal && precision <= INT128MAXPRECISION; } + /** + @brief The method detects whether decimal type is wide + using datatype and width. + */ + static constexpr inline bool isWideDecimalType(const ColDataTypeAlias dt, + const int32_t width) + { + return (width == MAXDECIMALWIDTH && + (dt == execplan::CalpontSystemCatalog::DECIMAL || + dt == execplan::CalpontSystemCatalog::UDECIMAL)); + } + /** @brief The method sets the legacy scale and precision of a wide decimal column which is the result of an arithmetic operation. diff --git a/dbcon/joblist/tupleconstantstep.cpp b/dbcon/joblist/tupleconstantstep.cpp index 2a6a2ee7f..ccbff89c7 100644 --- a/dbcon/joblist/tupleconstantstep.cpp +++ b/dbcon/joblist/tupleconstantstep.cpp @@ -512,31 +512,18 @@ void TupleConstantStep::fillInConstants(const rowgroup::Row& rowIn, rowgroup::Ro if (fIndexConst.size() > 1 || fIndexConst[0] != 0) { copyRow(fRowConst, &rowOut); - //memcpy(rowOut.getData(), fRowConst.getData(), fRowConst.getSize()); rowOut.setRid(rowIn.getRelRid()); for (uint64_t j = 0; j < fIndexMapping.size(); ++j) rowIn.copyField(rowOut, fIndexMapping[j], j); - - //rowIn.copyField(rowOut.getData() + rowOut.getOffset(fIndexMapping[j]), j); } else // only first column is constant { - //size_t n = rowOut.getOffset(rowOut.getColumnCount()) - rowOut.getOffset(1); rowOut.setRid(rowIn.getRelRid()); fRowConst.copyField(rowOut, 0, 0); - //fRowConst.copyField(rowOut.getData()+2, 0); // hardcoded 2 for rid length for (uint32_t i = 1; i < rowOut.getColumnCount(); i++) - // WIP MCOL-641 implement copyBinaryField inside copyField - if (UNLIKELY(utils::isWide(rowIn.getColumnWidth(i - 1)) && - (rowIn.getColType(i - 1) == CalpontSystemCatalog::DECIMAL || - rowIn.getColType(i - 1) == CalpontSystemCatalog::UDECIMAL))) - rowIn.copyBinaryField(rowOut, i, i - 1); - else - rowIn.copyField(rowOut, i, i - 1); - - //memcpy(rowOut.getData()+rowOut.getOffset(1), rowIn.getData()+2, n); + rowIn.copyField(rowOut, i, i - 1); } } diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index ad5d02233..f740f6e01 100755 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -1298,7 +1298,7 @@ void RowAggregation::doMinMax(const Row& rowIn, int64_t colIn, int64_t colOut, i case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - if (LIKELY(fRow.getColumnWidth(colIn) == datatypes::MAXDECIMALWIDTH)) + if (LIKELY(rowIn.getColumnWidth(colIn) == datatypes::MAXDECIMALWIDTH)) { updateIntMinMax(rowIn.getBinaryField(colIn), fRow.getBinaryField(colOut), diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 7cda3a60a..179493bb8 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -1203,19 +1203,32 @@ inline void Row::copyField(Row& out, uint32_t destIndex, uint32_t srcIndex) cons if (UNLIKELY(types[srcIndex] == execplan::CalpontSystemCatalog::VARBINARY || types[srcIndex] == execplan::CalpontSystemCatalog::BLOB || types[srcIndex] == execplan::CalpontSystemCatalog::TEXT)) + { out.setVarBinaryField(getVarBinaryStringField(srcIndex), destIndex); + } else if (UNLIKELY(isLongString(srcIndex))) + { out.setStringField(getStringPointer(srcIndex), getStringLength(srcIndex), destIndex); - //out.setStringField(getStringField(srcIndex), destIndex); + } else if (UNLIKELY(isShortString(srcIndex))) + { out.setUintField(getUintField(srcIndex), destIndex); + } else if (UNLIKELY(types[srcIndex] == execplan::CalpontSystemCatalog::LONGDOUBLE)) + { out.setLongDoubleField(getLongDoubleField(srcIndex), destIndex); + } + else if (UNLIKELY(datatypes::Decimal::isWideDecimalType( + types[srcIndex], colWidths[srcIndex]))) + { + copyBinaryField(out, destIndex, srcIndex); + } else + { out.setIntField(getIntField(srcIndex), destIndex); + } } -// WIP MCOL-641 inline void Row::copyBinaryField(Row& out, uint32_t destIndex, uint32_t srcIndex) const { out.setBinaryField(getBinaryField(srcIndex), 16, destIndex); @@ -1922,20 +1935,30 @@ inline void copyRow(const Row& in, Row* out, uint32_t colCount) in.getColTypes()[i] == execplan::CalpontSystemCatalog::BLOB || in.getColTypes()[i] == execplan::CalpontSystemCatalog::TEXT || in.getColTypes()[i] == execplan::CalpontSystemCatalog::CLOB)) + { out->setVarBinaryField(in.getVarBinaryStringField(i), i); + } else if (UNLIKELY(in.isLongString(i))) - //out->setStringField(in.getStringField(i), i); + { out->setStringField(in.getStringPointer(i), in.getStringLength(i), i); + } else if (UNLIKELY(in.isShortString(i))) + { out->setUintField(in.getUintField(i), i); + } else if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::LONGDOUBLE)) + { out->setLongDoubleField(in.getLongDoubleField(i), i); - else if (UNLIKELY((in.getColType(i) == execplan::CalpontSystemCatalog::DECIMAL || - in.getColType(i) == execplan::CalpontSystemCatalog::UDECIMAL) && - in.getColumnWidth(i) == datatypes::MAXDECIMALWIDTH)) + } + else if (UNLIKELY(datatypes::Decimal::isWideDecimalType( + in.getColType(i), in.getColumnWidth(i)))) + { in.copyBinaryField(*out, i, i); + } else + { out->setIntField(in.getIntField(i), i); + } } } From f63611c422f4d6422bdf5cb2c9b2d2aa7edc9ac2 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Fri, 15 May 2020 15:55:19 +0000 Subject: [PATCH 38/78] MCOL-641 This commit adds support for group_concat w/o ORDER BY. Small refactoring in Row methods. --- dbcon/joblist/groupconcat.cpp | 66 ++++++++++++++++------------------- tests/rowgroup-tests.cpp | 4 +-- utils/rowgroup/rowgroup.h | 17 ++++++--- 3 files changed, 46 insertions(+), 41 deletions(-) diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index 0c8c355a9..abada0b40 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -56,6 +56,7 @@ using namespace ordering; #include "jobstep.h" #include "jlf_common.h" #include "limitedorderby.h" +#include "mcs_decimal.h" namespace joblist { @@ -368,7 +369,7 @@ void GroupConcatAgUM::applyMapping(const boost::shared_array& mapping, cons // For some reason the rowgroup mapping fcns don't work right in this class. for (uint64_t i = 0; i < fRow.getColumnCount(); i++) { - if (fRow.getColumnWidth(i) > 8) + if (fRow.getColumnWidth(i) > datatypes::MAXLEGACYWIDTH) { if (fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::CHAR || fRow.getColTypes()[i] == execplan::CalpontSystemCatalog::VARCHAR || @@ -380,6 +381,10 @@ void GroupConcatAgUM::applyMapping(const boost::shared_array& mapping, cons { fRow.setLongDoubleField(row.getLongDoubleField(mapping[i]), i); } + else if (datatypes::Decimal::isWideDecimalType(fRow.getColType(i), fRow.getColumnWidth(i))) + { + row.copyBinaryField(fRow, i, mapping[i]); + } } else { @@ -440,25 +445,34 @@ void GroupConcator::outputRow(std::ostringstream& oss, const rowgroup::Row& row) case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: { int64_t intVal = row.getIntField(*i); - int scale = (int) row.getScale(*i); - if (scale == 0) - { - oss << intVal; - } - else - { - long double dblVal = intVal / pow(10.0, (double)scale); - oss << fixed << setprecision(scale) << dblVal; - } + oss << intVal; break; } + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + int scale = (int) row.getScale(*i); + //{ + // long double dblVal = intVal / pow(10.0, (double)scale); + // oss << fixed << setprecision(scale) << dblVal; + //} + char buf[utils::MAXLENGTH16BYTES]; + + int128_t* dec = row.getBinaryField(*i); + dataconvert::DataConvert::decimalToString(dec, + static_cast(scale), buf, + sizeof(buf), types[*i]); + oss << fixed << buf; + + break; + } + + case CalpontSystemCatalog::UTINYINT: case CalpontSystemCatalog::USMALLINT: case CalpontSystemCatalog::UMEDINT: @@ -609,27 +623,7 @@ int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row) case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - int64_t v = row.getIntField(*i); - double scale = row.getScale(*i); - - if (scale > 0) - { - v /= (int64_t) pow(10.0, scale); - - if (v < 0) fieldLen++; - - while ((v /= 10) != 0) fieldLen++; - - fieldLen += (int64_t) scale + 2;; - } - else - { - if (v < 0) fieldLen++; - - while ((v /= 10) != 0) fieldLen++; - - fieldLen += 1;; - } + fieldLen += row.getPrecision(*i)/2; break; } @@ -1092,8 +1086,10 @@ void GroupConcatNoOrder::getResult(uint8_t* buff, const string& sep) } size_t resultSize = oss.str().size(); + resultSize = (resultSize > fGroupConcatLen) ? fGroupConcatLen : resultSize; fOutputString.reset(new uint8_t[resultSize + 2]); - memset(fOutputString.get(), 0, resultSize + 2); + fOutputString[resultSize] = '\0'; + fOutputString[resultSize + 1] = '\0'; strncpy((char*)fOutputString.get(), oss.str().c_str(), resultSize); diff --git a/tests/rowgroup-tests.cpp b/tests/rowgroup-tests.cpp index 7d7604b63..be68a79cd 100644 --- a/tests/rowgroup-tests.cpp +++ b/tests/rowgroup-tests.cpp @@ -291,8 +291,8 @@ TEST_F(RowDecimalTest, CopyBinaryFieldCheck) col2Out = rOut.getBinaryField(1); EXPECT_NE(*col1In, *col1Out); EXPECT_NE(*col2In, *col2Out); - r.copyBinaryField(rOut, 0, 0); - r.copyBinaryField(rOut, 1, 1); + r.copyBinaryField(rOut, 0, 0); + r.copyBinaryField(rOut, 1, 1); col1Out = rOut.getBinaryField(0); col2Out = rOut.getBinaryField(1); EXPECT_EQ(*col1In, *col1Out); diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 179493bb8..cb6c20718 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -429,6 +429,8 @@ public: template inline void setBinaryField(T* strdata, uint32_t width, uint32_t colIndex); template + inline void setBinaryField(T* strdata, uint32_t colIndex); + template inline void setBinaryField_offset(T* strdata, uint32_t width, uint32_t colIndex); // support VARBINARY // Add 2-byte length at the CHARSET_INFO*beginning of the field. NULL and zero length field are @@ -475,7 +477,7 @@ public: // that's not string-table safe, this one is inline void copyField(Row& dest, uint32_t destIndex, uint32_t srcIndex) const; - // WIP MCOL-641 + template inline void copyBinaryField(Row& dest, uint32_t destIndex, uint32_t srcIndex) const; std::string toString() const; @@ -816,6 +818,12 @@ inline void Row::setBinaryField(T* value, uint32_t width, uint32_t colIndex) memcpy(&data[offsets[colIndex]], value, width); } +template +inline void Row::setBinaryField(T* value, uint32_t colIndex) +{ + *reinterpret_cast(&data[offsets[colIndex]]) = *value; +} + // This method !cannot! be applied to uint8_t* buffers. template inline void Row::setBinaryField_offset(T* value, uint32_t width, uint32_t offset) @@ -1221,7 +1229,7 @@ inline void Row::copyField(Row& out, uint32_t destIndex, uint32_t srcIndex) cons else if (UNLIKELY(datatypes::Decimal::isWideDecimalType( types[srcIndex], colWidths[srcIndex]))) { - copyBinaryField(out, destIndex, srcIndex); + copyBinaryField(out, destIndex, srcIndex); } else { @@ -1229,9 +1237,10 @@ inline void Row::copyField(Row& out, uint32_t destIndex, uint32_t srcIndex) cons } } +template inline void Row::copyBinaryField(Row& out, uint32_t destIndex, uint32_t srcIndex) const { - out.setBinaryField(getBinaryField(srcIndex), 16, destIndex); + out.setBinaryField(getBinaryField(srcIndex), destIndex); } inline void Row::setRid(uint64_t rid) @@ -1953,7 +1962,7 @@ inline void copyRow(const Row& in, Row* out, uint32_t colCount) else if (UNLIKELY(datatypes::Decimal::isWideDecimalType( in.getColType(i), in.getColumnWidth(i)))) { - in.copyBinaryField(*out, i, i); + in.copyBinaryField(*out, i, i); } else { From a7fcf39f2a9626a58fe8ee93f33e94877b9be04c Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Mon, 18 May 2020 11:02:27 +0000 Subject: [PATCH 39/78] MCOL-641 Fixed group_concat for narrow-DECIMALs. --- dbcon/joblist/groupconcat.cpp | 37 +++++++++++++++++++++++---------- utils/common/widedecimalutils.h | 1 + utils/funcexp/func_math.cpp | 3 ++- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index abada0b40..242b630b3 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -457,17 +457,32 @@ void GroupConcator::outputRow(std::ostringstream& oss, const rowgroup::Row& row) case CalpontSystemCatalog::UDECIMAL: { int scale = (int) row.getScale(*i); - //{ - // long double dblVal = intVal / pow(10.0, (double)scale); - // oss << fixed << setprecision(scale) << dblVal; - //} - char buf[utils::MAXLENGTH16BYTES]; - - int128_t* dec = row.getBinaryField(*i); - dataconvert::DataConvert::decimalToString(dec, - static_cast(scale), buf, - sizeof(buf), types[*i]); - oss << fixed << buf; + + if (LIKELY(row.getColumnWidth(*i) == datatypes::MAXDECIMALWIDTH)) + { + char buf[utils::MAXLENGTH16BYTES]; + + int128_t* dec = row.getBinaryField(*i); + dataconvert::DataConvert::decimalToString(dec, + static_cast(scale), buf, + sizeof(buf), types[*i]); + oss << fixed << buf; + } + else + { + int64_t intVal = row.getIntField(*i); + if (scale == 0) + { + oss << intVal; + } + else + { + char buf[utils::MAXLENGTH8BYTES]; + dataconvert::DataConvert::decimalToString(intVal, + scale, buf, sizeof(buf), types[*i]); + oss << fixed << buf; + } + } break; } diff --git a/utils/common/widedecimalutils.h b/utils/common/widedecimalutils.h index a27859828..3ebdb408b 100644 --- a/utils/common/widedecimalutils.h +++ b/utils/common/widedecimalutils.h @@ -30,6 +30,7 @@ namespace utils const uint64_t BINARYEMPTYVALUELOW = 1ULL; const uint64_t BINARYEMPTYVALUEHIGH = 0x8000000000000000ULL; const uint8_t MAXLENGTH16BYTES = 42; + const uint8_t MAXLENGTH8BYTES = 23; inline bool isWideDecimalNullValue(const int128_t& val) { diff --git a/utils/funcexp/func_math.cpp b/utils/funcexp/func_math.cpp index 13c738217..0c86cbb67 100644 --- a/utils/funcexp/func_math.cpp +++ b/utils/funcexp/func_math.cpp @@ -1911,7 +1911,8 @@ string Func_format::getStrVal(Row& row, char buf[80]; - dataconvert::DataConvert::decimalToString( decimal.value, decimal.scale, buf, 80, parm[0]->data()->resultType().colDataType); + dataconvert::DataConvert::decimalToString( decimal.value, + decimal.scale, buf, 80, parm[0]->data()->resultType().colDataType); value = buf; } From 51d77d74dfcfdecf0e234177e98aec88b959cb0c Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Mon, 18 May 2020 13:00:20 +0000 Subject: [PATCH 40/78] MCOL-641 Fix for GROUP BY on wide-DECIMALs. --- tests/rowgroup-tests.cpp | 30 +++++++++++++++++++----------- utils/rowgroup/rowgroup.h | 2 +- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/tests/rowgroup-tests.cpp b/tests/rowgroup-tests.cpp index be68a79cd..8beec7ace 100644 --- a/tests/rowgroup-tests.cpp +++ b/tests/rowgroup-tests.cpp @@ -70,13 +70,6 @@ protected: cscale.push_back(0); } - /*offsets.push_back(INITIAL_ROW_OFFSET); - offsets.push_back(16+INITIAL_ROW_OFFSET); - offsets.push_back(16*2+INITIAL_ROW_OFFSET); - roids.push_back(oid); roids.push_back(oid+1); - tkeys.push_back(1); tkeys.push_back(1); - cscale.push_back(0); cscale.push_back(0);*/ - rowgroup::RowGroup inRG(roids.size(), // column count offsets, // oldOffset roids, // column oids @@ -159,7 +152,7 @@ protected: // void TearDown() override {} - rowgroup::Row r, rOut; + rowgroup::Row r, rOut, sameRow, nonequiRow; rowgroup::Row rOutMappingCheck; rowgroup::RowGroup rg, rgOut; rowgroup::RGData rgD, rgDOut; @@ -208,8 +201,6 @@ TEST_F(RowDecimalTest, GetBinaryFieldCheck) rg.getRow(0, &r); uint128_t* u128Value; int128_t* s128Value; -// std::remove_reference::type uType; -// std::remove_reference::type sType; for (size_t i = 0; i < sValueVector.size(); i++) { @@ -302,7 +293,24 @@ TEST_F(RowDecimalTest, CopyBinaryFieldCheck) } } -// WIP TEST_F(RowDecimalTest, RowEqualsCheck) { + rg.getRow(0, &r); + rg.getRow(0, &sameRow); + rg.getRow(1, &nonequiRow); + + for (size_t i = 0; i < sValueVector.size(); i++) + { + EXPECT_TRUE(r.equals(sameRow)); + if (i < sValueVector.size()-1) + { + EXPECT_FALSE(r.equals(nonequiRow)); + } + r.nextRow(rowSize); + sameRow.nextRow(rowSize); + if (i < sValueVector.size()-1) + { + nonequiRow.nextRow(rowSize); + } + } } diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index cb6c20718..da01d0941 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -1359,7 +1359,7 @@ inline bool Row::equals(const Row& r2, uint32_t lastCol) const } else if (UNLIKELY(execplan::isDecimal(columnType))) { - if (getBinaryField(i) != r2.getBinaryField(i)) + if (*getBinaryField(i) != *r2.getBinaryField(i)) return false; } From eeebe83839bcdff64c7b7e5049218251a5a8e594 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 19 May 2020 07:02:46 +0000 Subject: [PATCH 41/78] MCOL-641 Fixed the incorrect if-condition. --- utils/rowgroup/rowaggregation.cpp | 51 +++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index f740f6e01..dd3c79caf 100755 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -1042,10 +1042,15 @@ void RowAggregation::initMapData(const Row& rowIn) sizeof(int128_t), colOutOffset); } - else + else if (fRow.getColumnWidth(colIn) <= datatypes::MAXLEGACYWIDTH) { fRow.setIntField(rowIn.getIntField(colIn), colOut); } + else + { + idbassert(0); + throw std::logic_error("RowAggregation::initMapData(): DECIMAL bad length."); + } break; } @@ -1200,10 +1205,15 @@ void RowAggregation::makeAggFieldsNull(Row& row) colWidth, offset); } - else if (colWidth == datatypes::MAXLEGACYWIDTH) + else if (colWidth <= datatypes::MAXLEGACYWIDTH) { row.setIntField(getUintNullValue(colDataType, colWidth), colOut); } + else + { + idbassert(0); + throw std::logic_error("RowAggregation::makeAggFieldsNull(): DECIMAL bad length."); + } break; } @@ -1304,12 +1314,18 @@ void RowAggregation::doMinMax(const Row& rowIn, int64_t colIn, int64_t colOut, i fRow.getBinaryField(colOut), colOut, funcType); } - else + else if (rowIn.getColumnWidth(colIn) <= datatypes::MAXLEGACYWIDTH) { int64_t valIn = rowIn.getIntField(colIn); int64_t valOut = fRow.getIntField(colOut); updateIntMinMax(valIn, valOut, colOut, funcType); } + else + { + idbassert(0); + throw std::logic_error("RowAggregation::doMinMax(): DECIMAL bad length."); + } + break; } @@ -1423,14 +1439,14 @@ void RowAggregation::doSum(const Row& rowIn, int64_t colIn, int64_t colOut, int case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - uint32_t width = fRowGroupOut->getColumnWidth(colOut); + uint32_t width = fRowGroupIn.getColumnWidth(colIn); isWideDataType = width == datatypes::MAXDECIMALWIDTH; if(LIKELY(isWideDataType)) { int128_t *dec = rowIn.getBinaryField(colIn); wideValInPtr = reinterpret_cast(dec); } - else + else if (width <= datatypes::MAXLEGACYWIDTH) { valIn = rowIn.getIntField(colIn); double scale = (double)(fRowGroupIn.getScale())[colIn]; @@ -1439,6 +1455,11 @@ void RowAggregation::doSum(const Row& rowIn, int64_t colIn, int64_t colOut, int valIn /= pow(10.0, scale); } } + else + { + idbassert(0); + throw std::logic_error("RowAggregation::doSum(): DECIMAL bad length."); + } break; } @@ -1896,14 +1917,14 @@ void RowAggregation::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, int6 case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - uint32_t width = fRowGroupOut->getColumnWidth(colOut); + uint32_t width = fRowGroupIn.getColumnWidth(colIn); isWideDataType = width == datatypes::MAXDECIMALWIDTH; if(LIKELY(isWideDataType)) { int128_t* dec = rowIn.getBinaryField(colIn); wideValInPtr = reinterpret_cast(dec); } - else + else if (width <= datatypes::MAXLEGACYWIDTH) { valIn = rowIn.getIntField(colIn); double scale = (double)(fRowGroupIn.getScale())[colIn]; @@ -1912,6 +1933,12 @@ void RowAggregation::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, int6 valIn /= pow(10.0, scale); } } + else + { + idbassert(0); + throw std::logic_error("RowAggregation::doAvg(): DECIMAL bad length."); + } + break; } @@ -4265,14 +4292,14 @@ void RowAggregationUMP2::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - uint32_t width = fRowGroupOut->getColumnWidth(colOut); + uint32_t width = fRowGroupIn.getColumnWidth(colIn); isWideDataType = width == datatypes::MAXDECIMALWIDTH; if(LIKELY(isWideDataType)) { int128_t* dec = rowIn.getBinaryField(colIn); wideValInPtr = reinterpret_cast(dec); } - else + else if (width <= datatypes::MAXLEGACYWIDTH) { valIn = rowIn.getIntField(colIn); double scale = (double)(fRowGroupIn.getScale())[colIn]; @@ -4281,6 +4308,12 @@ void RowAggregationUMP2::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, valIn /= pow(10.0, scale); } } + else + { + idbassert(0); + throw std::logic_error("RowAggregationUMP2::doAvg(): DECIMAL bad length."); + } + break; } From 778ff607c8cba6b2a35e5e20264f3efa2a7e2cdb Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 21 May 2020 09:28:17 +0000 Subject: [PATCH 42/78] MCOL-641 This commit disables length estimation for DECIMALs. --- dbcon/joblist/groupconcat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index 242b630b3..01f0a6c6b 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -638,7 +638,7 @@ int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row) case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - fieldLen += row.getPrecision(*i)/2; + fieldLen += 1; break; } From 17bad9eb0b8101d579bf955abb8f27f2b474d07a Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Mon, 1 Jun 2020 08:54:30 +0000 Subject: [PATCH 43/78] MCOL-641 Initial support for ORDER BY on wide DECIMALs. --- datatypes/CMakeLists.txt | 2 +- tests/CMakeLists.txt | 14 ++-- .../comparators-tests.cpp | 67 +++++++++++++------ utils/dataconvert/CMakeLists.txt | 6 -- utils/rowgroup/CMakeLists.txt | 6 -- utils/rowgroup/rowgroup.h | 1 - utils/windowfunction/CMakeLists.txt | 6 -- utils/windowfunction/idborderby.cpp | 39 ++++++++++- utils/windowfunction/idborderby.h | 9 +++ 9 files changed, 103 insertions(+), 47 deletions(-) rename {utils/windowfunction => tests}/comparators-tests.cpp (90%) diff --git a/datatypes/CMakeLists.txt b/datatypes/CMakeLists.txt index 660ddb15e..b632a9392 100644 --- a/datatypes/CMakeLists.txt +++ b/datatypes/CMakeLists.txt @@ -5,5 +5,5 @@ set(datatypes_LIB_SRCS add_library(datatypes SHARED ${datatypes_LIB_SRCS}) -install(TARGETS datatypes DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-libs) +install(TARGETS datatypes DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d0b88ffd1..1b10181e8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -3,23 +3,29 @@ include_directories( ${ENGINE_COMMON_INCLUDES} ) if (WITH_ROWGROUP_UT) add_executable(rowgroup_tests rowgroup-tests.cpp) target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) - install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) + install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) endif() if (WITH_ARITHMETICOPERATOR_UT) add_executable(arithmeticoperator_tests arithmeticoperator-tests.cpp) target_link_libraries(arithmeticoperator_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) - install(TARGETS arithmeticoperator_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) + install(TARGETS arithmeticoperator_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) endif() if (WITH_CSDECIMAL_UT) add_executable(mcs_decimal_tests mcs_decimal-tests.cpp) target_link_libraries(mcs_decimal_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) - install(TARGETS mcs_decimal_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) + install(TARGETS mcs_decimal_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) endif() if (WITH_DATACONVERT_UT) add_executable(dataconvert_tests dataconvert-tests.cpp) target_link_libraries(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) - install(TARGETS dataconvert_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) + install(TARGETS dataconvert_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) +endif() + +if (WITH_SORTING_COMPARATORS_UT) + add_executable(comparators_tests comparators-tests.cpp) + target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) + install(TARGETS comparators_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) endif() diff --git a/utils/windowfunction/comparators-tests.cpp b/tests/comparators-tests.cpp similarity index 90% rename from utils/windowfunction/comparators-tests.cpp rename to tests/comparators-tests.cpp index d2d5df316..83627fe56 100644 --- a/utils/windowfunction/comparators-tests.cpp +++ b/tests/comparators-tests.cpp @@ -36,11 +36,13 @@ #include #include #include "bytestream.h" -#include "idborderby.h" +#include "utils/windowfunction/idborderby.h" +#include "mcs_decimal.h" #define DEBUG #define MEMORY_LIMIT 14983602176 +using int128_t = __int128; using namespace std; using namespace joblist; using namespace messageqcpp; @@ -123,27 +125,29 @@ class FilterDriver : public CppUnit::TestFixture CPPUNIT_TEST(INT_TEST); CPPUNIT_TEST(FLOAT_TEST); + CPPUNIT_TEST(WIDEDT_TEST); CPPUNIT_TEST_SUITE_END(); private: // The tests creates an RG with 1 column of the cscDt type // then initialize RGData. After that it adds two numeric values (v1 < v2)and two NULL. - // Then creates comparator structores and run a number of tests. v1 < v2 - void testComparatorWithDT(execplan::CalpontSystemCatalog::ColDataType cscDt, + // Then creates comparator structures and run a number of tests. v1 < v2 + void testComparatorWithDT(execplan::CalpontSystemCatalog::ColDataType cscDt, uint32_t width, - bool generateRandValues) + bool generateRandValues, + uint8_t precision) { std::cout << std::endl << "------------------------------------------------------------" << std::endl; uint32_t oid =3001; - std::vector offsets, roids, tkeys, cscale, cprecision; + std::vector offsets, roids, tkeys, cscale, cprecision; std::vector types; offsets.push_back(2); offsets.push_back(2+width); roids.push_back(oid); tkeys.push_back(1); types.push_back(cscDt); cscale.push_back(0); - cprecision.push_back(20); + cprecision.push_back(precision); rowgroup::RowGroup inRG(1, //column count offsets, //oldOffset roids, // column oids @@ -295,7 +299,7 @@ private: r.setUintField<4>(joblist::INTNULL, 0); break; } - default: + case 8 : { r.setUintField<8>(42, 0); r.nextRow(rowSize); @@ -304,6 +308,21 @@ private: r.setUintField<8>(joblist::BIGINTNULL, 0); break; } + case 16 : + { + uint128_t dec = 42; + r.setBinaryField(&dec, 0); + r.nextRow(rowSize); + dec++; + r.setBinaryField(&dec, 0); + r.nextRow(rowSize); + r.setBinaryField( + const_cast(&datatypes::Decimal128Null), + 0); + break; + } + default : + std::cout << " This is the end. My only friend the end... " << std::endl; } break; } @@ -387,27 +406,33 @@ private: { //bool generateValues = true; bool fixedValues = false; - testComparatorWithDT(execplan::CalpontSystemCatalog::UTINYINT, 1, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::USMALLINT, 2, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::UMEDINT, 4, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::UBIGINT, 8, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::DATETIME, 8, fixedValues); + testComparatorWithDT(execplan::CalpontSystemCatalog::UTINYINT, 1, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::USMALLINT, 2, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::UMEDINT, 4, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::UBIGINT, 8, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::DATETIME, 8, fixedValues, 20); - testComparatorWithDT(execplan::CalpontSystemCatalog::TINYINT, 1, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::SMALLINT, 2, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::MEDINT, 4, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::DATE, 4, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::BIGINT, 8, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::DECIMAL, 8, fixedValues); + testComparatorWithDT(execplan::CalpontSystemCatalog::TINYINT, 1, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::SMALLINT, 2, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::MEDINT, 4, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::DATE, 4, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::BIGINT, 8, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::DECIMAL, 8, fixedValues, 20); - testComparatorWithDT(execplan::CalpontSystemCatalog::FLOAT, 4, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::DOUBLE, 8, fixedValues); - testComparatorWithDT(execplan::CalpontSystemCatalog::LONGDOUBLE, 8, fixedValues); + testComparatorWithDT(execplan::CalpontSystemCatalog::FLOAT, 4, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::DOUBLE, 8, fixedValues, 20); + testComparatorWithDT(execplan::CalpontSystemCatalog::LONGDOUBLE, 8, fixedValues, 20); } void FLOAT_TEST() { } + + void WIDEDT_TEST() + { + bool fixedValues = false; + testComparatorWithDT(execplan::CalpontSystemCatalog::DECIMAL, 16, fixedValues, 38); + } }; CPPUNIT_TEST_SUITE_REGISTRATION(FilterDriver); diff --git a/utils/dataconvert/CMakeLists.txt b/utils/dataconvert/CMakeLists.txt index 9a25dfb1b..3e37b3c0e 100644 --- a/utils/dataconvert/CMakeLists.txt +++ b/utils/dataconvert/CMakeLists.txt @@ -11,9 +11,3 @@ add_library(dataconvert SHARED ${dataconvert_LIB_SRCS}) add_dependencies(dataconvert loggingcpp) install(TARGETS dataconvert DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) - -if (WITH_DATACONVERT_UT) - add_executable(dataconvert_tests dataconvert-tests.cpp) - target_link_libraries(dataconvert_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) - install(TARGETS dataconvert_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) -endif() diff --git a/utils/rowgroup/CMakeLists.txt b/utils/rowgroup/CMakeLists.txt index 3fcde10df..876d620cf 100644 --- a/utils/rowgroup/CMakeLists.txt +++ b/utils/rowgroup/CMakeLists.txt @@ -15,9 +15,3 @@ add_dependencies(rowgroup loggingcpp) target_link_libraries(rowgroup ${NETSNMP_LIBRARIES} funcexp) install(TARGETS rowgroup DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) - -if (WITH_ROWGROUP_UT) - add_executable(rowgroup_tests rowgroup-tests.cpp) - target_link_libraries(rowgroup_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) - install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) -endif() diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index da01d0941..a8f84fbc8 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -66,7 +66,6 @@ typedef const struct charset_info_st CHARSET_INFO; // Workaround for my_global.h #define of isnan(X) causing a std::std namespace using int128_t = __int128; -using uint128_t = unsigned __int128; namespace rowgroup { diff --git a/utils/windowfunction/CMakeLists.txt b/utils/windowfunction/CMakeLists.txt index 636f72735..218a43803 100755 --- a/utils/windowfunction/CMakeLists.txt +++ b/utils/windowfunction/CMakeLists.txt @@ -29,9 +29,3 @@ add_library(windowfunction SHARED ${windowfunction_LIB_SRCS}) add_dependencies(windowfunction loggingcpp) install(TARGETS windowfunction DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine) - -if (WITH_SORTING_COMPARATORS_UT) - add_executable(comparators_tests comparators-tests.cpp) - target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) - install(TARGETS comparators_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) -endif() diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index f92e1a5c0..58b01ee38 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -47,6 +47,9 @@ using namespace rowgroup; #include "idborderby.h" #include "joblisttypes.h" +#include "mcs_decimal.h" + +using int128_t = __int128; #include "collation.h" @@ -166,6 +169,35 @@ int BigIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) return ret; } +int WideDecimalCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) +{ + l->row1().setData(r1); + l->row2().setData(r2); + + int ret = 0; + int128_t v1 = *(l->row1().getBinaryField_offset(keyColumnOffset)); + int128_t v2 = *(l->row2().getBinaryField_offset(keyColumnOffset)); + bool v1IsNull = v1 == datatypes::Decimal128Null; + bool v2IsNull = v2 == datatypes::Decimal128Null; + + if (v1IsNull || v2IsNull) + { + if (!v1IsNull && v2IsNull) + ret = fSpec.fNf; + else if (v1IsNull && !v2IsNull) + ret = -fSpec.fNf; + } + else + { + if (v1 > v2) + ret = fSpec.fAsc; + else if (v1 < v2) + ret = -fSpec.fAsc; + } + + return ret; +} + int UTinyIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2) { l->row1().setData(r1); @@ -537,6 +569,7 @@ void CompareRule::revertRules() void CompareRule::compileRules(const std::vector& spec, const rowgroup::RowGroup& rg) { const vector& types = rg.getColTypes(); + const auto& offsets = rg.getOffsets(); for (vector::const_iterator i = spec.begin(); i != spec.end(); i++) { @@ -574,14 +607,16 @@ void CompareRule::compileRules(const std::vector& spec, const rowgr uint32_t len = rg.getColumnWidth(i->fIndex); switch (len) { + case datatypes::MAXDECIMALWIDTH: + c = new WideDecimalCompare(*i, offsets[i->fIndex]); break; + case datatypes::MAXLEGACYWIDTH: + c = new BigIntCompare(*i); case 1 : c = new TinyIntCompare(*i); break; case 2 : c = new SmallIntCompare(*i); break; case 4 : c = new IntCompare(*i); break; - default: - c = new BigIntCompare(*i); } fCompares.push_back(c); diff --git a/utils/windowfunction/idborderby.h b/utils/windowfunction/idborderby.h index 98dce4b59..30f49c1de 100644 --- a/utils/windowfunction/idborderby.h +++ b/utils/windowfunction/idborderby.h @@ -143,6 +143,15 @@ public: int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); }; +class WideDecimalCompare : public Compare +{ + int keyColumnOffset; +public: + WideDecimalCompare(const IdbSortSpec& spec, int offset) : Compare(spec), keyColumnOffset(offset) { } + + int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer); +}; + // End of comparators for signed types // Comparators for unsigned types From a61c1906688a8ad367634328e3f413df5b62b6c0 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 9 Jun 2020 10:45:51 +0000 Subject: [PATCH 44/78] MCOL-641 Remove memset from group_concat. --- dbcon/joblist/groupconcat.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index 01f0a6c6b..942cd254f 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -939,8 +939,10 @@ void GroupConcatOrderBy::getResult(uint8_t* buff, const string& sep) } size_t resultSize = oss.str().size(); + resultSize = (resultSize > fGroupConcatLen) ? fGroupConcatLen : resultSize; fOutputString.reset(new uint8_t[resultSize + 2]); - memset(fOutputString.get(), 0, resultSize + 2); + fOutputString[resultSize] = '\0'; + fOutputString[resultSize + 1] = '\0'; strncpy((char*)fOutputString.get(), oss.str().c_str(), resultSize); From 8c02802ac1af20a61af308e8ea587c13b63e1ff7 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 25 Jun 2020 09:27:25 +0000 Subject: [PATCH 45/78] MCOL-641 Fix the cpimport issue when tables contains wide-DECIMAL and other types. --- writeengine/bulk/we_tableinfo.cpp | 123 +++++++++++++++++++++++------- writeengine/bulk/we_tableinfo.h | 7 ++ writeengine/shared/we_define.cpp | 1 + writeengine/shared/we_define.h | 1 + 4 files changed, 106 insertions(+), 26 deletions(-) diff --git a/writeengine/bulk/we_tableinfo.cpp b/writeengine/bulk/we_tableinfo.cpp index cb1a25d9a..06d96dafe 100644 --- a/writeengine/bulk/we_tableinfo.cpp +++ b/writeengine/bulk/we_tableinfo.cpp @@ -69,6 +69,31 @@ const std::string BOLD_STOP = "\033[0;39m"; namespace WriteEngine { +// Helpers +int TableInfo::compareHWMs(const int smallestColumnId, + const int widerColumnId, + const size_t widerColumnWidth, + const std::vector& segFileInfo, + int& colIdx) +{ + int rc = NO_ERROR; + if (widerColumnId < 0) + { + return rc; + } + HWM hwmLo = segFileInfo[smallestColumnId].fLocalHwm * widerColumnWidth; + HWM hwmHi = hwmLo + widerColumnWidth - 1; + + if ((segFileInfo[widerColumnId].fLocalHwm < hwmLo) || + (segFileInfo[widerColumnId].fLocalHwm > hwmHi)) + { + colIdx = widerColumnId; + rc = ERR_BRM_HWMS_OUT_OF_SYNC; + } + return rc; +} + + //------------------------------------------------------------------------------ // Puts the current thread to sleep for the specified number of milliseconds. // (Ex: used to wait for a Read buffer to become available.) @@ -2064,6 +2089,7 @@ int TableInfo::validateColumnHWMs( int byte2First = -1; int byte4First = -1; int byte8First = -1; + int byte16First = -1; // Make sure the HWMs for all 1-byte columns match; same for all 2-byte, // 4-byte, and 8-byte columns as well. @@ -2107,7 +2133,6 @@ int TableInfo::validateColumnHWMs( } case 8: - default: { if (byte8First == -1) byte8First = k; @@ -2115,16 +2140,34 @@ int TableInfo::validateColumnHWMs( k1 = byte8First; break; } - } // end of switch based on column width (1,2,4, or 8) + case 16: + { + if (byte16First == -1) + byte16First = k; + + k1 = byte16First; + break; + } + default: + { + ostringstream oss; + oss << stage << " Unsupported width for" + " OID-" << jobColK.mapOid << + "; column-" << jobColK.colName << + "; width-" << jobColK.width; + fLog->logMsg( oss.str(), ERR_BRM_UNSUPP_WIDTH, MSGLVL_ERROR ); + return ERR_BRM_UNSUPP_WIDTH; + } + } // end of switch based on column width. // Validate HWMs in jobTable if we have it, else use fColumns. const JobColumn& jobColK1 = ( (jobTable != 0) ? jobTable->colList[k1] : fColumns[k1].column ); -//std::cout << "dbg: comparing0 " << stage << " refcol-" << k1 << -// "; wid-" << jobColK1.width << "; hwm-" << segFileInfo[k1].fLocalHwm << -// " col-" << k << -// "; wid-" << jobColK.width << " ; hwm-"< col-" << k << + // "; wid-" << jobColK.width << " ; hwm-"<= 0) -// std::cout << "dbg: cross compare1 " << stage << " col-" << byte1First << -// "; wid-" << ( (jobTable != 0) ? jobTable->colList[byte1First].width : -// fColumns[byte1First].column.width ) << -// "; hwm-" << segFileInfo[byte1First].fLocalHwm << std::endl; + //if (byte1First >= 0) + // std::cout << "dbg: cross compare1 " << stage << " col-" << byte1First << + // "; wid-" << ( (jobTable != 0) ? jobTable->colList[byte1First].width : + // fColumns[byte1First].column.width ) << + // "; hwm-" << segFileInfo[byte1First].fLocalHwm << std::endl; -//if (byte2First >= 0) -// std::cout << "dbg: cross compare2 " << stage << " col-" << byte2First << -// "; wid-" << ( (jobTable != 0) ? jobTable->colList[byte2First].width : -// fColumns[byte2First].column.width ) << -// "; hwm-" << segFileInfo[byte2First].fLocalHwm << std::endl; + //if (byte2First >= 0) + // std::cout << "dbg: cross compare2 " << stage << " col-" << byte2First << + // "; wid-" << ( (jobTable != 0) ? jobTable->colList[byte2First].width : + // fColumns[byte2First].column.width ) << + // "; hwm-" << segFileInfo[byte2First].fLocalHwm << std::endl; -//if (byte4First >= 0) -// std::cout << "dbg: cross compare4 " << stage << " col-" << byte4First << -// "; wid-" << ( (jobTable != 0) ? jobTable->colList[byte4First].width : -// fColumns[byte4First].column.width ) << -// "; hwm-" << segFileInfo[byte4First].fLocalHwm << std::endl; + //if (byte4First >= 0) + // std::cout << "dbg: cross compare4 " << stage << " col-" << byte4First << + // "; wid-" << ( (jobTable != 0) ? jobTable->colList[byte4First].width : + // fColumns[byte4First].column.width ) << + // "; hwm-" << segFileInfo[byte4First].fLocalHwm << std::endl; -//if (byte8First >= 0) -// std::cout << "dbg: cross compare8 " << stage << " col-" << byte8First << -// "; wid-" << ( (jobTable != 0) ? jobTable->colList[byte8First].width : -// fColumns[byte8First].column.width ) << -// "; hwm-" << segFileInfo[byte8First].fLocalHwm << std::endl; + //if (byte8First >= 0) + // std::cout << "dbg: cross compare8 " << stage << " col-" << byte8First << + // "; wid-" << ( (jobTable != 0) ? jobTable->colList[byte8First].width : + // fColumns[byte8First].column.width ) << + // "; hwm-" << segFileInfo[byte8First].fLocalHwm << std::endl; // Validate/compare HWMs given a 1-byte column as a starting point if (byte1First >= 0) @@ -2259,6 +2302,11 @@ int TableInfo::validateColumnHWMs( goto errorCheck; } } + if ((rc = compareHWMs(byte1First, byte16First, 16, + segFileInfo, colIdx) < 0)) + { + goto errorCheck; + } } // Validate/compare HWMs given a 2-byte column as a starting point @@ -2293,6 +2341,13 @@ int TableInfo::validateColumnHWMs( goto errorCheck; } } + if ((rc = compareHWMs(byte2First, byte16First, 16, + segFileInfo, colIdx) < 0)) + { + goto errorCheck; + } + + } // Validate/compare HWMs given a 4-byte column as a starting point @@ -2313,6 +2368,22 @@ int TableInfo::validateColumnHWMs( goto errorCheck; } } + if ((rc = compareHWMs(byte4First, byte16First, 16, + segFileInfo, colIdx) < 0)) + { + goto errorCheck; + } + + } + if (byte8First >= 0) + { + refCol = byte8First; + if ((rc = compareHWMs(byte8First, byte16First, 16, + segFileInfo, colIdx) < 0)) + { + goto errorCheck; + } + } // To avoid repeating this message 6 times in the preceding source code, we diff --git a/writeengine/bulk/we_tableinfo.h b/writeengine/bulk/we_tableinfo.h index eddc552a2..b0c635461 100644 --- a/writeengine/bulk/we_tableinfo.h +++ b/writeengine/bulk/we_tableinfo.h @@ -187,6 +187,13 @@ private: int openTableFile(); // Open data file and set the buffer void reportTotals(double elapsedSec);//Report summary totals void sleepMS(long int ms); // Sleep method + // Compare column HWM with the examplar HWM. + int compareHWMs(const int smallestColumnId, + const int widerColumnId, + const size_t widerColumnWidth, + const std::vector& segFileInfo, + int& colIdx); + int synchronizeAutoInc(); // Sychronize AutoInc in BRM with syscat // Write the list of errors for this table diff --git a/writeengine/shared/we_define.cpp b/writeengine/shared/we_define.cpp index 6ec008826..704600f60 100644 --- a/writeengine/shared/we_define.cpp +++ b/writeengine/shared/we_define.cpp @@ -205,6 +205,7 @@ WErrorCodes::WErrorCodes() : fErrorCodes() fErrorCodes[ERR_BRM_SUSPEND] = " The system is in write suspended mode"; fErrorCodes[ERR_BRM_GET_SUSPEND] = " BRM error get the system suspend flag "; fErrorCodes[ERR_BRM_BAD_STRIPE_CNT] = " Incorrect number of column extents allocated in stripe"; + fErrorCodes[ERR_BRM_UNSUPP_WIDTH] = " Unsupported non-dictionary column width"; // DM error fErrorCodes[ERR_DM_CONVERT_OID] = " a DM Conversion error"; diff --git a/writeengine/shared/we_define.h b/writeengine/shared/we_define.h index da6337ca4..9a16b7444 100644 --- a/writeengine/shared/we_define.h +++ b/writeengine/shared/we_define.h @@ -308,6 +308,7 @@ const int ERR_BRM_GET_SHUTDOWN = ERR_BRMBASE + 43;// error getting BRM Shut const int ERR_BRM_SUSPEND = ERR_BRMBASE + 44;// BRM is set to Suspend writes const int ERR_BRM_GET_SUSPEND = ERR_BRMBASE + 45;// error getting BRM Suspend flag const int ERR_BRM_BAD_STRIPE_CNT = ERR_BRMBASE + 46;// Incorrect num of cols allocated in stripe +const int ERR_BRM_UNSUPP_WIDTH = ERR_BRMBASE + 47;// Non-dict column Width > allowed MAX. //-------------------------------------------------------------------------- // DM error From bd0d5af123547e44389e90e19f947c4104f8ae94 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 25 Jun 2020 14:16:13 +0000 Subject: [PATCH 46/78] Merge fixes. --- dbcon/execplan/predicateoperator.h | 399 ----------------------------- utils/funcexp/func_char.cpp | 11 +- utils/rowgroup/rowaggregation.cpp | 1 - utils/rowgroup/rowgroup.cpp | 37 ++- utils/rowgroup/rowgroup.h | 79 ------ 5 files changed, 32 insertions(+), 495 deletions(-) diff --git a/dbcon/execplan/predicateoperator.h b/dbcon/execplan/predicateoperator.h index 66120eb9c..f28b289f1 100644 --- a/dbcon/execplan/predicateoperator.h +++ b/dbcon/execplan/predicateoperator.h @@ -128,405 +128,6 @@ private: const CHARSET_INFO* cs; }; -inline bool PredicateOperator::getBoolVal(rowgroup::Row& row, bool& isNull, ReturnedColumn* lop, ReturnedColumn* rop) -{ - // like operator. both sides are string. - if (fOp == OP_LIKE || fOp == OP_NOTLIKE) - { - SP_CNX_Regex regex = rop->regex(); - - // Ugh. The strings returned by getStrVal have null padding out to the col width. boost::regex - // considers these nulls significant, but they're not in the pattern, so we need to strip - // them off... - const std::string& v = lop->getStrVal(row, isNull); -// char* c = (char*)alloca(v.length() + 1); -// memcpy(c, v.c_str(), v.length()); -// c[v.length()] = 0; -// std::string vv(c); - - if (regex) - { -#ifdef POSIX_REGEX - bool ret = regexec(regex.get(), v.c_str(), 0, NULL, 0) == 0; -#else - bool ret = boost::regex_match(v.c_str(), *regex); -#endif - return (((fOp == OP_LIKE) ? ret : !ret) && !isNull); - } - else - { -#ifdef POSIX_REGEX - regex_t regex; - std::string str = dataconvert::DataConvert::constructRegexp(rop->getStrVal(row, isNull)); - regcomp(®ex, str.c_str(), REG_NOSUB | REG_EXTENDED); - bool ret = regexec(®ex, v.c_str(), 0, NULL, 0) == 0; - regfree(®ex); -#else - boost::regex regex(dataconvert::DataConvert::constructRegexp(rop->getStrVal(row, isNull))); - bool ret = boost::regex_match(v.c_str(), regex); -#endif - return (((fOp == OP_LIKE) ? ret : !ret) && !isNull); - } - } - - // fOpType should have already been set on the connector during parsing - switch (fOperationType.colDataType) - { - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - { - if (fOp == OP_ISNULL) - { - lop->getIntVal(row, isNull); - bool ret = isNull; - isNull = false; - return ret; - } - - if (fOp == OP_ISNOTNULL) - { - lop->getIntVal(row, isNull); - bool ret = isNull; - isNull = false; - return !ret; - } - - if (isNull) - return false; - - int64_t val1 = lop->getIntVal(row, isNull); - - if (isNull) - return false; - - return numericCompare(val1, rop->getIntVal(row, isNull)) && !isNull; - } - - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - { - if (fOp == OP_ISNULL) - { - lop->getUintVal(row, isNull); - bool ret = isNull; - isNull = false; - return ret; - } - - if (fOp == OP_ISNOTNULL) - { - lop->getUintVal(row, isNull); - bool ret = isNull; - isNull = false; - return !ret; - } - - if (isNull) - return false; - - uint64_t val1 = lop->getUintVal(row, isNull); - - if (isNull) - return false; - - return numericCompare(val1, rop->getUintVal(row, isNull)) && !isNull; - } - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - if (fOp == OP_ISNULL) - { - lop->getDoubleVal(row, isNull); - bool ret = isNull; - isNull = false; - return ret; - } - - if (fOp == OP_ISNOTNULL) - { - lop->getDoubleVal(row, isNull); - bool ret = isNull; - isNull = false; - return !ret; - } - - if (isNull) - return false; - - double val1 = lop->getDoubleVal(row, isNull); - - if (isNull) - return false; - - return numericCompare(val1, rop->getDoubleVal(row, isNull)) && !isNull; - } - - case execplan::CalpontSystemCatalog::LONGDOUBLE: - { - if (fOp == OP_ISNULL) - { - lop->getLongDoubleVal(row, isNull); - bool ret = isNull; - isNull = false; - return ret; - } - - if (fOp == OP_ISNOTNULL) - { - lop->getLongDoubleVal(row, isNull); - bool ret = isNull; - isNull = false; - return !ret; - } - - if (isNull) - return false; - - long double val1 = lop->getLongDoubleVal(row, isNull); - if (isNull) - return false; - - long double val2 = rop->getLongDoubleVal(row, isNull); - if (isNull) - return false; - - // In many case, rounding error will prevent an eq compare to work - // In these cases, use the largest scale of the two items. - if (fOp == execplan::OP_EQ) - { - // In case a val is a representation of a very large integer, - // we won't want to just multiply by scale, as it may move - // significant digits out of scope. So we break them apart - // and compare each separately - int64_t scale = std::max(lop->resultType().scale, rop->resultType().scale); - if (scale) - { - long double intpart1; - long double fract1 = modfl(val1, &intpart1); - long double intpart2; - long double fract2 = modfl(val2, &intpart2); - if (numericCompare(intpart1, intpart2)) - { - double factor = pow(10.0, (double)scale); - fract1 = roundl(fract1 * factor); - fract2 = roundl(fract2 * factor); - return numericCompare(fract1, fract2); - } - else - { - return false; - } - } - } - return numericCompare(val1, val2); - } - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - if (fOp == OP_ISNULL) - { - lop->getDecimalVal(row, isNull); - bool ret = isNull; - isNull = false; - return ret; - } - - if (fOp == OP_ISNOTNULL) - { - lop->getDecimalVal(row, isNull); - bool ret = isNull; - isNull = false; - return !ret; - } - - if (isNull) - return false; - - IDB_Decimal val1 = lop->getDecimalVal(row, isNull); - - if (isNull) - return false; - - return numericCompare(val1, rop->getDecimalVal(row, isNull)) && !isNull; - } - - case execplan::CalpontSystemCatalog::DATE: - { - if (fOp == OP_ISNULL) - { - lop->getDateIntVal(row, isNull); - bool ret = isNull; - isNull = false; - return ret; - } - - if (fOp == OP_ISNOTNULL) - { - lop->getDateIntVal(row, isNull); - bool ret = isNull; - isNull = false; - return !ret; - } - - if (isNull) - return false; - - int64_t val1 = lop->getDateIntVal(row, isNull); - - if (isNull) - return false; - - return numericCompare(val1, (int64_t)rop->getDateIntVal(row, isNull)) && !isNull; - } - - case execplan::CalpontSystemCatalog::DATETIME: - { - if (fOp == OP_ISNULL) - { - lop->getDatetimeIntVal(row, isNull); - bool ret = isNull; - isNull = false; - return ret; - } - - if (fOp == OP_ISNOTNULL) - { - lop->getDatetimeIntVal(row, isNull); - bool ret = isNull; - isNull = false; - return !ret; - } - - if (isNull) - return false; - - int64_t val1 = lop->getDatetimeIntVal(row, isNull); - - if (isNull) - return false; - - return numericCompare(val1, rop->getDatetimeIntVal(row, isNull)) && !isNull; - } - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - if (fOp == OP_ISNULL) - { - lop->getTimestampIntVal(row, isNull); - bool ret = isNull; - isNull = false; - return ret; - } - - if (fOp == OP_ISNOTNULL) - { - lop->getTimestampIntVal(row, isNull); - bool ret = isNull; - isNull = false; - return !ret; - } - - if (isNull) - return false; - - int64_t val1 = lop->getTimestampIntVal(row, isNull); - - if (isNull) - return false; - - return numericCompare(val1, rop->getTimestampIntVal(row, isNull)) && !isNull; - } - - case execplan::CalpontSystemCatalog::TIME: - { - if (fOp == OP_ISNULL) - { - lop->getTimeIntVal(row, isNull); - bool ret = isNull; - isNull = false; - return ret; - } - - if (fOp == OP_ISNOTNULL) - { - lop->getTimeIntVal(row, isNull); - bool ret = isNull; - isNull = false; - return !ret; - } - - if (isNull) - return false; - - int64_t val1 = lop->getTimeIntVal(row, isNull); - - if (isNull) - return false; - - return numericCompare(val1, rop->getTimeIntVal(row, isNull)) && !isNull; - } - - - - case execplan::CalpontSystemCatalog::VARCHAR: - case execplan::CalpontSystemCatalog::CHAR: - case execplan::CalpontSystemCatalog::TEXT: - { - if (fOp == OP_ISNULL) - { - lop->getStrVal(row, isNull); - bool ret = isNull; - isNull = false; - return ret; - } - - if (fOp == OP_ISNOTNULL) - { - lop->getStrVal(row, isNull); - bool ret = isNull; - isNull = false; - return !ret; - } - - if (isNull) - return false; - - const std::string& val1 = lop->getStrVal(row, isNull); - if (isNull) - return false; - - return strTrimCompare(val1, rop->getStrVal(row, isNull)) && !isNull; -// return strCompare(val1, rop->getStrVal(row, isNull)) && !isNull; - - } - - // MCOL-641 WIP This is an incorrect assumption. - case execplan::CalpontSystemCatalog::VARBINARY: - case execplan::CalpontSystemCatalog::BLOB: - return false; - break; - - default: - { - std::ostringstream oss; - oss << "invalid predicate operation type: " << fOperationType.colDataType; - throw logging::InvalidOperationExcept(oss.str()); - } - } - - return false; -} - inline bool PredicateOperator::numericCompare(IDB_Decimal& op1, IDB_Decimal& op2) { switch (fOp) diff --git a/utils/funcexp/func_char.cpp b/utils/funcexp/func_char.cpp index d9904ba63..d3bec2ae9 100644 --- a/utils/funcexp/func_char.cpp +++ b/utils/funcexp/func_char.cpp @@ -155,12 +155,12 @@ string Func_char::getStrVal(Row& row, if (tmpval > static_cast(INT64_MAX)) tmpval = INT64_MAX; - - if ( !getChar((int64_t)tmpval, buf) ) + // WIP MCOL-641 + /*if ( !getChar((int64_t)tmpval, buf) ) { isNull = true; return ""; - } + }*/ } else { @@ -172,11 +172,12 @@ string Func_char::getStrVal(Row& row, if ( lefto > 4 ) value++; - if ( !getChar((int64_t)value, buf) ) + // WIP MCOL-641 + /*if ( !getChar((int64_t)value, buf) ) { isNull = true; return ""; - } + }*/ } } break; diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index dd3c79caf..f39dea607 100755 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -57,7 +57,6 @@ //..comment out NDEBUG to enable assertions, uncomment NDEBUG to disable //#define NDEBUG -#include "funcexp/utils_utf8.h" #include "mcs_decimal.h" using namespace std; diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index e8dc38f50..7a8ec8a28 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -1175,10 +1175,11 @@ bool Row::equals(const Row& r2, const std::vector& keyCols) const for (uint32_t i = 0; i < keyCols.size(); i++) { const uint32_t& col = keyCols[i]; + cscDataType columnType = getColType(col); - if (UNLIKELY(getColType(col) == execplan::CalpontSystemCatalog::VARCHAR || - (getColType(col) == execplan::CalpontSystemCatalog::CHAR && (colWidths[col] > 1)) || - getColType(col) == execplan::CalpontSystemCatalog::TEXT)) + if (UNLIKELY(columnType == execplan::CalpontSystemCatalog::VARCHAR || + (columnType == execplan::CalpontSystemCatalog::CHAR && (colWidths[col] > 1)) || + columnType == execplan::CalpontSystemCatalog::TEXT)) { CHARSET_INFO* cs = getCharset(col); if (cs->strnncollsp(getStringPointer(col), getStringLength(col), @@ -1187,7 +1188,7 @@ bool Row::equals(const Row& r2, const std::vector& keyCols) const return false; } } - else if (UNLIKELY(getColType(col) == execplan::CalpontSystemCatalog::BLOB)) + else if (UNLIKELY(columnType == execplan::CalpontSystemCatalog::BLOB)) { if (getStringLength(col) != r2.getStringLength(col)) return false; @@ -1197,19 +1198,25 @@ bool Row::equals(const Row& r2, const std::vector& keyCols) const } else { - if (getColType(col) == execplan::CalpontSystemCatalog::LONGDOUBLE) + if (UNLIKELY(columnType == execplan::CalpontSystemCatalog::LONGDOUBLE)) { if (getLongDoubleField(col) != r2.getLongDoubleField(col)) return false; } + else if (UNLIKELY(execplan::isDecimal(columnType))) + { + if (getBinaryField(col) != r2.getBinaryField(col)) + return false; + } else if (getUintField(col) != r2.getUintField(col)) + { return false; + } } } return true; } - bool Row::equals(const Row& r2, uint32_t lastCol) const { // This check fires with empty r2 only. @@ -1227,9 +1234,10 @@ bool Row::equals(const Row& r2, uint32_t lastCol) const // because binary equality is not equality for many charsets/collations for (uint32_t col = 0; col <= lastCol; col++) { - if (UNLIKELY(getColType(col) == execplan::CalpontSystemCatalog::VARCHAR || - (getColType(col) == execplan::CalpontSystemCatalog::CHAR && (colWidths[col] > 1)) || - getColType(col) == execplan::CalpontSystemCatalog::TEXT)) + cscDataType columnType = getColType(col); + if (UNLIKELY(columnType == execplan::CalpontSystemCatalog::VARCHAR || + (columnType == execplan::CalpontSystemCatalog::CHAR && (colWidths[col] > 1)) || + columnType == execplan::CalpontSystemCatalog::TEXT)) { CHARSET_INFO* cs = getCharset(col); if (cs->strnncollsp(getStringPointer(col), getStringLength(col), @@ -1238,7 +1246,7 @@ bool Row::equals(const Row& r2, uint32_t lastCol) const return false; } } - else if (UNLIKELY(getColType(col) == execplan::CalpontSystemCatalog::BLOB)) + else if (UNLIKELY(columnType == execplan::CalpontSystemCatalog::BLOB)) { if (getStringLength(col) != r2.getStringLength(col)) return false; @@ -1248,13 +1256,20 @@ bool Row::equals(const Row& r2, uint32_t lastCol) const } else { - if (getColType(col) == execplan::CalpontSystemCatalog::LONGDOUBLE) + if (UNLIKELY(columnType == execplan::CalpontSystemCatalog::LONGDOUBLE)) { if (getLongDoubleField(col) != r2.getLongDoubleField(col)) return false; } + else if (UNLIKELY(execplan::isDecimal(columnType))) + { + if (getBinaryField(col) != r2.getBinaryField(col)) + return false; + } else if (getUintField(col) != r2.getUintField(col)) + { return false; + } } } return true; diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index a8f84fbc8..22d47e16e 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -1301,85 +1301,6 @@ inline uint64_t Row::hash(uint32_t lastCol) const return ret; } -inline bool Row::equals(const Row& r2, const std::vector& keyCols) const -{ - for (uint32_t i = 0; i < keyCols.size(); i++) - { - const uint32_t& col = keyCols[i]; - - cscDataType columnType = getColType(i); - - if (!isLongString(col)) - { - if (UNLIKELY(columnType == execplan::CalpontSystemCatalog::LONGDOUBLE)) - { - if (getLongDoubleField(i) != r2.getLongDoubleField(i)) - return false; - } - else if (UNLIKELY(execplan::isDecimal(columnType))) - { - if (getBinaryField(i) != r2.getBinaryField(i)) - return false; - } - else if (getUintField(col) != r2.getUintField(col)) - return false; - } - else - { - if (getStringLength(col) != r2.getStringLength(col)) - return false; - - if (memcmp(getStringPointer(col), r2.getStringPointer(col), getStringLength(col))) - return false; - } - } - - return true; -} - -inline bool Row::equals(const Row& r2, uint32_t lastCol) const -{ - // This check fires with empty r2 only. - if (lastCol >= columnCount) - return true; - - if (!useStringTable && !r2.useStringTable) - return !(memcmp(&data[offsets[0]], &r2.data[offsets[0]], offsets[lastCol + 1] - offsets[0])); - - for (uint32_t i = 0; i <= lastCol; i++) - { - cscDataType columnType = getColType(i); - if (!isLongString(i)) - { - if (UNLIKELY(getColType(i) == execplan::CalpontSystemCatalog::LONGDOUBLE)) - { - if (getLongDoubleField(i) != r2.getLongDoubleField(i)) - return false; - } - else if (UNLIKELY(execplan::isDecimal(columnType))) - { - if (*getBinaryField(i) != *r2.getBinaryField(i)) - return false; - } - - else if (getUintField(i) != r2.getUintField(i)) - return false; - } - else - { - uint32_t len = getStringLength(i); - - if (len != r2.getStringLength(i)) - return false; - - if (memcmp(getStringPointer(i), r2.getStringPointer(i), len)) - return false; - } - } - - return true; -} - inline bool Row::equals(const Row& r2) const { return equals(r2, columnCount - 1); From 6aea838360592c34852fa49adaaf940102f4007f Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Mon, 13 Jul 2020 17:52:24 -0400 Subject: [PATCH 47/78] MCOL-641 Add support for functions (Part 2). --- datatypes/mcs_decimal.h | 39 +++++ dbcon/execplan/constantcolumn.cpp | 6 +- dbcon/execplan/constantcolumn.h | 3 +- dbcon/execplan/functioncolumn.h | 3 +- dbcon/execplan/simplecolumn_uint.h | 3 +- dbcon/execplan/treenode.h | 5 +- dbcon/mysql/ha_mcs_execplan.cpp | 3 +- primitives/primproc/columncommand.h | 2 +- utils/common/widedecimalutils.h | 4 +- utils/funcexp/func_between.cpp | 6 +- utils/funcexp/func_bitwise.cpp | 206 +++++++++++++++++++++----- utils/funcexp/func_case.cpp | 26 ++-- utils/funcexp/func_ceil.cpp | 53 ++++++- utils/funcexp/func_char.cpp | 5 +- utils/funcexp/func_date_add.cpp | 5 +- utils/funcexp/func_date_format.cpp | 6 + utils/funcexp/func_day.cpp | 5 + utils/funcexp/func_dayname.cpp | 6 + utils/funcexp/func_dayofweek.cpp | 6 + utils/funcexp/func_dayofyear.cpp | 3 +- utils/funcexp/func_elt.cpp | 1 + utils/funcexp/func_from_unixtime.cpp | 16 +- utils/funcexp/func_hex.cpp | 1 + utils/funcexp/func_hour.cpp | 7 +- utils/funcexp/func_if.cpp | 7 +- utils/funcexp/func_inet_ntoa.cpp | 2 + utils/funcexp/func_last_day.cpp | 6 + utils/funcexp/func_makedate.cpp | 2 + utils/funcexp/func_maketime.cpp | 3 + utils/funcexp/func_math.cpp | 158 ++++++++++++++------ utils/funcexp/func_microsecond.cpp | 6 + utils/funcexp/func_minute.cpp | 7 +- utils/funcexp/func_month.cpp | 5 + utils/funcexp/func_monthname.cpp | 1 + utils/funcexp/func_nullif.cpp | 6 + utils/funcexp/func_quarter.cpp | 1 + utils/funcexp/func_regexp.cpp | 18 ++- utils/funcexp/func_sec_to_time.cpp | 1 + utils/funcexp/func_second.cpp | 7 +- utils/funcexp/func_str_to_date.cpp | 6 + utils/funcexp/func_time.cpp | 7 +- utils/funcexp/func_time_format.cpp | 6 + utils/funcexp/func_time_to_sec.cpp | 6 + utils/funcexp/func_timediff.cpp | 2 + utils/funcexp/func_unix_timestamp.cpp | 6 + utils/funcexp/func_week.cpp | 6 + utils/funcexp/func_weekday.cpp | 6 + utils/funcexp/func_year.cpp | 1 + utils/funcexp/func_yearweek.cpp | 6 + utils/funcexp/funcexp.cpp | 1 + 50 files changed, 578 insertions(+), 125 deletions(-) diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index 1a905bd72..fa2ed3849 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -285,6 +285,45 @@ class Decimal return static_cast(value); } + /** + @brief The method converts a wide decimal value to an uint32_t. + */ + static inline uint32_t getUInt32FromWideDecimal(const int128_t& value) + { + if (value > static_cast(UINT32_MAX)) + return UINT32_MAX; + else if (value < 0) + return 0; + + return static_cast(value); + } + + /** + @brief The method converts a wide decimal value to an int32_t. + */ + static inline int32_t getInt32FromWideDecimal(const int128_t& value) + { + if (value > static_cast(INT32_MAX)) + return INT32_MAX; + else if (value < static_cast(INT32_MIN)) + return INT32_MIN; + + return static_cast(value); + } + + /** + @brief The method converts a wide decimal value to an uint64_t. + */ + static inline uint64_t getUInt64FromWideDecimal(const int128_t& value) + { + if (value > static_cast(UINT64_MAX)) + return UINT64_MAX; + else if (value < 0) + return 0; + + return static_cast(value); + } + /** @brief The method converts a __float128 value to a double. */ diff --git a/dbcon/execplan/constantcolumn.cpp b/dbcon/execplan/constantcolumn.cpp index 775ad88ed..f92a59dc6 100644 --- a/dbcon/execplan/constantcolumn.cpp +++ b/dbcon/execplan/constantcolumn.cpp @@ -236,7 +236,8 @@ ConstantColumn::ConstantColumn(const int64_t val, TYPE type) : fResultType.colWidth = 8; } -ConstantColumn::ConstantColumn(const uint64_t val, TYPE type) : +ConstantColumn::ConstantColumn(const uint64_t val, TYPE type, + int8_t scale, uint8_t precision) : ReturnedColumn(), fType(type) { @@ -252,7 +253,8 @@ ConstantColumn::ConstantColumn(const uint64_t val, TYPE type) : fResult.longDoubleVal = (long double)fResult.uintVal; fResult.decimalVal.value = fResult.uintVal; fResult.decimalVal.s128Value = fResult.uintVal; - fResult.decimalVal.scale = 0; + fResult.decimalVal.scale = scale; + fResult.decimalVal.precision = precision; fResultType.colDataType = CalpontSystemCatalog::UBIGINT; fResultType.colWidth = 8; } diff --git a/dbcon/execplan/constantcolumn.h b/dbcon/execplan/constantcolumn.h index a5c0a074b..334247c47 100644 --- a/dbcon/execplan/constantcolumn.h +++ b/dbcon/execplan/constantcolumn.h @@ -73,7 +73,8 @@ public: /** * ctor */ - ConstantColumn(const uint64_t val, TYPE type = NUM); // deprecate + ConstantColumn(const uint64_t val, TYPE type = NUM, + int8_t scale = 0, uint8_t precision = 0); // deprecate //There are more ctors below... /** diff --git a/dbcon/execplan/functioncolumn.h b/dbcon/execplan/functioncolumn.h index 4d398d9a8..874e7ef67 100644 --- a/dbcon/execplan/functioncolumn.h +++ b/dbcon/execplan/functioncolumn.h @@ -259,11 +259,12 @@ public: int128_t scaleMultiplier; int32_t scaleDiff = fResultType.scale - decimal.scale; datatypes::getScaleDivisor(scaleMultiplier, abs(scaleDiff)); + if (scaleMultiplier > 1) { if (scaleDiff > 0) { - // WIP MCOL-641 Unconditionall overflow check + // WIP MCOL-641 Unconditional overflow check datatypes::MultiplicationNoOverflowCheck mul; mul(decimal.s128Value, scaleMultiplier, decimal.s128Value); } diff --git a/dbcon/execplan/simplecolumn_uint.h b/dbcon/execplan/simplecolumn_uint.h index 7a6da72e4..f8b8a5c96 100644 --- a/dbcon/execplan/simplecolumn_uint.h +++ b/dbcon/execplan/simplecolumn_uint.h @@ -219,8 +219,7 @@ inline IDB_Decimal SimpleColumn_UINT::getDecimalVal(rowgroup::Row& row, boo isNull = true; fResult.decimalVal.value = (uint64_t)row.getUintField(fInputIndex); - // WIP MCOL-641 - fResult.decimalVal.precision = datatypes::INT64MAXPRECISION+1; + fResult.decimalVal.precision = datatypes::INT64MAXPRECISION; fResult.decimalVal.scale = 0; return fResult.decimalVal; } diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 56f21ccdd..b23024990 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -725,7 +725,10 @@ inline bool TreeNode::getBoolVal() case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - return (fResult.decimalVal.value != 0); + if (fResultType.colWidth == datatypes::MAXDECIMALWIDTH) + return (fResult.decimalVal.s128Value != 0); + else + return (fResult.decimalVal.value != 0); default: throw logging::InvalidConversionExcept("TreeNode::getBoolVal: Invalid conversion."); diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 6bc80e42c..2c4c2d775 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -3173,7 +3173,8 @@ ReturnedColumn* buildReturnedColumn( if (item->unsigned_flag) { - rc = new ConstantColumn((uint64_t)item->val_uint(), ConstantColumn::NUM); + rc = new ConstantColumn((uint64_t)item->val_uint(), ConstantColumn::NUM, + (int8_t) item->decimal_scale(), (uint8_t) item->decimal_precision()); } else { diff --git a/primitives/primproc/columncommand.h b/primitives/primproc/columncommand.h index 1c1a29cb9..d868ce2f9 100644 --- a/primitives/primproc/columncommand.h +++ b/primitives/primproc/columncommand.h @@ -84,7 +84,7 @@ public: makeAbsRids = m; } bool willPrefetch(); - const int64_t getLastLbid(); + int64_t getLastLbid(); void getLBIDList(uint32_t loopCount, std::vector* lbids); virtual SCommand duplicate(); diff --git a/utils/common/widedecimalutils.h b/utils/common/widedecimalutils.h index 3ebdb408b..2ed5b7309 100644 --- a/utils/common/widedecimalutils.h +++ b/utils/common/widedecimalutils.h @@ -58,7 +58,7 @@ namespace utils ptr[1] = BINARYEMPTYVALUEHIGH; } - inline void setWideDecimalNullValue(int128_t* val) + inline void setWideDecimalNullValue(int128_t* val) { uint64_t* ptr = reinterpret_cast(val); ptr[0] = BINARYNULLVALUELOW; @@ -72,7 +72,7 @@ namespace utils ptr[1] = BINARYEMPTYVALUEHIGH; } - inline void int128Max(int128_t& val) + inline void int128Max(int128_t& val) { uint64_t* ptr = reinterpret_cast(&val); ptr[0] = 0xFFFFFFFFFFFFFFFF; diff --git a/utils/funcexp/func_between.cpp b/utils/funcexp/func_between.cpp index ab6aef028..f4c0eabf6 100644 --- a/utils/funcexp/func_between.cpp +++ b/utils/funcexp/func_between.cpp @@ -237,6 +237,9 @@ inline bool getBool(rowgroup::Row& row, { IDB_Decimal val = pm[0]->data()->getDecimalVal(row, isNull); + if (isNull) + return false; + if (notBetween) { if (!numericGE(val, pm[1]->data()->getDecimalVal(row, isNull)) && !isNull) @@ -246,8 +249,7 @@ inline bool getBool(rowgroup::Row& row, return (!numericLE(val, pm[2]->data()->getDecimalVal(row, isNull)) && !isNull); } - return !isNull && - numericGE(val, pm[1]->data()->getDecimalVal(row, isNull)) && + return numericGE(val, pm[1]->data()->getDecimalVal(row, isNull)) && numericLE(val, pm[2]->data()->getDecimalVal(row, isNull)); } diff --git a/utils/funcexp/func_bitwise.cpp b/utils/funcexp/func_bitwise.cpp index beef7d290..5eb48252e 100644 --- a/utils/funcexp/func_bitwise.cpp +++ b/utils/funcexp/func_bitwise.cpp @@ -56,8 +56,12 @@ bool getUIntValFromParm( const execplan::SPTP& parm, uint64_t& value, bool& isNull, - const string& timeZone) + const string& timeZone, + bool& isBigVal, + int128_t& bigval) { + isBigVal = false; + switch (parm->data()->resultType().colDataType) { case execplan::CalpontSystemCatalog::BIGINT: @@ -104,10 +108,12 @@ bool getUIntValFromParm( if (parm->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) { + isBigVal = true; + if (parm->data()->resultType().colDataType == execplan::CalpontSystemCatalog::UDECIMAL && d.value < 0) { - value = 0; + bigval = 0; break; } @@ -126,12 +132,7 @@ bool getUIntValFromParm( if (tmpval < 0 && lefto < -4) tmpval--; - if (tmpval > static_cast(INT64_MAX)) - tmpval = INT64_MAX; - else if (tmpval < static_cast(INT64_MIN)) - tmpval = INT64_MIN; - - value = tmpval; + bigval = tmpval; } else { @@ -229,15 +230,39 @@ int64_t Func_bitand::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone()) || - !getUIntValFromParm(row, parm[1], val2, isNull, timeZone())) + int128_t bigval1 = 0; + int128_t bigval2 = 0; + bool isBigVal1; + bool isBigVal2; + + if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone(), isBigVal1, bigval1) || + !getUIntValFromParm(row, parm[1], val2, isNull, timeZone(), isBigVal2, bigval2)) { std::ostringstream oss; oss << "bitand: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); } - return val1 & val2; + if (LIKELY(!isBigVal1 && !isBigVal2)) + { + return val1 & val2; + } + + // Type promotion to int128_t + if (!isBigVal1) + bigval1 = val1; + + if (!isBigVal2) + bigval2 = val2; + + int128_t res = bigval1 & bigval2; + + if (res > static_cast(UINT64_MAX)) + res = UINT64_MAX; + else if (res < static_cast(INT64_MIN)) + res = INT64_MIN; + + return (int64_t) res; } @@ -265,15 +290,39 @@ int64_t Func_leftshift::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone()) || - !getUIntValFromParm(row, parm[1], val2, isNull, timeZone())) + int128_t bigval1 = 0; + int128_t bigval2 = 0; + bool isBigVal1; + bool isBigVal2; + + if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone(), isBigVal1, bigval1) || + !getUIntValFromParm(row, parm[1], val2, isNull, timeZone(), isBigVal2, bigval2)) { std::ostringstream oss; oss << "leftshift: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); } - return val1 << val2; + if (LIKELY(!isBigVal1 && !isBigVal2)) + { + return val1 << val2; + } + + // Type promotion to int128_t + if (!isBigVal1) + bigval1 = val1; + + if (!isBigVal2) + bigval2 = val2; + + int128_t res = bigval1 << bigval2; + + if (res > static_cast(UINT64_MAX)) + res = UINT64_MAX; + else if (res < static_cast(INT64_MIN)) + res = INT64_MIN; + + return (int64_t) res; } @@ -301,15 +350,39 @@ int64_t Func_rightshift::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone()) || - !getUIntValFromParm(row, parm[1], val2, isNull, timeZone())) + int128_t bigval1 = 0; + int128_t bigval2 = 0; + bool isBigVal1; + bool isBigVal2; + + if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone(), isBigVal1, bigval1) || + !getUIntValFromParm(row, parm[1], val2, isNull, timeZone(), isBigVal2, bigval2)) { std::ostringstream oss; oss << "rightshift: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); } - return val1 >> val2; + if (LIKELY(!isBigVal1 && !isBigVal2)) + { + return val1 >> val2; + } + + // Type promotion to int128_t + if (!isBigVal1) + bigval1 = val1; + + if (!isBigVal2) + bigval2 = val2; + + int128_t res = bigval1 >> bigval2; + + if (res > static_cast(UINT64_MAX)) + res = UINT64_MAX; + else if (res < static_cast(INT64_MIN)) + res = INT64_MIN; + + return (int64_t) res; } @@ -337,15 +410,39 @@ int64_t Func_bitor::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone()) || - !getUIntValFromParm(row, parm[1], val2, isNull, timeZone())) + int128_t bigval1 = 0; + int128_t bigval2 = 0; + bool isBigVal1; + bool isBigVal2; + + if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone(), isBigVal1, bigval1) || + !getUIntValFromParm(row, parm[1], val2, isNull, timeZone(), isBigVal2, bigval2)) { std::ostringstream oss; oss << "bitor: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); } - return val1 | val2; + if (LIKELY(!isBigVal1 && !isBigVal2)) + { + return val1 | val2; + } + + // Type promotion to int128_t + if (!isBigVal1) + bigval1 = val1; + + if (!isBigVal2) + bigval2 = val2; + + int128_t res = bigval1 | bigval2; + + if (res > static_cast(UINT64_MAX)) + res = UINT64_MAX; + else if (res < static_cast(INT64_MIN)) + res = INT64_MIN; + + return (int64_t) res; } uint64_t Func_bitor::getUintVal(rowgroup::Row& row, @@ -381,15 +478,39 @@ int64_t Func_bitxor::getIntVal(Row& row, uint64_t val1 = 0; uint64_t val2 = 0; - if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone()) || - !getUIntValFromParm(row, parm[1], val2, isNull, timeZone())) + int128_t bigval1 = 0; + int128_t bigval2 = 0; + bool isBigVal1; + bool isBigVal2; + + if (!getUIntValFromParm(row, parm[0], val1, isNull, timeZone(), isBigVal1, bigval1) || + !getUIntValFromParm(row, parm[1], val2, isNull, timeZone(), isBigVal2, bigval2)) { std::ostringstream oss; oss << "bitxor: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); } - return val1 ^ val2; + if (LIKELY(!isBigVal1 && !isBigVal2)) + { + return val1 ^ val2; + } + + // Type promotion to int128_t + if (!isBigVal1) + bigval1 = val1; + + if (!isBigVal2) + bigval2 = val2; + + int128_t res = bigval1 ^ bigval2; + + if (res > static_cast(UINT64_MAX)) + res = UINT64_MAX; + else if (res < static_cast(INT64_MIN)) + res = INT64_MIN; + + return (int64_t) res; } @@ -403,6 +524,20 @@ CalpontSystemCatalog::ColType Func_bit_count::operationType( FunctionParm& fp, C return resultType; } +inline int64_t bitCount(uint64_t val) +{ + // Refer to Hacker's Delight Chapter 5 + // for the bit counting algo used here + val = val - ((val >> 1) & 0x5555555555555555); + val = (val & 0x3333333333333333) + ((val >> 2) & 0x3333333333333333); + val = (val + (val >> 4)) & 0x0F0F0F0F0F0F0F0F; + val = val + (val >> 8); + val = val + (val >> 16); + val = val + (val >> 32); + + return (int64_t)(val & 0x000000000000007F); +} + int64_t Func_bit_count::getIntVal(Row& row, FunctionParm& parm, bool& isNull, @@ -416,24 +551,25 @@ int64_t Func_bit_count::getIntVal(Row& row, uint64_t val = 0; - if (!getUIntValFromParm(row, parm[0], val, isNull, timeZone())) + int128_t bigval = 0; + bool isBigVal; + + if (!getUIntValFromParm(row, parm[0], val, isNull, timeZone(), isBigVal, bigval)) { std::ostringstream oss; oss << "bit_count: datatype of " << execplan::colDataTypeToString(operationColType.colDataType); throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); } - // Refer to Hacker's Delight Chapter 5 - // for the bit counting algo used here - val = val - ((val >> 1) & 0x5555555555555555); - val = (val & 0x3333333333333333) + ((val >> 2) & 0x3333333333333333); - val = (val + (val >> 4)) & 0x0F0F0F0F0F0F0F0F; - val = val + (val >> 8); - val = val + (val >> 16); - val = val + (val >> 32); - - return (int64_t)(val & 0x000000000000007F); - + if (LIKELY(!isBigVal)) + { + return bitCount(val); + } + else + { + return (bitCount(*reinterpret_cast(&bigval)) + + bitCount(*(reinterpret_cast(&bigval) + 1))); + } } diff --git a/utils/funcexp/func_case.cpp b/utils/funcexp/func_case.cpp index 327c02cc0..0204d9abf 100644 --- a/utils/funcexp/func_case.cpp +++ b/utils/funcexp/func_case.cpp @@ -212,7 +212,7 @@ inline uint64_t simple_case_cmp(Row& row, for (i = 1; i <= whereCount; i++) { - if (ev == parm[i]->data()->getDecimalVal(row, isNull) && !isNull) + if (ev == parm[i]->data()->getDecimalVal(row, isNull) && !isNull) { foundIt = true; break; @@ -329,7 +329,6 @@ inline uint64_t searched_case_cmp(Row& row, uint64_t whereCount = hasElse ? (parm.size() - 1) / 2 : parm.size() / 2; bool foundIt = false; - for (i = 0; i < whereCount; i++) { if (parm[i]->getBoolVal(row, isNull)) @@ -362,9 +361,7 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, bool simpleCase) { uint64_t simple = simpleCase ? 1 : 0; - bool hasElse = (((fp.size()-simple) % 2) != 0); // if 1, then ELSE exist - - + bool hasElse = (((fp.size()-simple) % 2) != 0); // if 1, then ELSE exist uint64_t parmCount = hasElse ? (fp.size() - 2) : (fp.size() - 1); uint64_t whereCount = hasElse ? (fp.size() - 2 + simple) / 2 : (fp.size() - 1) / 2 + simple; @@ -382,8 +379,8 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, { // for SimpleCase, we return the type of the case expression, // which will always be in position 0. - if (i == 0 && simpleCase) - { + if (i == 0 && simpleCase) + { if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) @@ -393,15 +390,16 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, allStringO = false; oct = op.operationType(); } - i += 1; - } - // operation or result type - operation = ((i > 0+simple) && (i <= whereCount)); + i += 1; + } + + // operation or result type + operation = ((i > 0+simple) && (i <= whereCount)); if (fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::CHAR && - fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && - fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) + fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::TEXT && + fp[i]->data()->resultType().colDataType != CalpontSystemCatalog::VARCHAR) { // this is not a string column PredicateOperator op; @@ -414,7 +412,7 @@ CalpontSystemCatalog::ColType caseOperationType(FunctionParm& fp, allStringO = false; oct = op.operationType(); } - } + } // If any parm is of string type, the result type should be string. (same as if) else if (rct.colDataType != CalpontSystemCatalog::CHAR && diff --git a/utils/funcexp/func_ceil.cpp b/utils/funcexp/func_ceil.cpp index 5714b2349..73f8a4624 100644 --- a/utils/funcexp/func_ceil.cpp +++ b/utils/funcexp/func_ceil.cpp @@ -212,13 +212,62 @@ uint64_t Func_ceil::getUintVal(Row& row, case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::TINYINT: case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: { ret = (uint64_t)parm[0]->data()->getIntVal(row, isNull); } break; + // ceil(decimal(X,Y)) leads to this path if X, Y allows to + // downcast to INT otherwise Func_ceil::getDecimalVal() is called + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); + + if (isNull) + break; + + // negative scale is not supported by CNX yet + if (d.scale > 0) + { + if (d.scale > datatypes::INT128MAXPRECISION) + { + std::ostringstream oss; + oss << "ceil: datatype of " << execplan::colDataTypeToString(op_ct.colDataType) + << " with scale " << (int) d.scale << " is beyond supported scale"; + throw logging::IDBExcept(oss.str(), ERR_DATATYPE_NOT_SUPPORT); + } + + if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t tmp = d.s128Value; + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, d.scale); + d.s128Value /= scaleDivisor; + + // Add 1 if this is a positive number and there were values to the right of the + // decimal point so that we return the largest integer value not less than X. + if ((tmp - (d.s128Value * scaleDivisor)) > 0) + d.s128Value += 1; + + ret = datatypes::Decimal::getUInt64FromWideDecimal(d.s128Value); + } + else + { + int64_t tmp = d.value; + d.value /= helpers::powerOf10_c[d.scale]; + + // Add 1 if this is a positive number and there were values to the right of the + // decimal point so that we return the largest integer value not less than X. + if ((tmp - (d.value * helpers::powerOf10_c[d.scale])) > 0) + d.value += 1; + + ret = (uint64_t) d.value; + } + } + } + break; + case CalpontSystemCatalog::UBIGINT: case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::UMEDINT: diff --git a/utils/funcexp/func_char.cpp b/utils/funcexp/func_char.cpp index d3bec2ae9..f303bb8d3 100644 --- a/utils/funcexp/func_char.cpp +++ b/utils/funcexp/func_char.cpp @@ -153,8 +153,8 @@ string Func_char::getStrVal(Row& row, if (lefto > 4) tmpval++; - if (tmpval > static_cast(INT64_MAX)) - tmpval = INT64_MAX; + value = datatypes::Decimal::getInt32FromWideDecimal(tmpval); + // WIP MCOL-641 /*if ( !getChar((int64_t)tmpval, buf) ) { @@ -201,6 +201,7 @@ string Func_char::getStrVal(Row& row, numBytes += getChar(value, pBuf); } + isNull = false; /* Check whether we got a well-formed string */ MY_STRCOPY_STATUS status; diff --git a/utils/funcexp/func_date_add.cpp b/utils/funcexp/func_date_add.cpp index 5004f5521..8c0e547fa 100644 --- a/utils/funcexp/func_date_add.cpp +++ b/utils/funcexp/func_date_add.cpp @@ -749,9 +749,12 @@ int64_t Func_date_add::getIntVal(rowgroup::Row& row, } case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { - if (parm[0]->data()->resultType().scale) + if (parm[0]->data()->resultType().scale == 0) val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); + else + isNull = true; break; } diff --git a/utils/funcexp/func_date_format.cpp b/utils/funcexp/func_date_format.cpp index 105069c59..88a1ed99f 100644 --- a/utils/funcexp/func_date_format.cpp +++ b/utils/funcexp/func_date_format.cpp @@ -363,6 +363,7 @@ string Func_date_format::getStrVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -383,6 +384,11 @@ string Func_date_format::getStrVal(rowgroup::Row& row, dt.msecond = (uint32_t)((val & 0xfffff)); } } + else + { + isNull = true; + return ""; + } break; diff --git a/utils/funcexp/func_day.cpp b/utils/funcexp/func_day.cpp index c414fd5d4..9b7def395 100644 --- a/utils/funcexp/func_day.cpp +++ b/utils/funcexp/func_day.cpp @@ -118,6 +118,7 @@ int64_t Func_day::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -132,6 +133,10 @@ int64_t Func_day::getIntVal(rowgroup::Row& row, return (uint32_t)((val >> 38) & 0x3f); } } + else + { + isNull = true; + } break; diff --git a/utils/funcexp/func_dayname.cpp b/utils/funcexp/func_dayname.cpp index e46bfa46f..7a076435b 100644 --- a/utils/funcexp/func_dayname.cpp +++ b/utils/funcexp/func_dayname.cpp @@ -138,6 +138,7 @@ int64_t Func_dayname::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -154,6 +155,11 @@ int64_t Func_dayname::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); } } + else + { + isNull = true; + return -1; + } break; diff --git a/utils/funcexp/func_dayofweek.cpp b/utils/funcexp/func_dayofweek.cpp index 3a29f592d..d0b28d22e 100644 --- a/utils/funcexp/func_dayofweek.cpp +++ b/utils/funcexp/func_dayofweek.cpp @@ -136,6 +136,7 @@ int64_t Func_dayofweek::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -152,6 +153,11 @@ int64_t Func_dayofweek::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); } } + else + { + isNull = true; + return -1; + } break; diff --git a/utils/funcexp/func_dayofyear.cpp b/utils/funcexp/func_dayofyear.cpp index ee13730c1..cf9c37814 100644 --- a/utils/funcexp/func_dayofyear.cpp +++ b/utils/funcexp/func_dayofyear.cpp @@ -75,7 +75,7 @@ int64_t Func_dayofyear::getIntVal(rowgroup::Row& row, { dataconvert::TimeStamp timestamp(parm[0]->data()->getIntVal(row, isNull)); int64_t seconds = timestamp.second; - dataconvert::MySQLTime m_time; + dataconvert::MySQLTime m_time; dataconvert::gmtSecToMySQLTime(seconds, m_time, timeZone()); year = m_time.year; month = m_time.month; @@ -135,6 +135,7 @@ int64_t Func_dayofyear::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); diff --git a/utils/funcexp/func_elt.cpp b/utils/funcexp/func_elt.cpp index 2fd377dd4..b3bf6a3b3 100644 --- a/utils/funcexp/func_elt.cpp +++ b/utils/funcexp/func_elt.cpp @@ -71,6 +71,7 @@ string Func_elt::getStrVal(rowgroup::Row& row, } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); diff --git a/utils/funcexp/func_from_unixtime.cpp b/utils/funcexp/func_from_unixtime.cpp index d1168d2d6..ab88e39e0 100644 --- a/utils/funcexp/func_from_unixtime.cpp +++ b/utils/funcexp/func_from_unixtime.cpp @@ -61,10 +61,22 @@ DateTime getDateTime(rowgroup::Row& row, break; } case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { IDB_Decimal dec = parm[0]->data()->getDecimalVal(row, isNull); - val = dec.value / IDB_pow[dec.scale]; - msec = dec.value % IDB_pow[dec.scale]; + + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + int128_t scaleDivisor; + datatypes::getScaleDivisor(scaleDivisor, dec.scale); + val = datatypes::Decimal::getInt64FromWideDecimal(dec.s128Value / scaleDivisor); + msec = datatypes::Decimal::getUInt32FromWideDecimal(dec.s128Value % scaleDivisor); + } + else + { + val = dec.value / IDB_pow[dec.scale]; + msec = dec.value % IDB_pow[dec.scale]; + } break; } diff --git a/utils/funcexp/func_hex.cpp b/utils/funcexp/func_hex.cpp index 97e608981..b45f6b52a 100644 --- a/utils/funcexp/func_hex.cpp +++ b/utils/funcexp/func_hex.cpp @@ -91,6 +91,7 @@ string Func_hex::getStrVal(rowgroup::Row& row, case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { /* Return hex of unsigned longlong value */ double val = parm[0]->data()->getDoubleVal(row, isNull); diff --git a/utils/funcexp/func_hour.cpp b/utils/funcexp/func_hour.cpp index 918fe193d..af949a393 100644 --- a/utils/funcexp/func_hour.cpp +++ b/utils/funcexp/func_hour.cpp @@ -67,14 +67,19 @@ int64_t Func_hour::getIntVal(rowgroup::Row& row, } case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { - if (parm[0]->data()->resultType().scale) + if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); if (val == -1) isNull = true; } + else + { + isNull = true; + } break; } diff --git a/utils/funcexp/func_if.cpp b/utils/funcexp/func_if.cpp index 6d10d6829..168383b93 100644 --- a/utils/funcexp/func_if.cpp +++ b/utils/funcexp/func_if.cpp @@ -43,7 +43,7 @@ bool boolVal(SPTP& parm, Row& row, const string& timeZone) try { - ret = parm->getBoolVal(row, isNull); + ret = parm->getBoolVal(row, isNull) && !isNull; } catch (logging::NotImplementedExcept&) { @@ -67,7 +67,10 @@ bool boolVal(SPTP& parm, Row& row, const string& timeZone) break; case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - ret = (parm->data()->getDecimalVal(row, isNull).value != 0); + if (parm->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + ret = (parm->data()->getDecimalVal(row, isNull).s128Value != 0); + else + ret = (parm->data()->getDecimalVal(row, isNull).value != 0); break; case CalpontSystemCatalog::BIGINT: case CalpontSystemCatalog::SMALLINT: diff --git a/utils/funcexp/func_inet_ntoa.cpp b/utils/funcexp/func_inet_ntoa.cpp index 0d33d5ec1..ea528ceb6 100644 --- a/utils/funcexp/func_inet_ntoa.cpp +++ b/utils/funcexp/func_inet_ntoa.cpp @@ -157,6 +157,8 @@ std::string Func_inet_ntoa::getStrVal(rowgroup::Row& row, // else just get integer value if ((fp[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::DECIMAL) || + (fp[0]->data()->resultType().colDataType == + execplan::CalpontSystemCatalog::UDECIMAL) || (fp[0]->data()->resultType().colDataType == execplan::CalpontSystemCatalog::FLOAT) || (fp[0]->data()->resultType().colDataType == diff --git a/utils/funcexp/func_last_day.cpp b/utils/funcexp/func_last_day.cpp index 2099ea71a..b20e8502f 100644 --- a/utils/funcexp/func_last_day.cpp +++ b/utils/funcexp/func_last_day.cpp @@ -136,6 +136,7 @@ int64_t Func_last_day::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -152,6 +153,11 @@ int64_t Func_last_day::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); } } + else + { + isNull = true; + return -1; + } break; diff --git a/utils/funcexp/func_makedate.cpp b/utils/funcexp/func_makedate.cpp index 8efc3f171..64c2d5e5d 100644 --- a/utils/funcexp/func_makedate.cpp +++ b/utils/funcexp/func_makedate.cpp @@ -66,6 +66,7 @@ uint64_t makedate(rowgroup::Row& row, } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); @@ -149,6 +150,7 @@ uint64_t makedate(rowgroup::Row& row, } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[1]->data()->getDecimalVal(row, isNull); diff --git a/utils/funcexp/func_maketime.cpp b/utils/funcexp/func_maketime.cpp index 3bc7b8c27..6e315c7ec 100644 --- a/utils/funcexp/func_maketime.cpp +++ b/utils/funcexp/func_maketime.cpp @@ -72,6 +72,7 @@ string Func_maketime::getStrVal(rowgroup::Row& row, } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); @@ -135,6 +136,7 @@ string Func_maketime::getStrVal(rowgroup::Row& row, } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[1]->data()->getDecimalVal(row, isNull); @@ -204,6 +206,7 @@ string Func_maketime::getStrVal(rowgroup::Row& row, } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { IDB_Decimal d = parm[2]->data()->getDecimalVal(row, isNull); diff --git a/utils/funcexp/func_math.cpp b/utils/funcexp/func_math.cpp index 0c86cbb67..fd0c0753b 100644 --- a/utils/funcexp/func_math.cpp +++ b/utils/funcexp/func_math.cpp @@ -1849,70 +1849,134 @@ string Func_format::getStrVal(Row& row, { IDB_Decimal decimal = parm[0]->data()->getDecimalVal(row, isNull); + char buf[80]; + //perform rouding if needed if ( scale < 0 ) scale = 0; - if ( scale < decimal.scale ) + if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) { - int64_t d = 0; - int64_t p = 1; - - if (!isNull && parm.size() > 1) + if ( scale < decimal.scale ) { - d = scale; + int64_t d = 0; + int128_t p = 1; - if (!isNull) - helpers::decimalPlaceDec(d, p, decimal.scale); - } - - if (isNull) - break; - - int64_t x = decimal.value; - - if (d > 0) - { - x = x * p; - } - else if (d < 0) - { - int64_t h = p / 2; // 0.5 - - if ((x >= h) || (x <= -h)) + if (!isNull && parm.size() > 1) { - if (x >= 0) - x += h; - else - x -= h; + d = scale; - if (p != 0) - x = x / p; + if (!isNull) + helpers::decimalPlaceDec(d, p, decimal.scale); + } + + if (isNull) + break; + + int128_t x = decimal.s128Value; + + if (d > 0) + { + x = x * p; + } + else if (d < 0) + { + int128_t h = p / 2; // 0.5 + + if ((x >= h) || (x <= -h)) + { + if (x >= 0) + x += h; + else + x -= h; + + if (p != 0) + x = x / p; + else + x = 0; + } else + { x = 0; + } } - else + + // negative scale is not supported by CNX yet, set d to 0. + if (decimal.scale < 0) { - x = 0; + do + x *= 10; + + while (++decimal.scale < 0); } + + decimal.s128Value = x; } - // negative scale is not supported by CNX yet, set d to 0. - if (decimal.scale < 0) - { - do - x *= 10; - - while (++decimal.scale < 0); - } - - decimal.value = x; + dataconvert::DataConvert::decimalToString(&decimal.s128Value, + decimal.scale, buf, 80, parm[0]->data()->resultType().colDataType); } + else + { + if ( scale < decimal.scale ) + { + int64_t d = 0; + int64_t p = 1; - char buf[80]; + if (!isNull && parm.size() > 1) + { + d = scale; - dataconvert::DataConvert::decimalToString( decimal.value, - decimal.scale, buf, 80, parm[0]->data()->resultType().colDataType); + if (!isNull) + helpers::decimalPlaceDec(d, p, decimal.scale); + } + + if (isNull) + break; + + int64_t x = decimal.value; + + if (d > 0) + { + x = x * p; + } + else if (d < 0) + { + int64_t h = p / 2; // 0.5 + + if ((x >= h) || (x <= -h)) + { + if (x >= 0) + x += h; + else + x -= h; + + if (p != 0) + x = x / p; + else + x = 0; + } + else + { + x = 0; + } + } + + // negative scale is not supported by CNX yet, set d to 0. + if (decimal.scale < 0) + { + do + x *= 10; + + while (++decimal.scale < 0); + } + + decimal.value = x; + } + + dataconvert::DataConvert::decimalToString( decimal.value, + decimal.scale, buf, 80, parm[0]->data()->resultType().colDataType); + } value = buf; } @@ -1982,7 +2046,7 @@ string Func_format::getStrVal(Row& row, // pad extra with '0' if (*(value.data()) != '#') { - for ( int i = 0 ; i < pad ; i ++ ) + for ( int i = 0 ; i < pad ; i++ ) { value = value.append("0"); } @@ -1996,7 +2060,7 @@ string Func_format::getStrVal(Row& row, string::size_type pos = value.find ('-', 0); if (pos != string::npos) - end = 1;; + end = 1; while ((comma -= 3) > end) { diff --git a/utils/funcexp/func_microsecond.cpp b/utils/funcexp/func_microsecond.cpp index 12b2beb70..18b1cdabb 100644 --- a/utils/funcexp/func_microsecond.cpp +++ b/utils/funcexp/func_microsecond.cpp @@ -110,6 +110,7 @@ int64_t Func_microsecond::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -124,6 +125,11 @@ int64_t Func_microsecond::getIntVal(rowgroup::Row& row, microSecond = (uint32_t)((val & 0xfffff)); } } + else + { + isNull = true; + return -1; + } break; diff --git a/utils/funcexp/func_minute.cpp b/utils/funcexp/func_minute.cpp index fdad4e9cf..672008048 100644 --- a/utils/funcexp/func_minute.cpp +++ b/utils/funcexp/func_minute.cpp @@ -66,14 +66,19 @@ int64_t Func_minute::getIntVal(rowgroup::Row& row, } case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { - if (parm[0]->data()->resultType().scale) + if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); if (val == -1) isNull = true; } + else + { + isNull = true; + } break; } diff --git a/utils/funcexp/func_month.cpp b/utils/funcexp/func_month.cpp index ffb3f5fc1..f9cbfddd0 100644 --- a/utils/funcexp/func_month.cpp +++ b/utils/funcexp/func_month.cpp @@ -116,6 +116,7 @@ int64_t Func_month::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -130,6 +131,10 @@ int64_t Func_month::getIntVal(rowgroup::Row& row, return (unsigned)((val >> 44) & 0xf); } } + else + { + isNull = true; + } break; diff --git a/utils/funcexp/func_monthname.cpp b/utils/funcexp/func_monthname.cpp index f04c37158..7d41e3ba4 100644 --- a/utils/funcexp/func_monthname.cpp +++ b/utils/funcexp/func_monthname.cpp @@ -158,6 +158,7 @@ int64_t Func_monthname::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); diff --git a/utils/funcexp/func_nullif.cpp b/utils/funcexp/func_nullif.cpp index 90c546ea3..886c58f9c 100644 --- a/utils/funcexp/func_nullif.cpp +++ b/utils/funcexp/func_nullif.cpp @@ -430,6 +430,7 @@ int32_t Func_nullif::getDateIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -518,6 +519,7 @@ int64_t Func_nullif::getDatetimeIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -584,6 +586,7 @@ int64_t Func_nullif::getTimeIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -636,6 +639,7 @@ int64_t Func_nullif::getTimestampIntVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -689,6 +693,7 @@ double Func_nullif::getDoubleVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: @@ -779,6 +784,7 @@ long double Func_nullif::getLongDoubleVal(rowgroup::Row& row, case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: case execplan::CalpontSystemCatalog::VARCHAR: case execplan::CalpontSystemCatalog::CHAR: case execplan::CalpontSystemCatalog::TEXT: diff --git a/utils/funcexp/func_quarter.cpp b/utils/funcexp/func_quarter.cpp index c1dcda89e..35b9d93f4 100644 --- a/utils/funcexp/func_quarter.cpp +++ b/utils/funcexp/func_quarter.cpp @@ -115,6 +115,7 @@ int64_t Func_quarter::getIntVal(rowgroup::Row& row, } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { if (parm[0]->data()->resultType().scale == 0) { diff --git a/utils/funcexp/func_regexp.cpp b/utils/funcexp/func_regexp.cpp index 120a727e6..4a484c8e3 100644 --- a/utils/funcexp/func_regexp.cpp +++ b/utils/funcexp/func_regexp.cpp @@ -118,7 +118,14 @@ inline bool getBool(rowgroup::Row& row, char buf[80]; - dataconvert::DataConvert::decimalToString(d.value, d.scale, buf, 80, pm[0]->data()->resultType().colDataType); + if (pm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + dataconvert::DataConvert::decimalToString(&d.s128Value, d.scale, buf, 80, pm[0]->data()->resultType().colDataType); + } + else + { + dataconvert::DataConvert::decimalToString(d.value, d.scale, buf, 80, pm[0]->data()->resultType().colDataType); + } expr = buf; break; @@ -193,7 +200,14 @@ inline bool getBool(rowgroup::Row& row, char buf[80]; - dataconvert::DataConvert::decimalToString( d.value, d.scale, buf, 80, pm[1]->data()->resultType().colDataType); + if (pm[1]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + { + dataconvert::DataConvert::decimalToString(&d.s128Value, d.scale, buf, 80, pm[1]->data()->resultType().colDataType); + } + else + { + dataconvert::DataConvert::decimalToString(d.value, d.scale, buf, 80, pm[1]->data()->resultType().colDataType); + } pattern = buf; break; diff --git a/utils/funcexp/func_sec_to_time.cpp b/utils/funcexp/func_sec_to_time.cpp index afb532ec1..ff808a975 100644 --- a/utils/funcexp/func_sec_to_time.cpp +++ b/utils/funcexp/func_sec_to_time.cpp @@ -120,6 +120,7 @@ string Func_sec_to_time::getStrVal(rowgroup::Row& row, break; case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { const string& valStr = parm[0]->data()->getStrVal(row, isNull); diff --git a/utils/funcexp/func_second.cpp b/utils/funcexp/func_second.cpp index 475f7495e..ff09bf320 100644 --- a/utils/funcexp/func_second.cpp +++ b/utils/funcexp/func_second.cpp @@ -66,14 +66,19 @@ int64_t Func_second::getIntVal(rowgroup::Row& row, } case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { - if (parm[0]->data()->resultType().scale) + if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); if (val == -1) isNull = true; } + else + { + isNull = true; + } break; } diff --git a/utils/funcexp/func_str_to_date.cpp b/utils/funcexp/func_str_to_date.cpp index c4b0226f2..3a4c165ac 100644 --- a/utils/funcexp/func_str_to_date.cpp +++ b/utils/funcexp/func_str_to_date.cpp @@ -145,6 +145,7 @@ dataconvert::DateTime getDateTime (rowgroup::Row& row, } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { if (parm[0]->data()->resultType().scale == 0) { @@ -159,6 +160,11 @@ dataconvert::DateTime getDateTime (rowgroup::Row& row, return -1; } } + else + { + isNull = true; + return -1; + } break; } diff --git a/utils/funcexp/func_time.cpp b/utils/funcexp/func_time.cpp index 70202ac63..89d95b021 100644 --- a/utils/funcexp/func_time.cpp +++ b/utils/funcexp/func_time.cpp @@ -70,8 +70,9 @@ string Func_time::getStrVal(rowgroup::Row& row, } case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: { - if (parm[0]->data()->resultType().scale) + if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToTime(parm[0]->data()->getIntVal(row, isNull)); @@ -81,6 +82,10 @@ string Func_time::getStrVal(rowgroup::Row& row, //else // return *(reinterpret_cast(&val)); } + else + { + isNull = true; + } break; } diff --git a/utils/funcexp/func_time_format.cpp b/utils/funcexp/func_time_format.cpp index 5dd2f666a..c77e46934 100644 --- a/utils/funcexp/func_time_format.cpp +++ b/utils/funcexp/func_time_format.cpp @@ -128,6 +128,7 @@ string Func_time_format::getStrVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -145,6 +146,11 @@ string Func_time_format::getStrVal(rowgroup::Row& row, msec = (uint32_t)((val & 0xfffff)); } } + else + { + isNull = true; + return ""; + } break; diff --git a/utils/funcexp/func_time_to_sec.cpp b/utils/funcexp/func_time_to_sec.cpp index e19ce2f81..b8ab898b8 100644 --- a/utils/funcexp/func_time_to_sec.cpp +++ b/utils/funcexp/func_time_to_sec.cpp @@ -153,6 +153,7 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -169,6 +170,11 @@ int64_t Func_time_to_sec::getIntVal(rowgroup::Row& row, sec = (int32_t)((val >> 20) & 0x3f); } } + else + { + isNull = true; + return -1; + } break; diff --git a/utils/funcexp/func_timediff.cpp b/utils/funcexp/func_timediff.cpp index 3c629933f..184200528 100644 --- a/utils/funcexp/func_timediff.cpp +++ b/utils/funcexp/func_timediff.cpp @@ -199,6 +199,7 @@ string Func_timediff::getStrVal(rowgroup::Row& row, break; case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale != 0) { isNull = true; @@ -285,6 +286,7 @@ string Func_timediff::getStrVal(rowgroup::Row& row, break; case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: if (parm[1]->data()->resultType().scale != 0) { isNull = true; diff --git a/utils/funcexp/func_unix_timestamp.cpp b/utils/funcexp/func_unix_timestamp.cpp index 29a8f483a..4c43eaebb 100644 --- a/utils/funcexp/func_unix_timestamp.cpp +++ b/utils/funcexp/func_unix_timestamp.cpp @@ -131,6 +131,7 @@ int64_t Func_unix_timestamp::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -147,6 +148,11 @@ int64_t Func_unix_timestamp::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); } } + else + { + isNull = true; + return -1; + } break; diff --git a/utils/funcexp/func_week.cpp b/utils/funcexp/func_week.cpp index af024cfe4..874e95b42 100644 --- a/utils/funcexp/func_week.cpp +++ b/utils/funcexp/func_week.cpp @@ -139,6 +139,7 @@ int64_t Func_week::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -155,6 +156,11 @@ int64_t Func_week::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); } } + else + { + isNull = true; + return -1; + } break; diff --git a/utils/funcexp/func_weekday.cpp b/utils/funcexp/func_weekday.cpp index 04ca938b4..e8b59e620 100644 --- a/utils/funcexp/func_weekday.cpp +++ b/utils/funcexp/func_weekday.cpp @@ -135,6 +135,7 @@ int64_t Func_weekday::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -151,6 +152,11 @@ int64_t Func_weekday::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); } } + else + { + isNull = true; + return -1; + } break; diff --git a/utils/funcexp/func_year.cpp b/utils/funcexp/func_year.cpp index 2c5ec41f9..68f60ac20 100644 --- a/utils/funcexp/func_year.cpp +++ b/utils/funcexp/func_year.cpp @@ -118,6 +118,7 @@ int64_t Func_year::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); diff --git a/utils/funcexp/func_yearweek.cpp b/utils/funcexp/func_yearweek.cpp index 91bb60a8c..07e111c22 100644 --- a/utils/funcexp/func_yearweek.cpp +++ b/utils/funcexp/func_yearweek.cpp @@ -142,6 +142,7 @@ int64_t Func_yearweek::getIntVal(rowgroup::Row& row, break; case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: if (parm[0]->data()->resultType().scale == 0) { val = dataconvert::DataConvert::intToDatetime(parm[0]->data()->getIntVal(row, isNull)); @@ -158,6 +159,11 @@ int64_t Func_yearweek::getIntVal(rowgroup::Row& row, day = (uint32_t)((val >> 38) & 0x3f); } } + else + { + isNull = true; + return -1; + } break; diff --git a/utils/funcexp/funcexp.cpp b/utils/funcexp/funcexp.cpp index d9dc24180..3cf0373b9 100644 --- a/utils/funcexp/funcexp.cpp +++ b/utils/funcexp/funcexp.cpp @@ -473,6 +473,7 @@ void FuncExp::evaluate(rowgroup::Row& row, std::vector& expressi case CalpontSystemCatalog::UDECIMAL: { IDB_Decimal val = expression[i]->getDecimalVal(row, isNull); + if (expression[i]->resultType().colWidth == datatypes::MAXDECIMALWIDTH) { From ca53b6348a4edc4efea5be231a450bfb80d3dc1f Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Mon, 20 Jul 2020 19:32:17 +0000 Subject: [PATCH 48/78] MCOL-641 A bit lost during rebase --- dbcon/joblist/tupleaggregatestep.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index 6a0b0fe11..7ace89be5 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -3180,7 +3180,7 @@ void TupleAggregateStep::prep2PhasesAggregate( precisionProj, oidsProj, aggKey, scaleProj, width, oidsAggPm, keysAggPm, typeAggPm, scaleAggPm, precisionAggPm, widthAggPm); - scaleAggPm.push_back(0); + csNumAggPm.push_back(8); colAggPm++; } From d3bc68b02fb0cd5a762c69c689303391c97edb6b Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Fri, 24 Jul 2020 19:04:25 -0400 Subject: [PATCH 49/78] MCOL-641 Refactor initial extent elimination support. This commit also adds support in TupleHashJoinStep::forwardCPData, although we currently do not support wide decimals as join keys. Row estimation to determine large-side of the join is also updated. --- dbcon/dmlpackageproc/dmlpackageprocessor.cpp | 5 +- dbcon/joblist/lbidlist.cpp | 122 +++- dbcon/joblist/lbidlist.h | 11 +- dbcon/joblist/primitivestep.h | 12 +- dbcon/joblist/pseudocc-jl.cpp | 54 +- dbcon/joblist/rowestimator.cpp | 124 +++- dbcon/joblist/rowestimator.h | 8 +- dbcon/joblist/tuple-bps.cpp | 137 ++++- dbcon/joblist/tuplehashjoin.cpp | 10 +- dbcon/mysql/ha_mcs_partition.cpp | 545 +++++++++++++----- dbcon/mysql/is_columnstore_extents.cpp | 74 ++- primitives/linux-port/column.cpp | 4 +- .../primproc/batchprimitiveprocessor.cpp | 9 +- primitives/primproc/pseudocc.cpp | 45 +- primitives/primproc/pseudocc.h | 1 + tools/editem/editem.cpp | 124 +++- utils/common/hasher.h | 21 + utils/common/widedecimalutils.h | 3 + utils/dataconvert/dataconvert.h | 32 - utils/joiner/tuplejoiner.cpp | 147 +++-- utils/joiner/tuplejoiner.h | 4 +- utils/rowgroup/rowgroup.h | 1 + versioning/BRM/blockresolutionmanager.cpp | 4 +- versioning/BRM/dbrm.cpp | 5 +- versioning/BRM/extentmap.cpp | 29 +- writeengine/bulk/we_brmreporter.cpp | 37 +- writeengine/bulk/we_bulkloadbuffer.cpp | 5 +- writeengine/bulk/we_bulkloadbuffer.h | 5 +- writeengine/bulk/we_colextinf.cpp | 3 +- writeengine/bulk/we_colextinf.h | 9 +- writeengine/wrapper/we_colop.cpp | 9 +- writeengine/wrapper/writeengine.cpp | 8 +- 32 files changed, 1221 insertions(+), 386 deletions(-) diff --git a/dbcon/dmlpackageproc/dmlpackageprocessor.cpp b/dbcon/dmlpackageproc/dmlpackageprocessor.cpp index 19f8bc0ce..ad5bd1df0 100644 --- a/dbcon/dmlpackageproc/dmlpackageprocessor.cpp +++ b/dbcon/dmlpackageproc/dmlpackageprocessor.cpp @@ -52,6 +52,7 @@ using namespace joblist; using namespace messageqcpp; #include "tablelockdata.h" #include "exceptclasses.h" +#include "widedecimalutils.h" namespace { @@ -510,8 +511,8 @@ int DMLPackageProcessor::commitBatchAutoOnTransaction(uint64_t uniqueId, BRM::Tx aInfo.firstLbid = *iter; aInfo.max = numeric_limits::min(); // Not used aInfo.min = numeric_limits::max(); // Not used - dataconvert::DataConvert::int128Min(aInfo.bigMax); // Not used - dataconvert::DataConvert::int128Max(aInfo.bigMin); // Not used + utils::int128Min(aInfo.bigMax); // Not used + utils::int128Max(aInfo.bigMin); // Not used aInfo.seqNum = -1; cpInfos.push_back(aInfo); ++iter; diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index 90697bbfc..bc7ca9303 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -28,6 +28,7 @@ #include "brm.h" #include "brmtypes.h" #include "dataconvert.h" +#include "widedecimalutils.h" #include "mcs_decimal.h" #define IS_VERBOSE (fDebug >= 4) @@ -251,8 +252,8 @@ bool LBIDList::GetMinMax(T& min, T& max, int64_t& seq, int64_t lbid, { if (typeid(T) == typeid(__int128)) { - dataconvert::DataConvert::int128Min(mmp->bigMax); - dataconvert::DataConvert::int128Max(mmp->bigMin); + utils::int128Min(mmp->bigMax); + utils::int128Max(mmp->bigMin); } else { @@ -274,8 +275,8 @@ bool LBIDList::GetMinMax(T& min, T& max, int64_t& seq, int64_t lbid, return false; } -//TODO MCOL-641 Do we need support here? -bool LBIDList::GetMinMax(int64_t* min, int64_t* max, int64_t* seq, +template +bool LBIDList::GetMinMax(T* min, T* max, int64_t* seq, int64_t lbid, const tr1::unordered_map& entries, execplan::CalpontSystemCatalog::ColDataType colDataType) { @@ -296,13 +297,29 @@ bool LBIDList::GetMinMax(int64_t* min, int64_t* max, int64_t* seq, if (isUnsigned(colDataType)) { - mmp->max = 0; - mmp->min = static_cast(numeric_limits::max()); + if (typeid(T) == typeid(__int128)) + { + mmp->bigMax = 0; + mmp->bigMin = -1; + } + else + { + mmp->max = 0; + mmp->min = static_cast(numeric_limits::max()); + } } else { - mmp->max = numeric_limits::min(); - mmp->min = numeric_limits::max(); + if (typeid(T) == typeid(__int128)) + { + utils::int128Min(mmp->bigMax); + utils::int128Max(mmp->bigMin); + } + else + { + mmp->max = numeric_limits::min(); + mmp->min = numeric_limits::max(); + } } mmp->isValid = entry.partition.cprange.isValid; @@ -311,9 +328,19 @@ bool LBIDList::GetMinMax(int64_t* min, int64_t* max, int64_t* seq, return false; } - *min = entry.partition.cprange.lo_val; - *max = entry.partition.cprange.hi_val; + if (typeid(T) == typeid(__int128)) + { + *min = entry.partition.cprange.bigLoVal; + *max = entry.partition.cprange.bigHiVal; + } + else + { + *min = entry.partition.cprange.lo_val; + *max = entry.partition.cprange.hi_val; + } + *seq = entry.partition.cprange.sequenceNum; + return true; } @@ -653,11 +680,14 @@ inline bool LBIDList::compareVal(const T& Min, const T& Max, const T& value, cha return true; } -bool LBIDList::checkSingleValue(int64_t min, int64_t max, int64_t value, +template +bool LBIDList::checkSingleValue(T min, T max, T value, execplan::CalpontSystemCatalog::ColDataType type) { if (isCharType(type)) { + // MCOL-641 LBIDList::CasualPartitionDataType() returns false if + // width > 8 for a character type, so T cannot be __int128 here uint64_t mmin = order_swap(min); uint64_t mmax = order_swap(max); uint64_t vvalue = order_swap(value); @@ -665,8 +695,16 @@ bool LBIDList::checkSingleValue(int64_t min, int64_t max, int64_t value, } else if (isUnsigned(type)) { - return (static_cast(value) >= static_cast(min) && - static_cast(value) <= static_cast(max)); + if (typeid(T) == typeid(__int128)) + { + return (static_cast(value) >= static_cast(min) && + static_cast(value) <= static_cast(max)); + } + else + { + return (static_cast(value) >= static_cast(min) && + static_cast(value) <= static_cast(max)); + } } else { @@ -674,11 +712,14 @@ bool LBIDList::checkSingleValue(int64_t min, int64_t max, int64_t value, } } -bool LBIDList::checkRangeOverlap(int64_t min, int64_t max, int64_t tmin, int64_t tmax, +template +bool LBIDList::checkRangeOverlap(T min, T max, T tmin, T tmax, execplan::CalpontSystemCatalog::ColDataType type) { if (isCharType(type)) { + // MCOL-641 LBIDList::CasualPartitionDataType() returns false if + // width > 8 for a character type, so T cannot be __int128 here uint64_t min2 = order_swap(min); uint64_t max2 = order_swap(max); uint64_t tmin2 = order_swap(tmin); @@ -687,8 +728,16 @@ bool LBIDList::checkRangeOverlap(int64_t min, int64_t max, int64_t tmin, int64_t } else if (isUnsigned(type)) { - return (static_cast(tmin) <= static_cast(max) && - static_cast(tmax) >= static_cast(min)); + if (typeid(T) == typeid(__int128)) + { + return (static_cast(tmin) <= static_cast(max) && + static_cast(tmax) >= static_cast(min)); + } + else + { + return (static_cast(tmin) <= static_cast(max) && + static_cast(tmax) >= static_cast(min)); + } } else { @@ -708,8 +757,7 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, bool scan = true; int64_t value = 0; __int128 bigValue = 0; - dataconvert::Int128Pod_t* bigValuePod; - bigValuePod = reinterpret_cast(&bigValue); + uint64_t* int128Ptr = reinterpret_cast(&bigValue); bool bIsUnsigned = execplan::isUnsigned(ct.colDataType); bool bIsChar = execplan::isCharType(ct.colDataType); @@ -759,12 +807,13 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, uint64_t val = *(int64_t*)MsgDataPtr; value = static_cast(val); } + case 16: { unsigned __int128 val; - bigValuePod = reinterpret_cast(&val); - bigValuePod->lo = *reinterpret_cast(MsgDataPtr); - bigValuePod->hi = *(reinterpret_cast(MsgDataPtr) + 1); + int128Ptr = reinterpret_cast(&val); + int128Ptr[0] = *reinterpret_cast(MsgDataPtr); + int128Ptr[1] = *(reinterpret_cast(MsgDataPtr) + 1); bigValue = static_cast<__int128>(val); } } @@ -799,10 +848,11 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, int64_t val = *(int64_t*)MsgDataPtr; value = val; } + case 16: { - bigValuePod->lo = *reinterpret_cast(MsgDataPtr); - bigValuePod->hi = *(reinterpret_cast(MsgDataPtr) + 1); + int128Ptr[0] = *reinterpret_cast(MsgDataPtr); + int128Ptr[1] = *(reinterpret_cast(MsgDataPtr) + 1); } } } @@ -918,6 +968,16 @@ bool LBIDList::GetMinMax(int64_t& min, int64_t& max, int64_t& seq, int6 const std::vector* pEMEntries, execplan::CalpontSystemCatalog::ColDataType colDataType); +template +bool LBIDList::GetMinMax<__int128>(__int128* min, __int128* max, int64_t* seq, + int64_t lbid, const tr1::unordered_map& entries, + execplan::CalpontSystemCatalog::ColDataType colDataType); + +template +bool LBIDList::GetMinMax(int64_t* min, int64_t* max, int64_t* seq, + int64_t lbid, const tr1::unordered_map& entries, + execplan::CalpontSystemCatalog::ColDataType colDataType); + template void LBIDList::UpdateMinMax<__int128>(__int128 min, __int128 max, int64_t lbid, execplan::CalpontSystemCatalog::ColDataType type, bool validData = true); @@ -926,6 +986,22 @@ template void LBIDList::UpdateMinMax(int64_t min, int64_t max, int64_t lbid, execplan::CalpontSystemCatalog::ColDataType type, bool validData = true); +template +bool LBIDList::checkSingleValue<__int128>(__int128 min, __int128 max, __int128 value, + execplan::CalpontSystemCatalog::ColDataType type); + +template +bool LBIDList::checkSingleValue(int64_t min, int64_t max, int64_t value, + execplan::CalpontSystemCatalog::ColDataType type); + +template +bool LBIDList::checkRangeOverlap<__int128>(__int128 min, __int128 max, __int128 tmin, __int128 tmax, + execplan::CalpontSystemCatalog::ColDataType type); + +template +bool LBIDList::checkRangeOverlap(int64_t min, int64_t max, int64_t tmin, int64_t tmax, + execplan::CalpontSystemCatalog::ColDataType type); + } //namespace joblist // vim:ts=4 sw=4: diff --git a/dbcon/joblist/lbidlist.h b/dbcon/joblist/lbidlist.h index 0b2fccca6..934f42924 100644 --- a/dbcon/joblist/lbidlist.h +++ b/dbcon/joblist/lbidlist.h @@ -91,13 +91,14 @@ public: // Functions to handle min/max values per lbid for casual partitioning; // If pEMEntries is provided, then min/max will be extracted from that // vector, else extents in BRM will be searched. If type is unsigned, caller - // should static cast returned min and max to uint64_t + // should static cast returned min and max to uint64_t/uint128_t template bool GetMinMax(T& min, T& max, int64_t& seq, int64_t lbid, const std::vector* pEMEntries, execplan::CalpontSystemCatalog::ColDataType type); - bool GetMinMax(int64_t* min, int64_t* max, int64_t* seq, int64_t lbid, + template + bool GetMinMax(T* min, T* max, int64_t* seq, int64_t lbid, const std::tr1::unordered_map& entries, execplan::CalpontSystemCatalog::ColDataType type); @@ -115,10 +116,12 @@ public: const execplan::CalpontSystemCatalog::ColType& ct, const uint8_t BOP); - bool checkSingleValue(int64_t min, int64_t max, int64_t value, + template + bool checkSingleValue(T min, T max, T value, execplan::CalpontSystemCatalog::ColDataType type); - bool checkRangeOverlap(int64_t min, int64_t max, int64_t tmin, int64_t tmax, + template + bool checkRangeOverlap(T min, T max, T tmin, T tmax, execplan::CalpontSystemCatalog::ColDataType type); // check the column data type and the column size to determine if it diff --git a/dbcon/joblist/primitivestep.h b/dbcon/joblist/primitivestep.h index 603bc6ebd..e57443ff3 100644 --- a/dbcon/joblist/primitivestep.h +++ b/dbcon/joblist/primitivestep.h @@ -1327,7 +1327,8 @@ public: * Note that it is an adder not a setter. For an extent to be scanned, all calls * must have a non-empty intersection. */ - void addCPPredicates(uint32_t OID, const std::vector& vals, bool isRange); + void addCPPredicates(uint32_t OID, const std::vector<__int128>& vals, bool isRange, + bool isSmallSideWideDecimal); /* semijoin adds */ void setJoinFERG(const rowgroup::RowGroup& rg); @@ -1509,13 +1510,16 @@ private: /* Pseudo column filter processing. Think about refactoring into a separate class. */ bool processPseudoColFilters(uint32_t extentIndex, boost::shared_ptr > dbRootPMMap) const; - bool processOneFilterType(int8_t colWidth, int64_t value, uint32_t type) const; - bool processSingleFilterString(int8_t BOP, int8_t colWidth, int64_t val, const uint8_t* filterString, + template + bool processOneFilterType(int8_t colWidth, T value, uint32_t type) const; + template + bool processSingleFilterString(int8_t BOP, int8_t colWidth, T val, const uint8_t* filterString, uint32_t filterCount) const; bool processSingleFilterString_ranged(int8_t BOP, int8_t colWidth, int64_t min, int64_t max, const uint8_t* filterString, uint32_t filterCount) const; bool processLBIDFilter(const BRM::EMEntry& emEntry) const; - bool compareSingleValue(uint8_t COP, int64_t val1, int64_t val2) const; + template + bool compareSingleValue(uint8_t COP, T val1, T val2) const; bool compareRange(uint8_t COP, int64_t min, int64_t max, int64_t val) const; bool hasPCFilter, hasPMFilter, hasRIDFilter, hasSegmentFilter, hasDBRootFilter, hasSegmentDirFilter, hasPartitionFilter, hasMaxFilter, hasMinFilter, hasLBIDFilter, hasExtentIDFilter; diff --git a/dbcon/joblist/pseudocc-jl.cpp b/dbcon/joblist/pseudocc-jl.cpp index 8ad0ccb2e..65b5a2317 100644 --- a/dbcon/joblist/pseudocc-jl.cpp +++ b/dbcon/joblist/pseudocc-jl.cpp @@ -46,23 +46,57 @@ void PseudoCCJL::runCommand(ByteStream& bs) const { if (function == PSEUDO_EXTENTMAX) { - int64_t max = extents[currentExtentIndex].partition.cprange.hi_val; - int64_t min = extents[currentExtentIndex].partition.cprange.lo_val; + if (!datatypes::Decimal::isWideDecimalType(colType)) + { + int64_t max = extents[currentExtentIndex].partition.cprange.hi_val; + int64_t min = extents[currentExtentIndex].partition.cprange.lo_val; - if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) - bs << max; + if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) + bs << max; + else + bs << utils::getNullValue(colType.colDataType, colType.colWidth); + } else - bs << utils::getNullValue(colType.colDataType, colType.colWidth); + { + int128_t max = extents[currentExtentIndex].partition.cprange.bigHiVal; + int128_t min = extents[currentExtentIndex].partition.cprange.bigLoVal; + + if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) + bs << (uint128_t) max; + else + { + int128_t int128Null; + utils::setWideDecimalNullValue(int128Null); + bs << (uint128_t) int128Null; + } + } } else if (function == PSEUDO_EXTENTMIN) { - int64_t max = extents[currentExtentIndex].partition.cprange.hi_val; - int64_t min = extents[currentExtentIndex].partition.cprange.lo_val; + if (!datatypes::Decimal::isWideDecimalType(colType)) + { + int64_t max = extents[currentExtentIndex].partition.cprange.hi_val; + int64_t min = extents[currentExtentIndex].partition.cprange.lo_val; - if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) - bs << min; + if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) + bs << min; + else + bs << utils::getNullValue(colType.colDataType, colType.colWidth); + } else - bs << utils::getNullValue(colType.colDataType, colType.colWidth); + { + int128_t max = extents[currentExtentIndex].partition.cprange.bigHiVal; + int128_t min = extents[currentExtentIndex].partition.cprange.bigLoVal; + + if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) + bs << (uint128_t) min; + else + { + int128_t int128Null; + utils::setWideDecimalNullValue(int128Null); + bs << (uint128_t) int128Null; + } + } } else if (function == PSEUDO_EXTENTID) bs << extents[currentExtentIndex].range.start; diff --git a/dbcon/joblist/rowestimator.cpp b/dbcon/joblist/rowestimator.cpp index 21369332f..1a5e5f73d 100644 --- a/dbcon/joblist/rowestimator.cpp +++ b/dbcon/joblist/rowestimator.cpp @@ -140,12 +140,13 @@ uint64_t RowEstimator::adjustValue(const execplan::CalpontSystemCatalog::ColType // Estimates the number of distinct values given a min/max range. When the range has not been set, // rules from the requirements are used based on the column type. +template uint32_t RowEstimator::estimateDistinctValues(const execplan::CalpontSystemCatalog::ColType& ct, - const uint64_t& min, - const uint64_t& max, + const T& min, + const T& max, const char cpStatus) { - uint64_t ret = 10; + T ret = 10; // If no casual partitioning info available for extent. These rules were defined in the requirements. if (cpStatus != BRM::CP_VALID) @@ -199,7 +200,7 @@ uint32_t RowEstimator::estimateDistinctValues(const execplan::CalpontSystemCatal ret = max - min + 1; } - if (ret > fRowsPerExtent) + if (ret > (T) fRowsPerExtent) { ret = fRowsPerExtent; } @@ -210,7 +211,10 @@ uint32_t RowEstimator::estimateDistinctValues(const execplan::CalpontSystemCatal // Returns a floating point number between 0 and 1 representing the percentage of matching rows for the given predicate against // the given range. This function is used for estimating an individual operation such as col1 = 2. template -float RowEstimator::estimateOpFactor(const T& min, const T& max, const T& value, char op, uint8_t lcf, uint32_t distinctValues, char cpStatus) +float RowEstimator::estimateOpFactor(const T& min, const T& max, const T& value, + char op, uint8_t lcf, + uint32_t distinctValues, char cpStatus, + const execplan::CalpontSystemCatalog::ColType& ct) { float factor = 1.0; @@ -220,7 +224,10 @@ float RowEstimator::estimateOpFactor(const T& min, const T& max, const T& value, case COMPARE_NGE: if (cpStatus == BRM::CP_VALID) { - factor = (1.0 * value - min) / (max - min + 1); + if (!datatypes::Decimal::isWideDecimalType(ct)) + factor = (1.0 * value - min) / (max - min + 1); + else + factor = ((__float128) value - min) / (max - min + 1); } break; @@ -229,7 +236,10 @@ float RowEstimator::estimateOpFactor(const T& min, const T& max, const T& value, case COMPARE_NGT: if (cpStatus == BRM::CP_VALID) { - factor = (1.0 * value - min + 1) / (max - min + 1); + if (!datatypes::Decimal::isWideDecimalType(ct)) + factor = (1.0 * value - min + 1) / (max - min + 1); + else + factor = ((__float128) value - min + 1) / (max - min + 1); } break; @@ -238,7 +248,10 @@ float RowEstimator::estimateOpFactor(const T& min, const T& max, const T& value, case COMPARE_NLE: if (cpStatus == BRM::CP_VALID) { - factor = (1.0 * max - value) / (1.0 * max - min + 1); + if (!datatypes::Decimal::isWideDecimalType(ct)) + factor = (1.0 * max - value) / (1.0 * max - min + 1); + else + factor = (1.0 * max - value) / (1.0 * max - min + 1); } break; @@ -248,7 +261,10 @@ float RowEstimator::estimateOpFactor(const T& min, const T& max, const T& value, if (cpStatus == BRM::CP_VALID) { // TODO: Best way to convert to floating point arithmetic? - factor = (1.0 * max - value + 1) / (max - min + 1); + if (!datatypes::Decimal::isWideDecimalType(ct)) + factor = (1.0 * max - value + 1) / (max - min + 1); + else + factor = (1.0 * max - value + 1) / (max - min + 1); } break; @@ -287,11 +303,26 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, float factor = 1.0; float tempFactor = 1.0; + uint64_t adjustedMin, adjustedMax; + uint128_t adjustedBigMin, adjustedBigMax; + uint32_t distinctValuesEstimate; + // Adjust values based on column type and estimate the - uint64_t adjustedMin = adjustValue(ct, emEntry.partition.cprange.lo_val); - uint64_t adjustedMax = adjustValue(ct, emEntry.partition.cprange.hi_val); - uint32_t distinctValuesEstimate = estimateDistinctValues( - ct, adjustedMin, adjustedMax, emEntry.partition.cprange.isValid); + if (!datatypes::Decimal::isWideDecimalType(ct)) + { + adjustedMin = adjustValue(ct, emEntry.partition.cprange.lo_val); + adjustedMax = adjustValue(ct, emEntry.partition.cprange.hi_val); + distinctValuesEstimate = estimateDistinctValues( + ct, adjustedMin, adjustedMax, emEntry.partition.cprange.isValid); + } + else + { + adjustedBigMin = emEntry.partition.cprange.bigLoVal; + adjustedBigMax = emEntry.partition.cprange.bigHiVal; + distinctValuesEstimate = estimateDistinctValues( + ct, adjustedBigMin, adjustedBigMax, emEntry.partition.cprange.isValid); + } + // Loop through the operations and estimate the percentage of rows that will qualify. // For example, there are two operations for "col1 > 5 and col1 < 10": @@ -300,6 +331,7 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, int length = bs->length(), pos = 0; const char* msgDataPtr = (const char*) bs->buf(); int64_t value = 0; + int128_t bigValue = 0; bool firstQualifyingOrCondition = true; uint16_t comparisonLimit = (NOPS <= fMaxComparisons) ? NOPS : fMaxComparisons; @@ -343,6 +375,18 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, break; } + case 16: + { + if (ct.colDataType == execplan::CalpontSystemCatalog::DECIMAL || + ct.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) + { + uint128_t val = *(uint128_t*)msgDataPtr; + bigValue = static_cast(val); + break; + } + /* fall through */ + } + case 8: default: { @@ -377,6 +421,18 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, break; } + case 16: + { + if (ct.colDataType == execplan::CalpontSystemCatalog::DECIMAL || + ct.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) + { + int128_t val = *(int128_t*)msgDataPtr; + bigValue = static_cast(val); + break; + } + /* fall through */ + } + case 8: default: { @@ -404,15 +460,33 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, // Get the factor for the individual operation. if (bIsUnsigned) { - tempFactor = estimateOpFactor( - adjustedMin, adjustedMax, adjustValue(ct, value), op, lcf, - distinctValuesEstimate, emEntry.partition.cprange.isValid); + if (!datatypes::Decimal::isWideDecimalType(ct)) + { + tempFactor = estimateOpFactor( + adjustedMin, adjustedMax, adjustValue(ct, value), op, lcf, + distinctValuesEstimate, emEntry.partition.cprange.isValid, ct); + } + else + { + tempFactor = estimateOpFactor( + adjustedBigMin, adjustedBigMax, bigValue, op, lcf, + distinctValuesEstimate, emEntry.partition.cprange.isValid, ct); + } } else { - tempFactor = estimateOpFactor( - adjustedMin, adjustedMax, adjustValue(ct, value), op, lcf, - distinctValuesEstimate, emEntry.partition.cprange.isValid); + if (!datatypes::Decimal::isWideDecimalType(ct)) + { + tempFactor = estimateOpFactor( + adjustedMin, adjustedMax, adjustValue(ct, value), op, lcf, + distinctValuesEstimate, emEntry.partition.cprange.isValid, ct); + } + else + { + tempFactor = estimateOpFactor( + adjustedBigMin, adjustedBigMax, bigValue, op, lcf, + distinctValuesEstimate, emEntry.partition.cprange.isValid, ct); + } } #if ROW_EST_DEBUG @@ -611,4 +685,16 @@ uint64_t RowEstimator::estimateRowsForNonCPColumn(ColumnCommandJL& colCmd) return estimatedRows; } +template +uint32_t RowEstimator::estimateDistinctValues(const execplan::CalpontSystemCatalog::ColType& ct, + const int128_t& min, + const int128_t& max, + const char cpStatus); + +template +uint32_t RowEstimator::estimateDistinctValues(const execplan::CalpontSystemCatalog::ColType& ct, + const int64_t& min, + const int64_t& max, + const char cpStatus); + } //namespace joblist diff --git a/dbcon/joblist/rowestimator.h b/dbcon/joblist/rowestimator.h index 477bc2b30..a4c702a44 100644 --- a/dbcon/joblist/rowestimator.h +++ b/dbcon/joblist/rowestimator.h @@ -89,9 +89,10 @@ private: uint32_t daysThroughMonth(uint32_t mth); + template uint32_t estimateDistinctValues(const execplan::CalpontSystemCatalog::ColType& ct, - const uint64_t& min, - const uint64_t& max, + const T& min, + const T& max, const char cpStatus); /** @brief returns a factor between 0 and 1 for the estimate of rows that will qualify the given individual operation. @@ -106,7 +107,8 @@ private: */ template float estimateOpFactor(const T& min, const T& max, const T& value, char op, uint8_t lcf, - uint32_t distinctValues, char cpStatus); + uint32_t distinctValues, char cpStatus, + const execplan::CalpontSystemCatalog::ColType& ct); /** @brief returns a factor between 0 and 1 for the estimate of rows that will qualify * the given operation(s). diff --git a/dbcon/joblist/tuple-bps.cpp b/dbcon/joblist/tuple-bps.cpp index 4c65f2dc3..7df74cc54 100644 --- a/dbcon/joblist/tuple-bps.cpp +++ b/dbcon/joblist/tuple-bps.cpp @@ -1421,7 +1421,8 @@ void TupleBPS::sendJobs(const vector& jobs) } } -bool TupleBPS::compareSingleValue(uint8_t COP, int64_t val1, int64_t val2) const +template +bool TupleBPS::compareSingleValue(uint8_t COP, T val1, T val2) const { switch (COP) { @@ -1536,7 +1537,8 @@ bool TupleBPS::processSingleFilterString_ranged(int8_t BOP, int8_t colWidth, int return ret; } -bool TupleBPS::processSingleFilterString(int8_t BOP, int8_t colWidth, int64_t val, const uint8_t* filterString, +template +bool TupleBPS::processSingleFilterString(int8_t BOP, int8_t colWidth, T val, const uint8_t* filterString, uint32_t filterCount) const { uint j; @@ -1546,6 +1548,7 @@ bool TupleBPS::processSingleFilterString(int8_t BOP, int8_t colWidth, int64_t va { int8_t COP; int64_t val2; + int128_t bigVal2; bool thisPredicate; COP = *filterString++; filterString++; // skip the round var, don't think that applies here @@ -1572,11 +1575,19 @@ bool TupleBPS::processSingleFilterString(int8_t BOP, int8_t colWidth, int64_t va filterString += 8; break; + case 16: + bigVal2 = *((int128_t*) filterString); + filterString += 16; + break; + default: throw logic_error("invalid column width"); } - thisPredicate = compareSingleValue(COP, val, val2); + if (colWidth < 16) + thisPredicate = compareSingleValue(COP, (int64_t) val, val2); + else + thisPredicate = compareSingleValue(COP, (int128_t) val, bigVal2); if (j == 0) ret = thisPredicate; @@ -1590,7 +1601,8 @@ bool TupleBPS::processSingleFilterString(int8_t BOP, int8_t colWidth, int64_t va return ret; } -bool TupleBPS::processOneFilterType(int8_t colWidth, int64_t value, uint32_t type) const +template +bool TupleBPS::processOneFilterType(int8_t colWidth, T value, uint32_t type) const { const vector& filters = fBPP->getFilterSteps(); uint i; @@ -1681,9 +1693,13 @@ bool TupleBPS::processPseudoColFilters(uint32_t extentIndex, boost::shared_ptrsetJoinFERG(rg); } -void TupleBPS::addCPPredicates(uint32_t OID, const vector& vals, bool isRange) +void TupleBPS::addCPPredicates(uint32_t OID, const vector<__int128>& vals, bool isRange, + bool isSmallSideWideDecimal) { if (fTraceFlags & CalpontSelectExecutionPlan::IGNORE_CP || fOid < 3000) @@ -3200,6 +3221,7 @@ void TupleBPS::addCPPredicates(uint32_t OID, const vector& vals, bool i uint32_t i, j, k; int64_t min, max, seq; + __int128 bigMin, bigMax; bool isValid, intersection; vector colCmdVec = fBPP->getFilterSteps(); ColumnCommandJL* cmd; @@ -3223,7 +3245,9 @@ void TupleBPS::addCPPredicates(uint32_t OID, const vector& vals, bool i if (cmd != NULL && cmd->getOID() == OID) { - if (!ll.CasualPartitionDataType(cmd->getColType().colDataType, cmd->getColType().colWidth) + const execplan::CalpontSystemCatalog::ColType& colType = cmd->getColType(); + + if (!ll.CasualPartitionDataType(colType.colDataType, colType.colWidth) || cmd->isDict()) return; @@ -3250,25 +3274,73 @@ void TupleBPS::addCPPredicates(uint32_t OID, const vector& vals, bool i extentsPtr = &mref; } - for (j = 0; j < extents.size(); j++) + if (colType.colWidth <= 8) { - isValid = ll.GetMinMax(&min, &max, &seq, extents[j].range.start, *extentsPtr, - cmd->getColType().colDataType); - - if (isValid) + for (j = 0; j < extents.size(); j++) { - if (isRange) - runtimeCPFlags[j] = ll.checkRangeOverlap(min, max, vals[0], vals[1], - cmd->getColType().colDataType) && runtimeCPFlags[j]; - else + isValid = ll.GetMinMax(&min, &max, &seq, extents[j].range.start, *extentsPtr, + colType.colDataType); + + if (isValid) { - intersection = false; + if (isRange) + { + if (!isSmallSideWideDecimal) + { + runtimeCPFlags[j] = ll.checkRangeOverlap(min, max, (int64_t) vals[0], (int64_t) vals[1], + colType.colDataType) && runtimeCPFlags[j]; + } + else + { + runtimeCPFlags[j] = ll.checkRangeOverlap((__int128) min, (__int128) max, vals[0], vals[1], + colType.colDataType) && runtimeCPFlags[j]; + } + } + else + { + intersection = false; - for (k = 0; k < vals.size(); k++) - intersection = intersection || - ll.checkSingleValue(min, max, vals[k], cmd->getColType().colDataType); + for (k = 0; k < vals.size(); k++) + { + if (!isSmallSideWideDecimal) + { + intersection = intersection || + ll.checkSingleValue(min, max, (int64_t) vals[k], colType.colDataType); + } + else + { + intersection = intersection || + ll.checkSingleValue((__int128) min, (__int128) max, vals[k], colType.colDataType); + } + } - runtimeCPFlags[j] = intersection && runtimeCPFlags[j]; + runtimeCPFlags[j] = intersection && runtimeCPFlags[j]; + } + } + } + } + else + { + for (j = 0; j < extents.size(); j++) + { + isValid = ll.GetMinMax(&bigMin, &bigMax, &seq, extents[j].range.start, *extentsPtr, + colType.colDataType); + + if (isValid) + { + if (isRange) + runtimeCPFlags[j] = ll.checkRangeOverlap(bigMin, bigMax, vals[0], vals[1], + colType.colDataType) && runtimeCPFlags[j]; + else + { + intersection = false; + + for (k = 0; k < vals.size(); k++) + intersection = intersection || + ll.checkSingleValue(bigMin, bigMax, vals[k], colType.colDataType); + + runtimeCPFlags[j] = intersection && runtimeCPFlags[j]; + } } } } @@ -3326,5 +3398,22 @@ void TupleBPS::abort() abort_nolock(); } +template +bool TupleBPS::processOneFilterType(int8_t colWidth, int64_t value, uint32_t type) const; +template +bool TupleBPS::processOneFilterType(int8_t colWidth, int128_t value, uint32_t type) const; + +template +bool TupleBPS::processSingleFilterString(int8_t BOP, int8_t colWidth, int64_t val, const uint8_t* filterString, + uint32_t filterCount) const; +template +bool TupleBPS::processSingleFilterString(int8_t BOP, int8_t colWidth, int128_t val, const uint8_t* filterString, + uint32_t filterCount) const; + +template +bool TupleBPS::compareSingleValue(uint8_t COP, int64_t val1, int64_t val2) const; +template +bool TupleBPS::compareSingleValue(uint8_t COP, int128_t val1, int128_t val2) const; + } //namespace // vim:ts=4 sw=4: diff --git a/dbcon/joblist/tuplehashjoin.cpp b/dbcon/joblist/tuplehashjoin.cpp index e7ddbb235..35639814a 100644 --- a/dbcon/joblist/tuplehashjoin.cpp +++ b/dbcon/joblist/tuplehashjoin.cpp @@ -476,7 +476,9 @@ void TupleHashJoinStep::forwardCPData() for (col = 0; col < joiners[i]->getSmallKeyColumns().size(); col++) { - if (smallRGs[i].isLongString(joiners[i]->getSmallKeyColumns()[col])) + uint32_t idx = joiners[i]->getSmallKeyColumns()[col]; + + if (smallRGs[i].isLongString(idx)) continue; // @bug3683, not to add CP predicates if large side is not simple column @@ -484,8 +486,12 @@ void TupleHashJoinStep::forwardCPData() fFunctionJoinKeys.end()) continue; + bool isSmallSideWideDecimal = + datatypes::Decimal::isWideDecimalType(smallRGs[i].getColType(idx), smallRGs[i].getColumnWidth(idx)); + largeBPS->addCPPredicates(largeRG.getOIDs()[joiners[i]->getLargeKeyColumns()[col]], - joiners[i]->getCPData()[col], !joiners[i]->discreteCPValues()[col]); + joiners[i]->getCPData()[col], !joiners[i]->discreteCPValues()[col], + isSmallSideWideDecimal); } } } diff --git a/dbcon/mysql/ha_mcs_partition.cpp b/dbcon/mysql/ha_mcs_partition.cpp index 7271fba51..8b64ac3c3 100644 --- a/dbcon/mysql/ha_mcs_partition.cpp +++ b/dbcon/mysql/ha_mcs_partition.cpp @@ -48,6 +48,7 @@ using namespace BRM; #include "dataconvert.h" using namespace dataconvert; +#include "widedecimalutils.h" #include "ddlpkg.h" #include "sqlparser.h" using namespace ddlpackage; @@ -250,8 +251,8 @@ struct PartitionInfo max((uint64_t) - 0x8000000000000001LL), status(0) { - DataConvert::int128Min(bigMin); - DataConvert::int128Max(bigMax); + utils::int128Min(bigMin); + utils::int128Max(bigMax); }; }; @@ -311,7 +312,7 @@ const string format(T v, CalpontSystemCatalog::ColType& ct) } else { - char buf[MAX_DECIMAL_STRING_LENGTH]; + char buf[utils::MAXLENGTH16BYTES]; DataConvert::decimalToString((__int128*)&v, (unsigned)ct.scale, buf, sizeof(buf), ct.colDataType); oss << buf; } @@ -339,9 +340,10 @@ const string format(T v, CalpontSystemCatalog::ColType& ct) return oss.str(); } -int64_t IDB_format(char* str, CalpontSystemCatalog::ColType& ct, uint8_t& rf) +template +T IDB_format(char* str, CalpontSystemCatalog::ColType& ct, uint8_t& rf) { - int64_t v = 0; + T v = 0; bool pushWarning = false; rf = 0; boost::any anyVal = DataConvert::convertColumnData(ct, str, pushWarning, current_thd->variables.time_zone->get_name()->ptr(), false, true, false); @@ -433,8 +435,10 @@ int64_t IDB_format(char* str, CalpontSystemCatalog::ColType& ct, uint8_t& rf) #else v = boost::any_cast(anyVal); #endif - else + else if (ct.colWidth == execplan::CalpontSystemCatalog::EIGHT_BYTE) v = boost::any_cast(anyVal); + else + v = boost::any_cast(anyVal); break; @@ -656,6 +660,7 @@ void partitionByValue_common(UDF_ARGS* args, // input string schema, table, column; CalpontSystemCatalog::ColType ct; int64_t startVal, endVal; + int128_t bigStartVal, bigEndVal; uint8_t rfMin = 0, rfMax = 0; if (args->arg_count == 5) @@ -728,68 +733,136 @@ void partitionByValue_common(UDF_ARGS* args, // input { if (!args->args[2]) { - if (isUnsigned(ct.colDataType)) + if (!datatypes::Decimal::isWideDecimalType(ct)) { - startVal = 0; + if (isUnsigned(ct.colDataType)) + { + startVal = 0; + } + else + { + startVal = numeric_limits::min(); + } } else { - startVal = numeric_limits::min(); + if (isUnsigned(ct.colDataType)) + { + bigStartVal = 0; + } + else + { + utils::int128Min(bigStartVal); + } } } else { - startVal = IDB_format((char*) args->args[2], ct, rfMin); + if (!datatypes::Decimal::isWideDecimalType(ct)) + startVal = IDB_format((char*) args->args[2], ct, rfMin); + else + bigStartVal = IDB_format((char*) args->args[2], ct, rfMin); } if (!args->args[3]) { - if (isUnsigned(ct.colDataType)) + if (!datatypes::Decimal::isWideDecimalType(ct)) { - endVal = static_cast(numeric_limits::max()); + if (isUnsigned(ct.colDataType)) + { + endVal = static_cast(numeric_limits::max()); + } + else + { + endVal = numeric_limits::max(); + } } else { - endVal = numeric_limits::max(); + if (isUnsigned(ct.colDataType)) + { + bigEndVal = -1; + } + else + { + utils::int128Max(bigEndVal); + } } } else { - endVal = IDB_format((char*) args->args[3], ct, rfMax); + if (!datatypes::Decimal::isWideDecimalType(ct)) + endVal = IDB_format((char*) args->args[3], ct, rfMax); + else + bigEndVal = IDB_format((char*) args->args[3], ct, rfMax); } } else { if (!args->args[3]) { - if (isUnsigned(ct.colDataType)) + if (!datatypes::Decimal::isWideDecimalType(ct)) { - startVal = 0; + if (isUnsigned(ct.colDataType)) + { + startVal = 0; + } + else + { + startVal = numeric_limits::min(); + } } else { - startVal = numeric_limits::min(); + if (isUnsigned(ct.colDataType)) + { + bigStartVal = 0; + } + else + { + utils::int128Min(bigStartVal); + } } } else { - startVal = IDB_format((char*) args->args[3], ct, rfMin); + if (!datatypes::Decimal::isWideDecimalType(ct)) + startVal = IDB_format((char*) args->args[3], ct, rfMin); + else + bigStartVal = IDB_format((char*) args->args[3], ct, rfMin); } if (!args->args[4]) { - if (isUnsigned(ct.colDataType)) + if (!datatypes::Decimal::isWideDecimalType(ct)) { - endVal = static_cast(numeric_limits::max()); + if (isUnsigned(ct.colDataType)) + { + endVal = static_cast(numeric_limits::max()); + } + else + { + endVal = numeric_limits::max(); + } } else { - endVal = numeric_limits::max(); + if (isUnsigned(ct.colDataType)) + { + bigEndVal = -1; + } + else + { + utils::int128Max(bigEndVal); + } } } else { - endVal = IDB_format((char*) args->args[4], ct, rfMax); + if (!datatypes::Decimal::isWideDecimalType(ct)) + endVal = IDB_format((char*) args->args[4], ct, rfMax); + else + bigEndVal = IDB_format((char*) args->args[4], ct, rfMax); } } @@ -810,7 +883,13 @@ void partitionByValue_common(UDF_ARGS* args, // input partInfo.status |= ET_DISABLED; mapit = partMap.find(logicalPartNum); - int state = em.getExtentMaxMin(iter->range.start, partInfo.max, partInfo.min, seqNum); + + int state; + + if (!datatypes::Decimal::isWideDecimalType(ct)) + state = em.getExtentMaxMin(iter->range.start, partInfo.max, partInfo.min, seqNum); + else + state = em.getExtentMaxMin(iter->range.start, partInfo.bigMax, partInfo.bigMin, seqNum); // char column order swap if ((ct.colDataType == CalpontSystemCatalog::CHAR && ct.colWidth <= 8) || @@ -832,17 +911,35 @@ void partitionByValue_common(UDF_ARGS* args, // input if (mapit->second.status & CPINVALID) continue; - if (isUnsigned(ct.colDataType)) + if (!datatypes::Decimal::isWideDecimalType(ct)) { - mapit->second.min = - (static_cast(partInfo.min) < static_cast(mapit->second.min) ? partInfo.min : mapit->second.min); - mapit->second.max = - (static_cast(partInfo.max) > static_cast(mapit->second.max) ? partInfo.max : mapit->second.max); + if (isUnsigned(ct.colDataType)) + { + mapit->second.min = + (static_cast(partInfo.min) < static_cast(mapit->second.min) ? partInfo.min : mapit->second.min); + mapit->second.max = + (static_cast(partInfo.max) > static_cast(mapit->second.max) ? partInfo.max : mapit->second.max); + } + else + { + mapit->second.min = (partInfo.min < mapit->second.min ? partInfo.min : mapit->second.min); + mapit->second.max = (partInfo.max > mapit->second.max ? partInfo.max : mapit->second.max); + } } else { - mapit->second.min = (partInfo.min < mapit->second.min ? partInfo.min : mapit->second.min); - mapit->second.max = (partInfo.max > mapit->second.max ? partInfo.max : mapit->second.max); + if (isUnsigned(ct.colDataType)) + { + mapit->second.bigMin = + (static_cast(partInfo.bigMin) < static_cast(mapit->second.bigMin) ? partInfo.bigMin : mapit->second.bigMin); + mapit->second.bigMax = + (static_cast(partInfo.bigMax) > static_cast(mapit->second.bigMax) ? partInfo.bigMax : mapit->second.bigMax); + } + else + { + mapit->second.bigMin = (partInfo.bigMin < mapit->second.bigMin ? partInfo.bigMin : mapit->second.bigMin); + mapit->second.bigMax = (partInfo.bigMax > mapit->second.bigMax ? partInfo.bigMax : mapit->second.bigMax); + } } } } @@ -851,35 +948,72 @@ void partitionByValue_common(UDF_ARGS* args, // input for (mapit = partMap.begin(); mapit != partMap.end(); ++mapit) { // @bug 4595. check empty/null case - if (isUnsigned(ct.colDataType)) + if (!datatypes::Decimal::isWideDecimalType(ct)) { - if (!(mapit->second.status & CPINVALID) && - static_cast(mapit->second.min) >= static_cast(startVal) && - static_cast(mapit->second.max) <= static_cast(endVal) && - !(static_cast(mapit->second.min) == numeric_limits::max() && - static_cast(mapit->second.max == 0))) + if (isUnsigned(ct.colDataType)) { - if (rfMin == ROUND_POS && mapit->second.min == startVal) - continue; + if (!(mapit->second.status & CPINVALID) && + static_cast(mapit->second.min) >= static_cast(startVal) && + static_cast(mapit->second.max) <= static_cast(endVal) && + !(static_cast(mapit->second.min) == numeric_limits::max() && + static_cast(mapit->second.max == 0))) + { + if (rfMin == ROUND_POS && mapit->second.min == startVal) + continue; - if (rfMax == ROUND_NEG && mapit->second.max == endVal) - continue; + if (rfMax == ROUND_NEG && mapit->second.max == endVal) + continue; - partSet.insert(mapit->first); + partSet.insert(mapit->first); + } + } + else + { + if (!(mapit->second.status & CPINVALID) && mapit->second.min >= startVal && mapit->second.max <= endVal && + !(mapit->second.min == numeric_limits::max() && mapit->second.max == numeric_limits::min())) + { + if (rfMin == ROUND_POS && mapit->second.min == startVal) + continue; + + if (rfMax == ROUND_NEG && mapit->second.max == endVal) + continue; + + partSet.insert(mapit->first); + } } } else { - if (!(mapit->second.status & CPINVALID) && mapit->second.min >= startVal && mapit->second.max <= endVal && - !(mapit->second.min == numeric_limits::max() && mapit->second.max == numeric_limits::min())) + if (isUnsigned(ct.colDataType)) { - if (rfMin == ROUND_POS && mapit->second.min == startVal) - continue; + if (!(mapit->second.status & CPINVALID) && + static_cast(mapit->second.bigMin) >= static_cast(bigStartVal) && + static_cast(mapit->second.bigMax) <= static_cast(bigEndVal) && + !(static_cast(mapit->second.bigMin) == static_cast(-1) && + static_cast(mapit->second.bigMax == 0))) + { + if (rfMin == ROUND_POS && mapit->second.bigMin == bigStartVal) + continue; - if (rfMax == ROUND_NEG && mapit->second.max == endVal) - continue; + if (rfMax == ROUND_NEG && mapit->second.bigMax == bigEndVal) + continue; - partSet.insert(mapit->first); + partSet.insert(mapit->first); + } + } + else + { + if (!(mapit->second.status & CPINVALID) && mapit->second.bigMin >= bigStartVal && mapit->second.bigMax <= bigEndVal && + !(mapit->second.bigMin == utils::maxInt128 && mapit->second.bigMax == utils::minInt128)) + { + if (rfMin == ROUND_POS && mapit->second.bigMin == bigStartVal) + continue; + + if (rfMax == ROUND_NEG && mapit->second.bigMax == bigEndVal) + continue; + + partSet.insert(mapit->first); + } } } } @@ -1102,9 +1236,9 @@ extern "C" int state = CP_INVALID; - if (ct.colWidth <= 8) + if (!datatypes::Decimal::isWideDecimalType(ct)) state = em.getExtentMaxMin(iter->range.start, partInfo.max, partInfo.min, seqNum); - else if (ct.colWidth == 16) + else state = em.getExtentMaxMin(iter->range.start, partInfo.bigMax, partInfo.bigMin, seqNum); // char column order swap for compare @@ -1127,12 +1261,12 @@ extern "C" if (mapit->second.status & CPINVALID) continue; - if (ct.colWidth <= 8) + if (!datatypes::Decimal::isWideDecimalType(ct)) { mapit->second.min = (partInfo.min < mapit->second.min ? partInfo.min : mapit->second.min); mapit->second.max = (partInfo.max > mapit->second.max ? partInfo.max : mapit->second.max); } - else if (ct.colWidth == 16) + else { mapit->second.bigMin = (partInfo.bigMin < mapit->second.bigMin ? partInfo.bigMin : mapit->second.bigMin); mapit->second.bigMax = (partInfo.bigMax > mapit->second.bigMax ? partInfo.bigMax : mapit->second.bigMax); @@ -1156,7 +1290,7 @@ extern "C" ostringstream output; output.setf(ios::left, ios::adjustfield); - if (ct.colWidth <= 8) + if (!datatypes::Decimal::isWideDecimalType(ct)) { output << setw(10) << "Part#" << setw(30) << "Min" @@ -1165,18 +1299,18 @@ extern "C" else { output << setw(10) << "Part#" - << setw(40) << "Min" - << setw(40) << "Max" << "Status"; + << setw(42) << "Min" + << setw(42) << "Max" << "Status"; } int64_t maxLimit = numeric_limits::max(); int64_t minLimit = numeric_limits::min(); __int128 bigMaxLimit, bigMinLimit; - DataConvert::int128Max(bigMaxLimit); - DataConvert::int128Min(bigMinLimit); + utils::int128Max(bigMaxLimit); + utils::int128Min(bigMinLimit); unsigned __int128 ubigMaxLimit, ubigMinLimit; - DataConvert::uint128Max(ubigMaxLimit); + utils::uint128Max(ubigMaxLimit); ubigMinLimit = 0; // char column order swap for compare in subsequent loop @@ -1197,16 +1331,16 @@ extern "C" if (partIt->second.status & CPINVALID) { - if (ct.colWidth <= 8) + if (!datatypes::Decimal::isWideDecimalType(ct)) output << setw(30) << "N/A" << setw(30) << "N/A"; else - output << setw(40) << "N/A" << setw(40) << "N/A"; + output << setw(42) << "N/A" << setw(42) << "N/A"; } else { if ((isUnsigned(ct.colDataType))) { - if (ct.colWidth <= 8) + if (!datatypes::Decimal::isWideDecimalType(ct)) { if (static_cast(partIt->second.min) == numeric_limits::max() && static_cast(partIt->second.max) == numeric_limits::min()) @@ -1218,14 +1352,14 @@ extern "C" { if (static_cast(partIt->second.bigMin) == ubigMaxLimit && static_cast(partIt->second.bigMax) == ubigMinLimit) - output << setw(40) << "Empty/Null" << setw(40) << "Empty/Null"; + output << setw(42) << "Empty/Null" << setw(42) << "Empty/Null"; else - output << setw(40) << format(partIt->second.bigMin, ct) << setw(40) << format(partIt->second.bigMax, ct); + output << setw(42) << format(partIt->second.bigMin, ct) << setw(42) << format(partIt->second.bigMax, ct); } } else { - if (ct.colWidth <= 8) + if (!datatypes::Decimal::isWideDecimalType(ct)) { if (partIt->second.min == maxLimit && partIt->second.max == minLimit) output << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; @@ -1235,9 +1369,9 @@ extern "C" else { if (partIt->second.bigMin == bigMaxLimit && partIt->second.bigMax == bigMinLimit) - output << setw(40) << "Empty/Null" << setw(40) << "Empty/Null"; + output << setw(42) << "Empty/Null" << setw(42) << "Empty/Null"; else - output << setw(40) << format(partIt->second.bigMin, ct) << setw(40) << format(partIt->second.bigMax, ct); + output << setw(42) << format(partIt->second.bigMin, ct) << setw(42) << format(partIt->second.bigMax, ct); } } } @@ -1801,6 +1935,7 @@ extern "C" CalpontSystemCatalog::ColType ct; string errMsg; int64_t startVal, endVal; + int128_t bigStartVal, bigEndVal; uint8_t rfMin = 0, rfMax = 0; try @@ -1859,68 +1994,136 @@ extern "C" { if (!args->args[2]) { - if (isUnsigned(ct.colDataType)) + if (!datatypes::Decimal::isWideDecimalType(ct)) { - startVal = 0; + if (isUnsigned(ct.colDataType)) + { + startVal = 0; + } + else + { + startVal = numeric_limits::min(); + } } else { - startVal = numeric_limits::min(); + if (isUnsigned(ct.colDataType)) + { + bigStartVal = 0; + } + else + { + utils::int128Min(bigStartVal); + } } } else { - startVal = IDB_format((char*) args->args[2], ct, rfMin); + if (!datatypes::Decimal::isWideDecimalType(ct)) + startVal = IDB_format((char*) args->args[2], ct, rfMin); + else + bigStartVal = IDB_format((char*) args->args[2], ct, rfMin); } if (!args->args[3]) { - if (isUnsigned(ct.colDataType)) + if (!datatypes::Decimal::isWideDecimalType(ct)) { - endVal = static_cast(numeric_limits::max()); + if (isUnsigned(ct.colDataType)) + { + endVal = static_cast(numeric_limits::max()); + } + else + { + endVal = numeric_limits::max(); + } } else { - endVal = numeric_limits::max(); + if (isUnsigned(ct.colDataType)) + { + bigEndVal = -1; + } + else + { + utils::int128Max(bigEndVal); + } } } else { - endVal = IDB_format((char*) args->args[3], ct, rfMax); + if (!datatypes::Decimal::isWideDecimalType(ct)) + endVal = IDB_format((char*) args->args[3], ct, rfMax); + else + bigEndVal = IDB_format((char*) args->args[3], ct, rfMax); } } else { if (!args->args[3]) { - if (isUnsigned(ct.colDataType)) + if (!datatypes::Decimal::isWideDecimalType(ct)) { - startVal = 0; + if (isUnsigned(ct.colDataType)) + { + startVal = 0; + } + else + { + startVal = numeric_limits::min(); + } } else { - startVal = numeric_limits::min(); + if (isUnsigned(ct.colDataType)) + { + bigStartVal = 0; + } + else + { + utils::int128Min(bigStartVal); + } } } else { - startVal = IDB_format((char*) args->args[3], ct, rfMin); + if (!datatypes::Decimal::isWideDecimalType(ct)) + startVal = IDB_format((char*) args->args[3], ct, rfMin); + else + bigStartVal = IDB_format((char*) args->args[3], ct, rfMin); } if (!args->args[4]) { - if (isUnsigned(ct.colDataType)) + if (!datatypes::Decimal::isWideDecimalType(ct)) { - endVal = static_cast(numeric_limits::max()); + if (isUnsigned(ct.colDataType)) + { + endVal = static_cast(numeric_limits::max()); + } + else + { + endVal = numeric_limits::max(); + } } else { - endVal = numeric_limits::max(); + if (isUnsigned(ct.colDataType)) + { + bigEndVal = -1; + } + else + { + utils::int128Max(bigEndVal); + } } } else { - endVal = IDB_format((char*) args->args[4], ct, rfMax); + if (!datatypes::Decimal::isWideDecimalType(ct)) + endVal = IDB_format((char*) args->args[4], ct, rfMax); + else + bigEndVal = IDB_format((char*) args->args[4], ct, rfMax); } } @@ -1943,7 +2146,13 @@ extern "C" partInfo.status |= ET_DISABLED; mapit = partMap.find(logicalPartNum); - int state = em.getExtentMaxMin(iter->range.start, partInfo.max, partInfo.min, seqNum); + + int state; + + if (!datatypes::Decimal::isWideDecimalType(ct)) + state = em.getExtentMaxMin(iter->range.start, partInfo.max, partInfo.min, seqNum); + else + state = em.getExtentMaxMin(iter->range.start, partInfo.bigMax, partInfo.bigMin, seqNum); // char column order swap if ((ct.colDataType == CalpontSystemCatalog::CHAR && ct.colWidth <= 8) || @@ -1965,17 +2174,35 @@ extern "C" if (mapit->second.status & CPINVALID) continue; - if (isUnsigned(ct.colDataType)) + if (!datatypes::Decimal::isWideDecimalType(ct)) { - mapit->second.min = - (static_cast(partInfo.min) < static_cast(mapit->second.min) ? partInfo.min : mapit->second.min); - mapit->second.max = - (static_cast(partInfo.max) > static_cast(mapit->second.max) ? partInfo.max : mapit->second.max); + if (isUnsigned(ct.colDataType)) + { + mapit->second.min = + (static_cast(partInfo.min) < static_cast(mapit->second.min) ? partInfo.min : mapit->second.min); + mapit->second.max = + (static_cast(partInfo.max) > static_cast(mapit->second.max) ? partInfo.max : mapit->second.max); + } + else + { + mapit->second.min = (partInfo.min < mapit->second.min ? partInfo.min : mapit->second.min); + mapit->second.max = (partInfo.max > mapit->second.max ? partInfo.max : mapit->second.max); + } } else { - mapit->second.min = (partInfo.min < mapit->second.min ? partInfo.min : mapit->second.min); - mapit->second.max = (partInfo.max > mapit->second.max ? partInfo.max : mapit->second.max); + if (isUnsigned(ct.colDataType)) + { + mapit->second.bigMin = + (static_cast(partInfo.bigMin) < static_cast(mapit->second.bigMin) ? partInfo.bigMin : mapit->second.bigMin); + mapit->second.bigMax = + (static_cast(partInfo.bigMax) > static_cast(mapit->second.bigMax) ? partInfo.bigMax : mapit->second.bigMax); + } + else + { + mapit->second.bigMin = (partInfo.bigMin < mapit->second.bigMin ? partInfo.bigMin : mapit->second.bigMin); + mapit->second.bigMax = (partInfo.bigMax > mapit->second.bigMax ? partInfo.bigMax : mapit->second.bigMax); + } } } } @@ -2010,57 +2237,115 @@ extern "C" for (mapit = partMap.begin(); mapit != partMap.end(); ++mapit) { // @bug 4595. check empty/null case - if (!(mapit->second.status & CPINVALID) && mapit->second.min >= startVal && mapit->second.max <= endVal && - !(mapit->second.min == numeric_limits::max() && mapit->second.max == numeric_limits::min())) + if (!datatypes::Decimal::isWideDecimalType(ct)) { - if (rfMin == ROUND_POS && mapit->second.min == startVal) - continue; - - if (rfMax == ROUND_NEG && mapit->second.max == endVal) - continue; - - // print header - if (noPartFound) + if (!(mapit->second.status & CPINVALID) && mapit->second.min >= startVal && mapit->second.max <= endVal && + !(mapit->second.min == numeric_limits::max() && mapit->second.max == numeric_limits::min())) { - output.setf(ios::left, ios::adjustfield); - output << setw(10) << "Part#" - << setw(30) << "Min" - << setw(30) << "Max" << "Status"; - } + if (rfMin == ROUND_POS && mapit->second.min == startVal) + continue; - noPartFound = false; + if (rfMax == ROUND_NEG && mapit->second.max == endVal) + continue; - // print part info - ostringstream oss; - oss << mapit->first; - output << "\n " << setw(10) << oss.str(); - - if (mapit->second.status & CPINVALID) - { - output << setw(30) << "N/A" << setw(30) << "N/A"; - } - else - { - if ((isUnsigned(ct.colDataType))) + // print header + if (noPartFound) { - if (static_cast(mapit->second.min) > static_cast(mapit->second.max)) - output << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; - else - output << setw(30) << format(mapit->second.min, ct) << setw(30) << format(mapit->second.max, ct); + output.setf(ios::left, ios::adjustfield); + output << setw(10) << "Part#" + << setw(30) << "Min" + << setw(30) << "Max" << "Status"; + } + + noPartFound = false; + + // print part info + ostringstream oss; + oss << mapit->first; + output << "\n " << setw(10) << oss.str(); + + if (mapit->second.status & CPINVALID) + { + output << setw(30) << "N/A" << setw(30) << "N/A"; } else { - if (mapit->second.min > mapit->second.max) - output << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; + if ((isUnsigned(ct.colDataType))) + { + if (static_cast(mapit->second.min) > static_cast(mapit->second.max)) + output << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; + else + output << setw(30) << format(mapit->second.min, ct) << setw(30) << format(mapit->second.max, ct); + } else - output << setw(30) << format(mapit->second.min, ct) << setw(30) << format(mapit->second.max, ct); + { + if (mapit->second.min > mapit->second.max) + output << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; + else + output << setw(30) << format(mapit->second.min, ct) << setw(30) << format(mapit->second.max, ct); + } } - } - if (mapit->second.status & ET_DISABLED) - output << "Disabled"; - else - output << "Enabled"; + if (mapit->second.status & ET_DISABLED) + output << "Disabled"; + else + output << "Enabled"; + } + } + else + { + if (!(mapit->second.status & CPINVALID) && mapit->second.bigMin >= bigStartVal && mapit->second.bigMax <= bigEndVal && + !(mapit->second.bigMin == utils::maxInt128 && mapit->second.bigMax == utils::minInt128)) + { + if (rfMin == ROUND_POS && mapit->second.bigMin == bigStartVal) + continue; + + if (rfMax == ROUND_NEG && mapit->second.bigMax == bigEndVal) + continue; + + // print header + if (noPartFound) + { + output.setf(ios::left, ios::adjustfield); + output << setw(10) << "Part#" + << setw(42) << "Min" + << setw(42) << "Max" << "Status"; + } + + noPartFound = false; + + // print part info + ostringstream oss; + oss << mapit->first; + output << "\n " << setw(10) << oss.str(); + + if (mapit->second.status & CPINVALID) + { + output << setw(42) << "N/A" << setw(42) << "N/A"; + } + else + { + if ((isUnsigned(ct.colDataType))) + { + if (static_cast(mapit->second.bigMin) > static_cast(mapit->second.bigMax)) + output << setw(42) << "Empty/Null" << setw(42) << "Empty/Null"; + else + output << setw(42) << format(mapit->second.bigMin, ct) << setw(42) << format(mapit->second.bigMax, ct); + } + else + { + if (mapit->second.bigMin > mapit->second.bigMax) + output << setw(42) << "Empty/Null" << setw(42) << "Empty/Null"; + else + output << setw(42) << format(mapit->second.bigMin, ct) << setw(42) << format(mapit->second.bigMax, ct); + } + } + + if (mapit->second.status & ET_DISABLED) + output << "Disabled"; + else + output << "Enabled"; + } } } diff --git a/dbcon/mysql/is_columnstore_extents.cpp b/dbcon/mysql/is_columnstore_extents.cpp index 188ce31dc..d93152a52 100644 --- a/dbcon/mysql/is_columnstore_extents.cpp +++ b/dbcon/mysql/is_columnstore_extents.cpp @@ -27,6 +27,9 @@ #include "dbrm.h" #include "objectidmanager.h" #include "is_columnstore.h" +#include "mcs_decimal.h" +#include "widedecimalutils.h" +#include "dataconvert.h" // Required declaration as it isn't in a MairaDB include bool schema_table_store_record(THD* thd, TABLE* table); @@ -37,8 +40,10 @@ ST_FIELD_INFO is_columnstore_extents_fields[] = Show::Column("OBJECT_TYPE", Show::Varchar(64), NOT_NULL), // 1 Show::Column("LOGICAL_BLOCK_START", Show::SLonglong(0), NOT_NULL), // 2 Show::Column("LOGICAL_BLOCK_END", Show::SLonglong(0), NOT_NULL), // 3 - Show::Column("MIN_VALUE", Show::SLonglong(0), NULLABLE), // 4 - Show::Column("MAX_VALUE", Show::SLonglong(0), NULLABLE), // 5 + // length=3800 here because sql/sql_i_s.h sets + // decimal_precision() as (length / 100) % 100). Not sure why. + Show::Column("MIN_VALUE", Show::Decimal(3800), NULLABLE), // 4 + Show::Column("MAX_VALUE", Show::Decimal(3800), NULLABLE), // 5 Show::Column("WIDTH", Show::ULong(0), NOT_NULL), // 6 Show::Column("DBROOT", Show::ULong(0), NOT_NULL), // 7 Show::Column("PARTITION_ID", Show::ULong(0), NOT_NULL), // 8 @@ -75,26 +80,61 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM* emp, TABLE* table, THD* th { table->field[1]->store("Column", strlen("Column"), cs); - if (iter->partition.cprange.lo_val == std::numeric_limits::max() || - iter->partition.cprange.lo_val <= (std::numeric_limits::min() + 2)) + if (iter->colWid != datatypes::MAXDECIMALWIDTH) { - table->field[4]->set_null(); - } - else - { - table->field[4]->set_notnull(); - table->field[4]->store(iter->partition.cprange.lo_val); - } + if (iter->partition.cprange.lo_val == std::numeric_limits::max() || + iter->partition.cprange.lo_val <= (std::numeric_limits::min() + 1)) + { + table->field[4]->set_null(); + } + else + { + table->field[4]->set_notnull(); + table->field[4]->store(iter->partition.cprange.lo_val); + } - if (iter->partition.cprange.hi_val == std::numeric_limits::max() || - iter->partition.cprange.hi_val <= (std::numeric_limits::min() + 2)) - { - table->field[5]->set_null(); + if (iter->partition.cprange.hi_val <= (std::numeric_limits::min() + 1)) + { + table->field[5]->set_null(); + } + else + { + table->field[5]->set_notnull(); + table->field[5]->store(iter->partition.cprange.hi_val); + } } else { - table->field[5]->set_notnull(); - table->field[5]->store(iter->partition.cprange.hi_val); + if (iter->partition.cprange.bigLoVal == utils::maxInt128 || + iter->partition.cprange.bigLoVal <= (utils::minInt128 + 1)) + { + table->field[4]->set_null(); + } + else + { + table->field[4]->set_notnull(); + + char buf[utils::MAXLENGTH16BYTES]; + dataconvert::DataConvert::decimalToString( + &iter->partition.cprange.bigLoVal, + 0, buf, sizeof(buf), execplan::CalpontSystemCatalog::DECIMAL); + table->field[4]->store(buf, strlen(buf), table->field[4]->charset()); + } + + if (iter->partition.cprange.bigHiVal <= (utils::minInt128 + 1)) + { + table->field[5]->set_null(); + } + else + { + table->field[5]->set_notnull(); + + char buf[utils::MAXLENGTH16BYTES]; + dataconvert::DataConvert::decimalToString( + &iter->partition.cprange.bigHiVal, + 0, buf, sizeof(buf), execplan::CalpontSystemCatalog::DECIMAL); + table->field[5]->store(buf, strlen(buf), table->field[5]->charset()); + } } table->field[6]->store(iter->colWid); diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index d9724d727..446d81e05 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -1597,8 +1597,8 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, } else { - dataconvert::DataConvert::int128Max(out->Min); - dataconvert::DataConvert::int128Min(out->Max); + utils::int128Max(out->Min); + utils::int128Min(out->Max); } } else diff --git a/primitives/primproc/batchprimitiveprocessor.cpp b/primitives/primproc/batchprimitiveprocessor.cpp index 4a2021daa..999f95518 100644 --- a/primitives/primproc/batchprimitiveprocessor.cpp +++ b/primitives/primproc/batchprimitiveprocessor.cpp @@ -54,6 +54,7 @@ using namespace boost; #include "MonitorProcMem.h" #include "threadnaming.h" #include "vlarray.h" +#include "widedecimalutils.h" #define MAX64 0x7fffffffffffffffLL #define MIN64 0x8000000000000000LL @@ -1098,8 +1099,8 @@ void BatchPrimitiveProcessor::initProcessor() } else { - dataconvert::DataConvert::int128Min(bigMaxVal); - dataconvert::DataConvert::int128Max(bigMinVal); + utils::int128Min(bigMaxVal); + utils::int128Max(bigMinVal); } // @bug 1269, initialize data used by execute() for async loading blocks @@ -2213,8 +2214,8 @@ int BatchPrimitiveProcessor::operator()() } else { - dataconvert::DataConvert::int128Min(bigMaxVal); - dataconvert::DataConvert::int128Max(bigMinVal); + utils::int128Min(bigMaxVal); + utils::int128Max(bigMinVal); } validCPData = false; #ifdef PRIMPROC_STOPWATCH diff --git a/primitives/primproc/pseudocc.cpp b/primitives/primproc/pseudocc.cpp index 7550c1b47..4fb3ba8d8 100644 --- a/primitives/primproc/pseudocc.cpp +++ b/primitives/primproc/pseudocc.cpp @@ -42,6 +42,7 @@ SCommand PseudoCC::duplicate() ret.reset(pseudo); pseudo->function = function; pseudo->valueFromUM = valueFromUM; + pseudo->bigValueFromUM = bigValueFromUM; ColumnCommand::duplicate(pseudo); return ret; } @@ -55,9 +56,21 @@ void PseudoCC::createCommand(messageqcpp::ByteStream& bs) void PseudoCC::resetCommand(messageqcpp::ByteStream& bs) { - if (function == PSEUDO_EXTENTMAX || function == PSEUDO_EXTENTMIN || function == PSEUDO_EXTENTID) + if (function == PSEUDO_EXTENTMAX || function == PSEUDO_EXTENTMIN) + { + if (!datatypes::Decimal::isWideDecimalType(colType)) + bs >> valueFromUM; + else + bs >> bigValueFromUM; + } + else if (function == PSEUDO_EXTENTID) + { bs >> valueFromUM; + if (datatypes::Decimal::isWideDecimalType(colType)) + bigValueFromUM = valueFromUM; + } + ColumnCommand::resetCommand(bs); } @@ -91,8 +104,11 @@ void PseudoCC::loadData() case 8: loadPMNumber(); break; + case 16: - cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + loadPMNumber(); + break; + default: cout << "PC::loadData(): bad column width" << endl; break; @@ -119,6 +135,10 @@ void PseudoCC::loadData() loadRIDs(); break; + case 16: + loadRIDs(); + break; + default: cout << "PC::loadData(): bad column width" << endl; break; @@ -144,8 +164,11 @@ void PseudoCC::loadData() case 8: loadSegmentNum(); break; + case 16: - cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + loadSegmentNum(); + break; + default: cout << "PC::loadData(): bad column width" << endl; break; @@ -173,7 +196,9 @@ void PseudoCC::loadData() break; case 16: - cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + loadPartitionNum(); + break; + default: cout << "PC::loadData(): bad column width" << endl; break; @@ -201,7 +226,9 @@ void PseudoCC::loadData() break; case 16: - cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + loadLBID(); + break; + default: cout << "PC::loadData(): bad column width" << endl; break; @@ -227,8 +254,10 @@ void PseudoCC::loadData() case 8: loadDBRootNum(); break; + case 16: - cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + loadDBRootNum(); + break; default: cout << "PC::loadData(): bad column width" << endl; @@ -258,8 +287,10 @@ void PseudoCC::loadData() case 8: loadSingleValue(valueFromUM); break; + case 16: - cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + loadSingleValue(bigValueFromUM); + break; default: cout << "PC::loadData(): bad column width" << endl; diff --git a/primitives/primproc/pseudocc.h b/primitives/primproc/pseudocc.h index 1a4d84c44..d59cc3360 100644 --- a/primitives/primproc/pseudocc.h +++ b/primitives/primproc/pseudocc.h @@ -49,6 +49,7 @@ private: uint32_t function; uint64_t valueFromUM; + uint128_t bigValueFromUM; }; diff --git a/tools/editem/editem.cpp b/tools/editem/editem.cpp index f2b392dad..c8cc31683 100644 --- a/tools/editem/editem.cpp +++ b/tools/editem/editem.cpp @@ -44,6 +44,9 @@ using namespace config; #include "dataconvert.h" using namespace dataconvert; +#include "widedecimalutils.h" +#include "mcs_decimal.h" + #include "liboamcpp.h" #undef REALLY_DANGEROUS @@ -164,40 +167,97 @@ const string charcolToString(int64_t v) //------------------------------------------------------------------------------ // Formats an integer to it's date, datetime, or char equivalent //------------------------------------------------------------------------------ -const string fmt(int64_t v) +template +const string fmt(T v) { ostringstream oss; if (tflg) { - oss << DataConvert::dateToString(v); + oss << DataConvert::dateToString((int64_t) v); } else if (sflg) { - oss << DataConvert::datetimeToString(v); + oss << DataConvert::datetimeToString((int64_t) v); } else if (aflg) { - oss << charcolToString(v); + oss << charcolToString((int64_t) v); } else if (mflg) { - oss << v; + if (typeid(T) != typeid(int128_t)) + { + oss << (int64_t) v; + } + else + { + char buf[utils::MAXLENGTH16BYTES]; + + int128_t tmp = v; + + dataconvert::DataConvert::decimalToString( + &tmp, 0, buf, sizeof(buf), execplan::CalpontSystemCatalog::DECIMAL); + + oss << buf; + } } else if (uflg) { - if (static_cast(v) > numeric_limits::max() - 2) - oss << "notset"; + if (typeid(T) != typeid(int128_t)) + { + if (static_cast(v) > numeric_limits::max() - 2) + oss << "notset"; + else + oss << static_cast(v); + } else - oss << static_cast(v); + { + if (v <= utils::minInt128 + 1) + { + oss << "notset"; + } + else + { + char buf[utils::MAXLENGTH16BYTES]; + + int128_t tmp = static_cast(v); + + dataconvert::DataConvert::decimalToString( + &tmp, 0, buf, sizeof(buf), execplan::CalpontSystemCatalog::DECIMAL); + + oss << buf; + } + } } else { - if (v == numeric_limits::max() || - v <= (numeric_limits::min() + 2)) - oss << "notset"; + if (typeid(T) != typeid(int128_t)) + { + if (v == numeric_limits::max() || + v <= (numeric_limits::min() + 1)) + oss << "notset"; + else + oss << (int64_t) v; + } else - oss << v; + { + if (v == utils::maxInt128 || (v <= utils::minInt128 + 1)) + { + oss << "notset"; + } + else + { + char buf[utils::MAXLENGTH16BYTES]; + + int128_t tmp = v; + + dataconvert::DataConvert::decimalToString( + &tmp, 0, buf, sizeof(buf), execplan::CalpontSystemCatalog::DECIMAL); + + oss << buf; + } + } } return oss.str(); @@ -234,6 +294,8 @@ int dumpone(OID_t oid, unsigned int sortOrder) std::vector::iterator end; int64_t max; int64_t min; + int128_t bigMax; + int128_t bigMin; int32_t seqNum; bool header; bool needtrailer = false; @@ -262,8 +324,6 @@ int dumpone(OID_t oid, unsigned int sortOrder) while (iter != end) { uint32_t lbidRangeSize = iter->range.size * 1024; - max = iter->partition.cprange.hi_val; - min = iter->partition.cprange.lo_val; seqNum = iter->partition.cprange.sequenceNum; int state = iter->partition.cprange.isValid; @@ -287,10 +347,26 @@ int dumpone(OID_t oid, unsigned int sortOrder) if (vflg) cout << oid << ' '; - cout << iter->range.start << " - " << - (iter->range.start + lbidRangeSize - 1) << - " (" << lbidRangeSize << ") min: " << fmt(min) << - ", max: " << fmt(max) << ", seqNum: " << seqNum << ", state: "; + if (iter->colWid != datatypes::MAXDECIMALWIDTH) + { + max = iter->partition.cprange.hi_val; + min = iter->partition.cprange.lo_val; + + cout << iter->range.start << " - " << + (iter->range.start + lbidRangeSize - 1) << + " (" << lbidRangeSize << ") min: " << fmt(min) << + ", max: " << fmt(max) << ", seqNum: " << seqNum << ", state: "; + } + else + { + bigMax = iter->partition.cprange.bigHiVal; + bigMin = iter->partition.cprange.bigLoVal; + + cout << iter->range.start << " - " << + (iter->range.start + lbidRangeSize - 1) << + " (" << lbidRangeSize << ") min: " << fmt(bigMin) << + ", max: " << fmt(bigMax) << ", seqNum: " << seqNum << ", state: "; + } switch (state) { @@ -415,7 +491,7 @@ int clearAllCPData() if (entries.empty()) continue; - bool isBinaryColumn = entries[0].colWid > 8; + bool isBinaryColumn = (entries[0].colWid == datatypes::MAXDECIMALWIDTH); BRM::CPInfo cpInfo; BRM::CPInfoList_t vCpInfo; @@ -427,8 +503,8 @@ int clearAllCPData() } else { - dataconvert::DataConvert::int128Min(cpInfo.bigMax); - dataconvert::DataConvert::int128Max(cpInfo.bigMin); + utils::int128Min(cpInfo.bigMax); + utils::int128Max(cpInfo.bigMin); } cpInfo.seqNum = -1; @@ -466,7 +542,7 @@ int clearmm(OID_t oid) return 1; } - bool isBinaryColumn = entries[0].colWid > 8; + bool isBinaryColumn = (entries[0].colWid == datatypes::MAXDECIMALWIDTH); // @bug 2280. Changed to use the batch interface to clear the CP info to make the clear option faster. BRM::CPInfo cpInfo; @@ -479,8 +555,8 @@ int clearmm(OID_t oid) } else { - dataconvert::DataConvert::int128Min(cpInfo.bigMax); - dataconvert::DataConvert::int128Max(cpInfo.bigMin); + utils::int128Min(cpInfo.bigMax); + utils::int128Max(cpInfo.bigMin); } cpInfo.seqNum = -1; diff --git a/utils/common/hasher.h b/utils/common/hasher.h index 4978689c4..27d7fbd01 100644 --- a/utils/common/hasher.h +++ b/utils/common/hasher.h @@ -30,6 +30,8 @@ #include #include +using int128_t = __int128; + namespace utils { /** @brief class Hasher @@ -346,6 +348,25 @@ public: } }; +// TODO a copy of these classes also exists in primitiveprocessor.h; consolidate +class Hash128 +{ +public: + inline size_t operator()(const int128_t i) const + { + return *reinterpret_cast(&i); + } +}; + +class Equal128 +{ +public: + inline bool operator()(const int128_t f1, const int128_t f2) const + { + return f1 == f2; + } +}; + //------------------------------------------------------------------------------ /** @brief class TupleHasher * diff --git a/utils/common/widedecimalutils.h b/utils/common/widedecimalutils.h index 2ed5b7309..6efa1b595 100644 --- a/utils/common/widedecimalutils.h +++ b/utils/common/widedecimalutils.h @@ -32,6 +32,9 @@ namespace utils const uint8_t MAXLENGTH16BYTES = 42; const uint8_t MAXLENGTH8BYTES = 23; + const int128_t minInt128 = int128_t(0x8000000000000000LL) << 64; + const int128_t maxInt128 = (int128_t(0x7FFFFFFFFFFFFFFFLL) << 64) + 0xFFFFFFFFFFFFFFFFLL; + inline bool isWideDecimalNullValue(const int128_t& val) { const uint64_t* ptr = reinterpret_cast(&val); diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 7c9f69278..2636919e0 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -161,21 +161,10 @@ const int32_t MIN_TIMESTAMP_VALUE = 0; namespace dataconvert { -// Decimal has maximum 38 digits with 3 extra chars for dot(.), minus(-), null character(\0) -const int MAX_DECIMAL_STRING_LENGTH = 41; - // WIP MCOL-641 using int128_t = __int128; using uint128_t = unsigned __int128; -struct Int128Pod_struct -{ - uint64_t lo; - uint64_t hi; -}; - -typedef Int128Pod_struct Int128Pod_t; - enum CalpontDateTimeFormat { CALPONTDATE_ENUM = 1, // date format is: "YYYY-MM-DD" @@ -1067,27 +1056,6 @@ public: static size_t writeFractionalPart(int128_t* dec, char* p, const unsigned int buflen, const uint8_t scale); - static inline void int128Max(int128_t& i) - { - Int128Pod_t *pod = reinterpret_cast(&i); - pod->lo = 0xFFFFFFFFFFFFFFFF; - pod->hi = 0x7FFFFFFFFFFFFFFF; - } - - static inline void int128Min(int128_t& i) - { - Int128Pod_t *pod = reinterpret_cast(&i); - pod->lo = 0; - pod->hi = 0x8000000000000000; - } - - static inline void uint128Max(uint128_t& i) - { - Int128Pod_t *pod = reinterpret_cast(&i); - pod->lo = 0xFFFFFFFFFFFFFFFF; - pod->hi = 0xFFFFFFFFFFFFFFFF; - } - static inline std::string constructRegexp(const std::string& str); static inline void trimWhitespace(int64_t& charData); static inline bool isEscapedChar(char c) diff --git a/utils/joiner/tuplejoiner.cpp b/utils/joiner/tuplejoiner.cpp index a8e853553..ee405dd89 100644 --- a/utils/joiner/tuplejoiner.cpp +++ b/utils/joiner/tuplejoiner.cpp @@ -20,16 +20,19 @@ #include #include #include -#ifdef _MSC_VER -#include -#else +#ifndef _MSC_VER #include +#else +#include #endif + #include "hasher.h" #include "lbidlist.h" #include "spinlock.h" #include "vlarray.h" +#include "widedecimalutils.h" + using namespace std; using namespace rowgroup; using namespace utils; @@ -102,18 +105,38 @@ TupleJoiner::TupleJoiner( smallKeyColumns.push_back(smallJoinColumn); largeKeyColumns.push_back(largeJoinColumn); discreteValues.reset(new bool[1]); - cpValues.reset(new vector[1]); + cpValues.reset(new vector[1]); discreteValues[0] = false; if (smallRG.isUnsigned(smallKeyColumns[0])) { - cpValues[0].push_back(numeric_limits::max()); - cpValues[0].push_back(0); + if (datatypes::Decimal::isWideDecimalType( + smallRG.getColType(smallKeyColumns[0]), + smallRG.getColumnWidth(smallKeyColumns[0]))) + { + cpValues[0].push_back((int128_t) -1); + cpValues[0].push_back(0); + } + else + { + cpValues[0].push_back((int128_t) numeric_limits::max()); + cpValues[0].push_back(0); + } } else { - cpValues[0].push_back(numeric_limits::max()); - cpValues[0].push_back(numeric_limits::min()); + if (datatypes::Decimal::isWideDecimalType( + smallRG.getColType(smallKeyColumns[0]), + smallRG.getColumnWidth(smallKeyColumns[0]))) + { + cpValues[0].push_back(utils::maxInt128); + cpValues[0].push_back(utils::minInt128); + } + else + { + cpValues[0].push_back((int128_t) numeric_limits::max()); + cpValues[0].push_back((int128_t) numeric_limits::min()); + } } if (smallRG.isUnsigned(smallJoinColumn) != largeRG.isUnsigned(largeJoinColumn)) @@ -195,20 +218,40 @@ TupleJoiner::TupleJoiner( storedKeyAlloc[i].setAllocSize(keyLength); discreteValues.reset(new bool[smallKeyColumns.size()]); - cpValues.reset(new vector[smallKeyColumns.size()]); + cpValues.reset(new vector[smallKeyColumns.size()]); for (i = 0; i < smallKeyColumns.size(); i++) { discreteValues[i] = false; if (isUnsigned(smallRG.getColTypes()[smallKeyColumns[i]])) { - cpValues[i].push_back(static_cast(numeric_limits::max())); - cpValues[i].push_back(0); + if (datatypes::Decimal::isWideDecimalType( + smallRG.getColType(smallKeyColumns[i]), + smallRG.getColumnWidth(smallKeyColumns[i]))) + { + cpValues[i].push_back((int128_t) -1); + cpValues[i].push_back(0); + } + else + { + cpValues[i].push_back((int128_t) numeric_limits::max()); + cpValues[i].push_back(0); + } } else { - cpValues[i].push_back(numeric_limits::max()); - cpValues[i].push_back(numeric_limits::min()); + if (datatypes::Decimal::isWideDecimalType( + smallRG.getColType(smallKeyColumns[i]), + smallRG.getColumnWidth(smallKeyColumns[i]))) + { + cpValues[i].push_back(utils::maxInt128); + cpValues[i].push_back(utils::minInt128); + } + else + { + cpValues[i].push_back(numeric_limits::max()); + cpValues[i].push_back(numeric_limits::min()); + } } } } @@ -678,8 +721,9 @@ void TupleJoiner::doneInserting() for (col = 0; col < smallKeyColumns.size(); col++) { - tr1::unordered_set uniquer; - tr1::unordered_set::iterator uit; + typedef std::tr1::unordered_set unordered_set_int128; + unordered_set_int128 uniquer; + unordered_set_int128::iterator uit; sthash_t::iterator sthit; hash_t::iterator hit; ldhash_t::iterator ldit; @@ -758,6 +802,12 @@ void TupleJoiner::doneInserting() } } } + else if (datatypes::Decimal::isWideDecimalType( + smallRow.getColType(smallKeyColumns[col]), + smallRow.getColumnWidth(smallKeyColumns[col]))) + { + uniquer.insert(*((int128_t*)smallRow.getBinaryField(smallKeyColumns[col]))); + } else if (smallRow.isUnsigned(smallKeyColumns[col])) { uniquer.insert((int64_t)smallRow.getUintField(smallKeyColumns[col])); @@ -1080,21 +1130,22 @@ void TupleJoiner::updateCPData(const Row& r) { int64_t val = r.getIntField(colIdx); - if (order_swap(val) < order_swap(min) || - min == numeric_limits::max()) + if (order_swap(val) < order_swap((int64_t) min) || + ((int64_t) min) == numeric_limits::max()) { min = val; } - if (order_swap(val) > order_swap(max) || - max == numeric_limits::min()) + if (order_swap(val) > order_swap((int64_t) max) || + ((int64_t) max) == numeric_limits::min()) { max = val; } } else if (r.isUnsigned(colIdx)) { - uint64_t uval; + uint128_t uval; + if (r.getColType(colIdx) == CalpontSystemCatalog::LONGDOUBLE) { double dval = (double)roundl(r.getLongDoubleField(smallKeyColumns[col])); @@ -1114,20 +1165,27 @@ void TupleJoiner::updateCPData(const Row& r) } } } + else if (datatypes::Decimal::isWideDecimalType( + r.getColType(colIdx), + r.getColumnWidth(colIdx))) + { + uval = *((int128_t*)r.getBinaryField(colIdx)); + } else { uval = r.getUintField(colIdx); } - if (uval > static_cast(max)) - max = static_cast(uval); + if (uval > static_cast(max)) + max = static_cast(uval); - if (uval < static_cast(min)) - min = static_cast(uval); + if (uval < static_cast(min)) + min = static_cast(uval); } else { - int64_t val = 0; + int128_t val = 0; + if (r.getColType(colIdx) == CalpontSystemCatalog::LONGDOUBLE) { double dval = (double)roundl(r.getLongDoubleField(colIdx)); @@ -1147,13 +1205,12 @@ void TupleJoiner::updateCPData(const Row& r) } } } - else if (r.getColumnWidth(colIdx) == datatypes::MAXDECIMALWIDTH - && (r.getColType(colIdx) == CalpontSystemCatalog::DECIMAL - || r.getColType(colIdx) == CalpontSystemCatalog::UDECIMAL)) + else if (datatypes::Decimal::isWideDecimalType( + r.getColType(colIdx), + r.getColumnWidth(colIdx))) { - // WIP MCOL-641 + val = *((int128_t*)r.getBinaryField(colIdx)); } - else { val = r.getIntField(colIdx); @@ -1681,20 +1738,40 @@ boost::shared_ptr TupleJoiner::copyForDiskJoin() ret->uniqueLimit = uniqueLimit; ret->discreteValues.reset(new bool[smallKeyColumns.size()]); - ret->cpValues.reset(new vector[smallKeyColumns.size()]); + ret->cpValues.reset(new vector[smallKeyColumns.size()]); for (uint32_t i = 0; i < smallKeyColumns.size(); i++) { ret->discreteValues[i] = false; if (isUnsigned(smallRG.getColTypes()[smallKeyColumns[i]])) { - ret->cpValues[i].push_back(static_cast(numeric_limits::max())); - ret->cpValues[i].push_back(0); + if (datatypes::Decimal::isWideDecimalType( + smallRG.getColType(smallKeyColumns[i]), + smallRG.getColumnWidth(smallKeyColumns[i]))) + { + ret->cpValues[i].push_back((int128_t) -1); + ret->cpValues[i].push_back(0); + } + else + { + ret->cpValues[i].push_back((int128_t) numeric_limits::max()); + ret->cpValues[i].push_back(0); + } } else { - ret->cpValues[i].push_back(numeric_limits::max()); - ret->cpValues[i].push_back(numeric_limits::min()); + if (datatypes::Decimal::isWideDecimalType( + smallRG.getColType(smallKeyColumns[i]), + smallRG.getColumnWidth(smallKeyColumns[i]))) + { + ret->cpValues[i].push_back(utils::maxInt128); + ret->cpValues[i].push_back(utils::minInt128); + } + else + { + ret->cpValues[i].push_back(numeric_limits::max()); + ret->cpValues[i].push_back(numeric_limits::min()); + } } } diff --git a/utils/joiner/tuplejoiner.h b/utils/joiner/tuplejoiner.h index d91297845..c1be13eac 100644 --- a/utils/joiner/tuplejoiner.h +++ b/utils/joiner/tuplejoiner.h @@ -287,7 +287,7 @@ public: { return discreteValues; } - inline const boost::scoped_array >& getCPData() + inline const boost::scoped_array >& getCPData() { return cpValues; } @@ -413,7 +413,7 @@ private: /* Runtime casual partitioning support */ void updateCPData(const rowgroup::Row& r); boost::scoped_array discreteValues; - boost::scoped_array > cpValues; // if !discreteValues, [0] has min, [1] has max + boost::scoped_array > cpValues; // if !discreteValues, [0] has min, [1] has max uint32_t uniqueLimit; bool finished; diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 22d47e16e..c6a1daed8 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -66,6 +66,7 @@ typedef const struct charset_info_st CHARSET_INFO; // Workaround for my_global.h #define of isnan(X) causing a std::std namespace using int128_t = __int128; +using uint128_t = unsigned __int128; namespace rowgroup { diff --git a/versioning/BRM/blockresolutionmanager.cpp b/versioning/BRM/blockresolutionmanager.cpp index 7127f3f43..6004a3832 100644 --- a/versioning/BRM/blockresolutionmanager.cpp +++ b/versioning/BRM/blockresolutionmanager.cpp @@ -107,10 +107,10 @@ int BlockResolutionManager::saveState(string filename) throw() saveExtentMap(emFilename); // truncate teh file if already exists since no truncate in HDFS. - const char* filename = journalFilename.c_str(); + const char* filename_p = journalFilename.c_str(); IDBDataFile* journal = IDBDataFile::open( - IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "wb", 0); + IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), filename_p, "wb", 0); delete journal; vbbm.save(vbbmFilename); diff --git a/versioning/BRM/dbrm.cpp b/versioning/BRM/dbrm.cpp index 028efad55..bd27c3aec 100644 --- a/versioning/BRM/dbrm.cpp +++ b/versioning/BRM/dbrm.cpp @@ -31,6 +31,7 @@ #include #include "dataconvert.h" +#include "widedecimalutils.h" #include "oamcache.h" #include "rwlock.h" #include "mastersegmenttable.h" @@ -4560,8 +4561,8 @@ void DBRM::invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN } else { - dataconvert::DataConvert::int128Min(aInfo.bigMax); - dataconvert::DataConvert::int128Max(aInfo.bigMin); + utils::int128Min(aInfo.bigMax); + utils::int128Max(aInfo.bigMin); } } } diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index d549da71f..70d12a383 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -55,6 +55,7 @@ namespace bi = boost::interprocess; #include "mastersegmenttable.h" #include "blocksize.h" #include "dataconvert.h" +#include "widedecimalutils.h" #include "oamcache.h" #include "IDBDataFile.h" #include "IDBPolicy.h" @@ -118,8 +119,8 @@ EMCasualPartition_struct::EMCasualPartition_struct() { lo_val = numeric_limits::max(); hi_val = numeric_limits::min(); - dataconvert::DataConvert::int128Max(bigLoVal); - dataconvert::DataConvert::int128Min(bigHiVal); + utils::int128Max(bigLoVal); + utils::int128Min(bigHiVal); sequenceNum = 0; isValid = CP_INVALID; } @@ -360,8 +361,8 @@ int ExtentMap::_markInvalid(const LBID_t lbid, const execplan::CalpontSystemCata { fExtentMap[i].partition.cprange.lo_val = numeric_limits::max(); fExtentMap[i].partition.cprange.hi_val = numeric_limits::min(); - dataconvert::DataConvert::int128Max(fExtentMap[i].partition.cprange.bigLoVal); - dataconvert::DataConvert::int128Min(fExtentMap[i].partition.cprange.bigHiVal); + utils::int128Max(fExtentMap[i].partition.cprange.bigLoVal); + utils::int128Min(fExtentMap[i].partition.cprange.bigHiVal); } incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); @@ -1048,7 +1049,7 @@ bool ExtentMap::isValidCPRange(const T& max, const T& min, execplan::CalpontSyst else { unsigned __int128 temp; - dataconvert::DataConvert::uint128Max(temp); + utils::uint128Max(temp); if ( (static_cast(min) >= (temp - 1)) || (static_cast(max) >= (temp - 1)) ) @@ -1070,7 +1071,7 @@ bool ExtentMap::isValidCPRange(const T& max, const T& min, execplan::CalpontSyst else { __int128 temp; - dataconvert::DataConvert::int128Min(temp); + utils::int128Min(temp); if ( (min <= (temp + 1)) || (max <= (temp + 1)) ) @@ -1112,8 +1113,8 @@ int ExtentMap::getMaxMin(const LBID_t lbid, if (typeid(T) == typeid(__int128)) { __int128 tmpMax, tmpMin; - dataconvert::DataConvert::int128Min(tmpMax); - dataconvert::DataConvert::int128Max(tmpMin); + utils::int128Min(tmpMax); + utils::int128Max(tmpMin); max = tmpMax; min = tmpMin; } @@ -2730,8 +2731,8 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, } else { - dataconvert::DataConvert::int128Max(e->partition.cprange.bigLoVal); - dataconvert::DataConvert::int128Min(e->partition.cprange.bigHiVal); + utils::int128Max(e->partition.cprange.bigLoVal); + utils::int128Min(e->partition.cprange.bigHiVal); } } @@ -2941,8 +2942,8 @@ LBID_t ExtentMap::_createColumnExtentExactFile(uint32_t size, int OID, } else { - dataconvert::DataConvert::int128Max(e->partition.cprange.bigLoVal); - dataconvert::DataConvert::int128Min(e->partition.cprange.bigHiVal); + utils::int128Max(e->partition.cprange.bigLoVal); + utils::int128Min(e->partition.cprange.bigHiVal); } } @@ -3128,8 +3129,8 @@ LBID_t ExtentMap::_createDictStoreExtent(uint32_t size, int OID, e->status = EXTENTUNAVAILABLE;// @bug 1911 mark extent as in process e->partition.cprange.lo_val = numeric_limits::max(); e->partition.cprange.hi_val = numeric_limits::min(); - dataconvert::DataConvert::int128Max(e->partition.cprange.bigLoVal); - dataconvert::DataConvert::int128Min(e->partition.cprange.bigHiVal); + utils::int128Max(e->partition.cprange.bigLoVal); + utils::int128Min(e->partition.cprange.bigHiVal); e->partition.cprange.sequenceNum = 0; e->partition.cprange.isValid = CP_INVALID; diff --git a/writeengine/bulk/we_brmreporter.cpp b/writeengine/bulk/we_brmreporter.cpp index 0787700be..f86490bec 100644 --- a/writeengine/bulk/we_brmreporter.cpp +++ b/writeengine/bulk/we_brmreporter.cpp @@ -34,6 +34,9 @@ #include "we_log.h" #include "cacheutils.h" #include "IDBPolicy.h" +#include "widedecimalutils.h" +#include "mcs_decimal.h" +#include "dataconvert.h" namespace WriteEngine { @@ -294,11 +297,11 @@ void BRMReporter::sendHWMToFile( ) //------------------------------------------------------------------------------ // Send Casual Partition update information to BRM //------------------------------------------------------------------------------ -// TODO MCOL-641 void BRMReporter::sendCPToFile( ) { if (fCPInfo.size() > 0) { + char buf[utils::MAXLENGTH16BYTES]; std::ostringstream oss; oss << "Writing " << fCPInfo.size() << " CP updates for table " << fTableName << " to report file " << fRptFileName; @@ -306,12 +309,32 @@ void BRMReporter::sendCPToFile( ) for (unsigned int i = 0; i < fCPInfo.size(); i++) { - fRptFile << "CP: " << fCPInfo[i].startLbid << ' ' << - fCPInfo[i].max << ' ' << - fCPInfo[i].min << ' ' << - fCPInfo[i].seqNum << ' ' << - fCPInfo[i].type << ' ' << - fCPInfo[i].newExtent << std::endl; + if (!datatypes::Decimal::isWideDecimalType(fCPInfo[i].type, fCPInfo[i].colWidth)) + { + fRptFile << "CP: " << fCPInfo[i].startLbid << ' ' << + fCPInfo[i].max << ' ' << + fCPInfo[i].min << ' ' << + fCPInfo[i].seqNum << ' ' << + fCPInfo[i].type << ' ' << + fCPInfo[i].newExtent << std::endl; + } + else + { + std::string bigMin, bigMax; + + dataconvert::DataConvert::decimalToString(&fCPInfo[i].bigMin, 0, buf, utils::MAXLENGTH16BYTES, fCPInfo[i].type); + bigMin = buf; + + dataconvert::DataConvert::decimalToString(&fCPInfo[i].bigMax, 0, buf, utils::MAXLENGTH16BYTES, fCPInfo[i].type); + bigMax = buf; + + fRptFile << "CP: " << fCPInfo[i].startLbid << ' ' << + bigMax << ' ' << + bigMin << ' ' << + fCPInfo[i].seqNum << ' ' << + fCPInfo[i].type << ' ' << + fCPInfo[i].newExtent << std::endl; + } } } } diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index 79d6cf275..18e6bae00 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -40,6 +40,7 @@ #include "dataconvert.h" #include "exceptclasses.h" #include "mcs_decimal.h" +#include "widedecimalutils.h" #include "joblisttypes.h" @@ -1735,8 +1736,8 @@ int BulkLoadBuffer::parseCol(ColumnInfo& columnInfo) } else { - dataconvert::DataConvert::int128Max(bufStats.bigMinBufferVal); - dataconvert::DataConvert::int128Min(bufStats.bigMaxBufferVal); + utils::int128Max(bufStats.bigMinBufferVal); + utils::int128Min(bufStats.bigMaxBufferVal); } updateCPInfoPendingFlag = false; } diff --git a/writeengine/bulk/we_bulkloadbuffer.h b/writeengine/bulk/we_bulkloadbuffer.h index 3932f39aa..bbdb2378d 100644 --- a/writeengine/bulk/we_bulkloadbuffer.h +++ b/writeengine/bulk/we_bulkloadbuffer.h @@ -31,6 +31,7 @@ #include "we_columninfo.h" #include "calpontsystemcatalog.h" #include "dataconvert.h" +#include "widedecimalutils.h" namespace WriteEngine { @@ -66,8 +67,8 @@ public: { minBufferVal = MAX_BIGINT; maxBufferVal = MIN_BIGINT; - dataconvert::DataConvert::int128Max(bigMinBufferVal); - dataconvert::DataConvert::int128Min(bigMaxBufferVal); + utils::int128Max(bigMinBufferVal); + utils::int128Min(bigMaxBufferVal); } } }; diff --git a/writeengine/bulk/we_colextinf.cpp b/writeengine/bulk/we_colextinf.cpp index a020c7362..f94433689 100644 --- a/writeengine/bulk/we_colextinf.cpp +++ b/writeengine/bulk/we_colextinf.cpp @@ -27,6 +27,7 @@ #include "we_colextinf.h" #include "dataconvert.h" +#include "widedecimalutils.h" #include #include @@ -93,7 +94,7 @@ void ColExtInf::addOrUpdateEntryTemplate( RID lastInputRow, // MAX_INT and maxVal will be MIN_INT (see getCPInfoForBRM()). __int128 bigMinValInit; - dataconvert::DataConvert::int128Max(bigMinValInit); + utils::int128Max(bigMinValInit); if ((iter->second.fMinVal == LLONG_MIN && width <= 8) || (iter->second.fbigMinVal == bigMinValInit && width > 8)) // init the range { diff --git a/writeengine/bulk/we_colextinf.h b/writeengine/bulk/we_colextinf.h index e7215c5c4..e0e516616 100644 --- a/writeengine/bulk/we_colextinf.h +++ b/writeengine/bulk/we_colextinf.h @@ -41,6 +41,7 @@ #include "brmtypes.h" #include "we_type.h" #include "dataconvert.h" +#include "widedecimalutils.h" namespace WriteEngine { @@ -65,8 +66,8 @@ public: fMaxVal(LLONG_MIN), fNewExtent(true) { - dataconvert::DataConvert::int128Min(fbigMaxVal); - dataconvert::DataConvert::int128Max(fbigMinVal); + utils::int128Min(fbigMaxVal); + utils::int128Max(fbigMinVal); } // Used to create entry for an existing extent we are going to add data to. @@ -76,8 +77,8 @@ public: fMaxVal(LLONG_MIN), fNewExtent(bIsNewExtent) { - dataconvert::DataConvert::int128Min(fbigMaxVal); - dataconvert::DataConvert::int128Max(fbigMinVal); + utils::int128Min(fbigMaxVal); + utils::int128Max(fbigMinVal); } // Used to create entry for a new extent, with LBID not yet allocated diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index a997f0999..7eda7fc7c 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -40,6 +40,7 @@ using namespace std; using namespace execplan; #include "dataconvert.h" +#include "widedecimalutils.h" #include "IDBDataFile.h" #include "IDBPolicy.h" @@ -1140,8 +1141,8 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi } else { - dataconvert::DataConvert::int128Min(cpInfo.bigMax); - dataconvert::DataConvert::int128Max(cpInfo.bigMin); + utils::int128Min(cpInfo.bigMax); + utils::int128Max(cpInfo.bigMin); } } @@ -1193,8 +1194,8 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi } else { - dataconvert::DataConvert::int128Min(cpInfo1.bigMax); - dataconvert::DataConvert::int128Max(cpInfo1.bigMin); + utils::int128Min(cpInfo1.bigMax); + utils::int128Max(cpInfo1.bigMin); } } diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index bb7489a48..24407d513 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -1170,8 +1170,8 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid, } else { - dataconvert::DataConvert::int128Min(cpInfo.bigMax); - dataconvert::DataConvert::int128Max(cpInfo.bigMin); + utils::int128Min(cpInfo.bigMax); + utils::int128Max(cpInfo.bigMin); } } @@ -1916,8 +1916,8 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid, } else { - dataconvert::DataConvert::int128Min(cpInfo.bigMax); - dataconvert::DataConvert::int128Max(cpInfo.bigMin); + utils::int128Min(cpInfo.bigMax); + utils::int128Max(cpInfo.bigMin); } } From 5287e6860b7b866c06f2f4b3f50a08abbf0c9944 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Mon, 27 Jul 2020 11:02:59 -0400 Subject: [PATCH 50/78] MCOL-641 Upgrade code for extent map from v4 to v5. v5 (current) version of the extent map supports int128_t min/max values. This commit provides upgrade capability to load_brm program that when reads an extent map saved to disk earlier with save_brm, with a EM_MAGIC_V4 entry, converts it to the current v5 "wider" extent map. --- versioning/BRM/extentmap.cpp | 82 +++++++++++++++++++++++++++++------- versioning/BRM/extentmap.h | 52 +++++++++++++++++++---- 2 files changed, 111 insertions(+), 23 deletions(-) diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index 70d12a383..e5f84ea53 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -74,6 +74,7 @@ namespace bi = boost::interprocess; #define EM_MAGIC_V2 0x76f78b1d #define EM_MAGIC_V3 0x76f78b1e #define EM_MAGIC_V4 0x76f78b1f +#define EM_MAGIC_V5 0x76f78b20 #ifndef NDEBUG #define ASSERT(x) \ @@ -163,7 +164,7 @@ EMCasualPartition_struct& EMCasualPartition_struct::operator= (const EMCasualPar } //------------------------------------------------------------------------------ -// Version 4 EmEntry methods +// Version 5 EmEntry methods //------------------------------------------------------------------------------ EMEntry::EMEntry() @@ -1276,7 +1277,7 @@ void ExtentMap::reserveLBIDRange(LBID_t start, uint8_t size) */ -void ExtentMap::loadVersion4(IDBDataFile* in) +void ExtentMap::loadVersion4or5(IDBDataFile* in, bool upgradeV4ToV5) { int emNumElements = 0, flNumElements = 0; @@ -1287,8 +1288,8 @@ void ExtentMap::loadVersion4(IDBDataFile* in) if ((size_t) nbytes != sizeof(int) + sizeof(int)) { - log_errno("ExtentMap::loadVersion4(): read "); - throw runtime_error("ExtentMap::loadVersion4(): read failed. Check the error log."); + log_errno("ExtentMap::loadVersion4or5(): read "); + throw runtime_error("ExtentMap::loadVersion4or5(): read failed. Check the error log."); } void *fExtentMapPtr = static_cast(fExtentMap); @@ -1317,18 +1318,66 @@ void ExtentMap::loadVersion4(IDBDataFile* in) growEMShmseg(nrows); } - size_t progress = 0, writeSize = emNumElements * sizeof(EMEntry); int err; - char *writePos = (char *) fExtentMap; - while (progress < writeSize) + char* writePos; + size_t progress, writeSize; + + if (!upgradeV4ToV5) { - err = in->read(writePos + progress, writeSize - progress); - if (err <= 0) + progress = 0; + writeSize = emNumElements * sizeof(EMEntry); + writePos = (char *) fExtentMap; + + while (progress < writeSize) { - log_errno("ExtentMap::loadVersion4(): read "); - throw runtime_error("ExtentMap::loadVersion4(): read failed. Check the error log."); + err = in->read(writePos + progress, writeSize - progress); + if (err <= 0) + { + log_errno("ExtentMap::loadVersion4or5(): read "); + throw runtime_error("ExtentMap::loadVersion4or5(): read failed. Check the error log."); + } + progress += (uint) err; } - progress += (uint) err; + } + else + { + // We are upgrading extent map from v4 to v5. + for (int i = 0; i < emNumElements; i++) + { + EMEntry_v4 emEntryV4; + progress = 0; + writeSize = sizeof(EMEntry_v4); + writePos = (char *) &(emEntryV4); + while (progress < writeSize) + { + err = in->read(writePos + progress, writeSize - progress); + if (err <= 0) + { + log_errno("ExtentMap::loadVersion4or5(): read "); + throw runtime_error("ExtentMap::loadVersion4or5(): read failed during upgrade. Check the error log."); + } + progress += (uint) err; + } + + fExtentMap[i].range.start = emEntryV4.range.start; + fExtentMap[i].range.size = emEntryV4.range.size; + fExtentMap[i].fileID = emEntryV4.fileID; + fExtentMap[i].blockOffset = emEntryV4.blockOffset; + fExtentMap[i].HWM = emEntryV4.HWM; + fExtentMap[i].partitionNum = emEntryV4.partitionNum; + fExtentMap[i].segmentNum = emEntryV4.segmentNum; + fExtentMap[i].dbRoot = emEntryV4.dbRoot; + fExtentMap[i].colWid = emEntryV4.colWid; + fExtentMap[i].status = emEntryV4.status; + fExtentMap[i].partition.cprange.hi_val = emEntryV4.partition.cprange.hi_val; + fExtentMap[i].partition.cprange.lo_val = emEntryV4.partition.cprange.lo_val; + fExtentMap[i].partition.cprange.sequenceNum = emEntryV4.partition.cprange.sequenceNum; + fExtentMap[i].partition.cprange.isValid = emEntryV4.partition.cprange.isValid; + utils::int128Max(fExtentMap[i].partition.cprange.bigLoVal); + utils::int128Min(fExtentMap[i].partition.cprange.bigHiVal); + } + + std::cout<read((char*) &emVersion, sizeof(int)); - if (bytes == (int) sizeof(int) && emVersion == EM_MAGIC_V4) - loadVersion4(in.get()); + if (bytes == (int) sizeof(int) && + (emVersion == EM_MAGIC_V4 || emVersion == EM_MAGIC_V5)) + { + loadVersion4or5(in.get(), emVersion == EM_MAGIC_V4); + } else { log("ExtentMap::load(): That file is not a valid ExtentMap image"); @@ -1486,7 +1538,7 @@ void ExtentMap::save(const string& filename) throw ios_base::failure("ExtentMap::save(): open failed. Check the error log."); } - loadSize[0] = EM_MAGIC_V4; + loadSize[0] = EM_MAGIC_V5; loadSize[1] = fEMShminfo->currentSize / sizeof(EMEntry); loadSize[2] = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); // needs to send all entries diff --git a/versioning/BRM/extentmap.h b/versioning/BRM/extentmap.h index e3d4634d8..3aa0da0ca 100644 --- a/versioning/BRM/extentmap.h +++ b/versioning/BRM/extentmap.h @@ -100,17 +100,43 @@ const char CP_INVALID = 0; const char CP_UPDATING = 1; const char CP_VALID = 2; +// The _v4 structs are defined below for upgrading extent map +// from v4 to v5; see ExtentMap::loadVersion4or5 for details. +struct EMCasualPartition_struct_v4 +{ + RangePartitionData_t hi_val; // This needs to be reinterpreted as unsigned for uint64_t column types. + RangePartitionData_t lo_val; + int32_t sequenceNum; + char isValid; //CP_INVALID - No min/max and no DML in progress. CP_UPDATING - Update in progress. CP_VALID- min/max is valid +}; + +struct EMPartition_struct_v4 +{ + EMCasualPartition_struct_v4 cprange; +}; + +struct EMEntry_v4 +{ + InlineLBIDRange range; + int fileID; + uint32_t blockOffset; + HWM_t HWM; + uint32_t partitionNum; // starts at 0 + uint16_t segmentNum; // starts at 0 + uint16_t dbRoot; // starts at 1 to match Columnstore.xml + uint16_t colWid; + int16_t status; //extent avail for query or not, or out of service + EMPartition_struct_v4 partition; +}; + +// MCOL-641: v5 structs of the extent map. This version supports int128_t min +// and max values for casual partitioning. struct EMCasualPartition_struct { RangePartitionData_t hi_val; // This needs to be reinterpreted as unsigned for uint64_t column types. RangePartitionData_t lo_val; int32_t sequenceNum; char isValid; //CP_INVALID - No min/max and no DML in progress. CP_UPDATING - Update in progress. CP_VALID- min/max is valid - EXPORT EMCasualPartition_struct(); - EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum); - EXPORT EMCasualPartition_struct(const __int128 bigLo, const __int128 bigHi, const int32_t seqNum); - EXPORT EMCasualPartition_struct(const EMCasualPartition_struct& em); - EXPORT EMCasualPartition_struct& operator= (const EMCasualPartition_struct& em); union { __int128 bigLoVal; @@ -121,12 +147,17 @@ struct EMCasualPartition_struct __int128 bigHiVal; int64_t hiVal; }; + EXPORT EMCasualPartition_struct(); + EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum); + EXPORT EMCasualPartition_struct(const __int128 bigLo, const __int128 bigHi, const int32_t seqNum); + EXPORT EMCasualPartition_struct(const EMCasualPartition_struct& em); + EXPORT EMCasualPartition_struct& operator= (const EMCasualPartition_struct& em); }; typedef EMCasualPartition_struct EMCasualPartition_t; struct EMPartition_struct { - EMCasualPartition_t cprange; + EMCasualPartition_t cprange; }; typedef EMPartition_struct EMPartition_t; @@ -973,8 +1004,13 @@ private: int _markInvalid(const LBID_t lbid, const execplan::CalpontSystemCatalog::ColDataType colDataType); - void loadVersion4(std::ifstream& in); - void loadVersion4(idbdatafile::IDBDataFile* in); + /** @brief Loads the extent map from a file into memory. + * + * @param in (in) the file to load the extent map from. + * @param upgradeV4ToV5 (in) flag indicating whether we are upgrading + * extent map from v4 to v5. + */ + void loadVersion4or5(idbdatafile::IDBDataFile* in, bool upgradeV4ToV5); ExtentMapImpl* fPExtMapImpl; FreeListImpl* fPFreeListImpl; From 638202417f7cc748b2f970d9f97e23cc36ab973b Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 30 Jul 2020 17:28:11 -0500 Subject: [PATCH 51/78] MCOL-4171 --- dbcon/execplan/arithmeticoperator.h | 2 +- dbcon/execplan/simplecolumn.cpp | 2 +- dbcon/execplan/windowfunctioncolumn.cpp | 36 ++- dbcon/joblist/groupconcat.cpp | 4 +- dbcon/mysql/ha_mcs_impl.cpp | 2 +- utils/messageqcpp/bytestream.cpp | 27 +++ utils/messageqcpp/bytestream.h | 13 + utils/regr/moda.cpp | 51 ++-- utils/regr/moda.h | 52 +++- utils/rowgroup/rowaggregation.cpp | 10 +- utils/rowgroup/rowgroup.h | 29 ++- utils/udfsdk/mcsv1_udaf.cpp | 2 + utils/udfsdk/mcsv1_udaf.h | 10 + utils/windowfunction/frameboundrange.cpp | 47 +++- utils/windowfunction/idborderby.cpp | 19 +- utils/windowfunction/wf_count.cpp | 22 +- utils/windowfunction/wf_count.h | 2 +- utils/windowfunction/wf_lead_lag.cpp | 34 ++- utils/windowfunction/wf_lead_lag.h | 2 +- utils/windowfunction/wf_min_max.cpp | 32 ++- utils/windowfunction/wf_min_max.h | 2 +- utils/windowfunction/wf_nth_value.cpp | 32 ++- utils/windowfunction/wf_nth_value.h | 2 +- utils/windowfunction/wf_ntile.cpp | 2 +- utils/windowfunction/wf_ntile.h | 2 +- utils/windowfunction/wf_percentile.cpp | 32 ++- utils/windowfunction/wf_percentile.h | 2 +- utils/windowfunction/wf_ranking.cpp | 2 +- utils/windowfunction/wf_ranking.h | 2 +- utils/windowfunction/wf_row_number.cpp | 2 +- utils/windowfunction/wf_row_number.h | 2 +- utils/windowfunction/wf_stats.cpp | 42 +++- utils/windowfunction/wf_stats.h | 2 +- utils/windowfunction/wf_sum_avg.cpp | 251 +++++++++++--------- utils/windowfunction/wf_sum_avg.h | 29 ++- utils/windowfunction/wf_udaf.cpp | 97 ++++++-- utils/windowfunction/wf_udaf.h | 2 +- utils/windowfunction/windowfunctiontype.cpp | 133 +++++++++-- utils/windowfunction/windowfunctiontype.h | 8 + writeengine/wrapper/writeengine.cpp | 13 +- 40 files changed, 807 insertions(+), 250 deletions(-) diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index 5286ab25c..416daeea4 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -196,7 +196,7 @@ public: return TreeNode::getBoolVal(); } void adjustResultType(const CalpontSystemCatalog::ColType& m); - constexpr inline bool getOverflowCheck() const + const inline bool getOverflowCheck() const { return fDecimalOverflowCheck; } diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index 09328ae9f..7201a366c 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -503,7 +503,7 @@ bool SimpleColumn::singleTable(CalpontSystemCatalog::TableAliasName& tan) void SimpleColumn::evaluate(Row& row, bool& isNull) { // WIP Move this block into an appropriate place - if (UNLIKELY(fInputOffset == -1)) + if (UNLIKELY((int)(fInputOffset == (uint32_t)-1))) { fInputOffset = row.getOffset(fInputIndex); } diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index f14d9656f..d32e41651 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -52,6 +52,8 @@ using namespace rowgroup; #include "joblisttypes.h" using namespace joblist; +#include "widedecimalutils.h" + #ifdef _MSC_VER #define strcasecmp stricmp #endif @@ -387,9 +389,16 @@ void WindowFunctionColumn::adjustResultType() boost::iequals(fFunctionName, "AVG") || boost::iequals(fFunctionName, "AVG_DISTINCT")) { - fResultType.colDataType = CalpontSystemCatalog::LONGDOUBLE; - fResultType.colWidth = sizeof(long double); - fResultType.precision = -1; + if (fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::DECIMAL) + { + fResultType.colWidth = sizeof(int128_t); + } + else + { + fResultType.colDataType = CalpontSystemCatalog::LONGDOUBLE; + fResultType.colWidth = sizeof(long double); + fResultType.precision = -1; + } } } @@ -661,18 +670,35 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull) break; } - default: + case 8: { if (row.equals<8>(BIGINTNULL, fInputIndex)) isNull = true; else { - fResult.decimalVal.value = (int64_t)row.getUintField<8>(fInputIndex); + fResult.decimalVal.value = row.getIntField<8>(fInputIndex); fResult.decimalVal.scale = (unsigned)fResultType.scale; } break; } + + case 16: + { + int128_t dec = row.getInt128Field(fInputIndex); + if (utils::isWideDecimalNullValue(dec)) + isNull = true; + else + { + fResult.decimalVal.s128Value = dec; + fResult.decimalVal.scale = (unsigned)fResultType.scale; + } + + break; + } + default: + // Should log error + break; } break; diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index 942cd254f..09b57a325 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -938,7 +938,7 @@ void GroupConcatOrderBy::getResult(uint8_t* buff, const string& sep) rowStack.pop(); } - size_t resultSize = oss.str().size(); + int64_t resultSize = oss.str().size(); resultSize = (resultSize > fGroupConcatLen) ? fGroupConcatLen : resultSize; fOutputString.reset(new uint8_t[resultSize + 2]); fOutputString[resultSize] = '\0'; @@ -1102,7 +1102,7 @@ void GroupConcatNoOrder::getResult(uint8_t* buff, const string& sep) fDataQueue.pop(); } - size_t resultSize = oss.str().size(); + int64_t resultSize = oss.str().size(); resultSize = (resultSize > fGroupConcatLen) ? fGroupConcatLen : resultSize; fOutputString.reset(new uint8_t[resultSize + 2]); fOutputString[resultSize] = '\0'; diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index f334a3375..2724d2ba1 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -2342,9 +2342,9 @@ int ha_mcs_impl_discover_existence(const char* schema, const char* name) int ha_mcs_impl_direct_update_delete_rows(bool execute, ha_rows *affected_rows, const std::vector& condStack) { THD* thd = current_thd; - int rc = 0; cal_impl_if::gp_walk_info gwi; gwi.thd = thd; + int rc = 0; if (thd->slave_thread && !get_replication_slave(thd) && ( thd->lex->sql_command == SQLCOM_INSERT || diff --git a/utils/messageqcpp/bytestream.cpp b/utils/messageqcpp/bytestream.cpp index edfa9f90b..ec6872dcb 100644 --- a/utils/messageqcpp/bytestream.cpp +++ b/utils/messageqcpp/bytestream.cpp @@ -236,6 +236,17 @@ ByteStream& ByteStream::operator<<(const uint64_t o) } // WIP MCOL-641 +ByteStream& ByteStream::operator<<(const int128_t& o) +{ + if (fBuf == 0 || (fCurInPtr - fBuf + 16U > fMaxLen + ISSOverhead)) + growBuf(fMaxLen + BlockSize); + + *((int128_t*) fCurInPtr) = o; + fCurInPtr += 16; + + return *this; +} + ByteStream& ByteStream::operator<<(const uint128_t& o) { if (fBuf == 0 || (fCurInPtr - fBuf + 16U > fMaxLen + ISSOverhead)) @@ -332,6 +343,13 @@ ByteStream& ByteStream::operator>>(uint64_t& o) } // WIP MCOL-641 +ByteStream& ByteStream::operator>>(int128_t& o) +{ + peek(o); + fCurOutPtr += 16; + return *this; +} + ByteStream& ByteStream::operator>>(uint128_t& o) { peek(o); @@ -420,6 +438,15 @@ void ByteStream::peek(uint64_t& o) const } // WIP MCOL-641 +void ByteStream::peek(int128_t& o) const +{ + + if (length() < 16) + throw underflow_error("ByteStream>int128_t: not enough data in stream to fill datatype"); + + o = *((int128_t*) fCurOutPtr); +} + void ByteStream::peek(uint128_t& o) const { diff --git a/utils/messageqcpp/bytestream.h b/utils/messageqcpp/bytestream.h index aea3c60b1..59aa1679c 100644 --- a/utils/messageqcpp/bytestream.h +++ b/utils/messageqcpp/bytestream.h @@ -45,6 +45,7 @@ class ByteStreamTestSuite; #define EXPORT #endif +using int128_t = __int128; using uint128_t = unsigned __int128; namespace messageqcpp @@ -147,6 +148,10 @@ public: */ EXPORT ByteStream& operator<<(const uint64_t o); // WIP MCOL-641 + /** + * push an uint128_t onto the end of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator<<(const int128_t& o); /** * push an uint128_t onto the end of the stream. The byte order is whatever the native byte order is. */ @@ -216,6 +221,10 @@ public: */ EXPORT ByteStream& operator>>(uint64_t& o); // WIP MCOL-641 + /** + * extract an uint128_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator>>(int128_t& o); /** * extract an uint128_t from the front of the stream. The byte order is whatever the native byte order is. */ @@ -291,6 +300,10 @@ public: */ EXPORT void peek(uint64_t& o) const; // WIP MCOL-641 + /** + * Peek at an uint128_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT void peek(int128_t& o) const; /** * Peek at an uint128_t from the front of the stream. The byte order is whatever the native byte order is. */ diff --git a/utils/regr/moda.cpp b/utils/regr/moda.cpp index caf6cd1f8..e1bc44a9a 100644 --- a/utils/regr/moda.cpp +++ b/utils/regr/moda.cpp @@ -103,9 +103,12 @@ mcsv1_UDAF* moda::getImpl(mcsv1Context* context) case 4: data->modaImpl = &moda_impl_int32; break; - default: + case 8: data->modaImpl = &moda_impl_int64; break; + case 16: + data->modaImpl = &moda_impl_int128; + break; } break; case execplan::CalpontSystemCatalog::UTINYINT: @@ -179,11 +182,18 @@ mcsv1_UDAF::ReturnCode moda::init(mcsv1Context* context, { context->setColWidth(4); } - else + else if (colTypes[0].precision < 19) { context->setColWidth(8); } + else + { + context->setColWidth(16); + } + + context->setScale(colTypes[0].scale); } + context->setPrecision(colTypes[0].precision); mcsv1_UDAF* impl = getImpl(context); @@ -203,10 +213,7 @@ template mcsv1_UDAF::ReturnCode Moda_impl_T::init(mcsv1Context* context, ColumnDatum* colTypes) { - context->setScale(context->getScale()); - context->setPrecision(19); return mcsv1_UDAF::SUCCESS; - } template @@ -224,7 +231,7 @@ mcsv1_UDAF::ReturnCode Moda_impl_T::nextValue(mcsv1Context* context, ColumnDa { static_any::any& valIn = valsIn[0].columnData; ModaData* data = static_cast(context->getUserData()); - std::unordered_map* map = data->getMap(); + std::unordered_map >* map = data->getMap(); if (valIn.empty()) { @@ -261,9 +268,9 @@ mcsv1_UDAF::ReturnCode Moda_impl_T::subEvaluate(mcsv1Context* context, const ModaData* outData = static_cast(context->getUserData()); const ModaData* inData = static_cast(userDataIn); - std::unordered_map* outMap = outData->getMap(); - std::unordered_map* inMap = inData->getMap(); - typename std::unordered_map::const_iterator iter; + std::unordered_map >* outMap = outData->getMap(); + std::unordered_map >* inMap = inData->getMap(); + typename std::unordered_map >::const_iterator iter; for (iter = inMap->begin(); iter != inMap->end(); ++iter) { @@ -283,7 +290,7 @@ mcsv1_UDAF::ReturnCode Moda_impl_T::evaluate(mcsv1Context* context, static_an long double avg = 0; T val = 0; ModaData* data = static_cast(context->getUserData()); - std::unordered_map* map = data->getMap(); + std::unordered_map >* map = data->getMap(); if (map->size() == 0) { @@ -292,7 +299,7 @@ mcsv1_UDAF::ReturnCode Moda_impl_T::evaluate(mcsv1Context* context, static_an } avg = data->fCount ? data->fSum / data->fCount : 0; - typename std::unordered_map::iterator iter; + typename std::unordered_map >::iterator iter; for (iter = map->begin(); iter != map->end(); ++iter) { @@ -303,11 +310,13 @@ mcsv1_UDAF::ReturnCode Moda_impl_T::evaluate(mcsv1Context* context, static_an } else if (iter->second == maxCnt) { + T absval = val >= 0 ? val : -val; + T absfirst = iter->first >= 0 ? iter->first : -iter->first; // Tie breaker: choose the closest to avg. If still tie, choose smallest long double dist1 = val > avg ? (long double)val-avg : avg-(long double)val; long double dist2 = iter->first > avg ? (long double)iter->first-avg : avg-(long double)iter->first; if ((dist1 > dist2) - || ((dist1 == dist2) && (std::fabs(val) > std::fabs(iter->first)))) + || ((dist1 == dist2) && (absval > absfirst))) { val = iter->first; } @@ -328,7 +337,7 @@ mcsv1_UDAF::ReturnCode Moda_impl_T::dropValue(mcsv1Context* context, ColumnDa { static_any::any& valDropped = valsDropped[0].columnData; ModaData* data = static_cast(context->getUserData()); - std::unordered_map* map = data->getMap(); + std::unordered_map >* map = data->getMap(); if (valDropped.empty()) { @@ -379,9 +388,12 @@ void ModaData::serialize(messageqcpp::ByteStream& bs) const case 4: serializeMap(bs); break; - default: + case 8: serializeMap(bs); break; + case 16: + serializeMap(bs); + break; } break; case execplan::CalpontSystemCatalog::UTINYINT: @@ -447,9 +459,12 @@ void ModaData::unserialize(messageqcpp::ByteStream& bs) case 4: unserializeMap(bs); break; - default: + case 8: unserializeMap(bs); break; + case 16: + unserializeMap(bs); + break; } break; case execplan::CalpontSystemCatalog::UTINYINT: @@ -519,10 +534,14 @@ void ModaData::cleanup() clear(); deleteMap(); break; - default: + case 8: clear(); deleteMap(); break; + case 16: + clear(); + deleteMap(); + break; } break; case execplan::CalpontSystemCatalog::UTINYINT: diff --git a/utils/regr/moda.h b/utils/regr/moda.h index f8c0ef822..ae47f8462 100644 --- a/utils/regr/moda.h +++ b/utils/regr/moda.h @@ -45,6 +45,7 @@ #include "mcsv1_udaf.h" #include "calpontsystemcatalog.h" #include "windowfunctioncolumn.h" +#include "hasher.h" #if defined(_MSC_VER) && defined(xxxRGNODE_DLLEXPORT) #define EXPORT __declspec(dllexport) @@ -54,6 +55,38 @@ namespace mcsv1sdk { +// A hasher that handles int128_t +template +struct hasher +{ + inline size_t operator()(T val) const + { + return fHasher((char*) &val, sizeof(T)); + } + +private: + utils::Hasher fHasher; +}; + +template<> +struct hasher +{ + inline size_t operator()(long double val) const + { + if (sizeof(long double) == 8) // Probably just MSC, but you never know. + { + return fHasher((char*) &val, sizeof(long double)); + } + else + { + // For Linux x86_64, long double is stored in 128 bits, but only 80 are significant + return fHasher((char*) &val, 10); + } + } +private: + utils::Hasher fHasher; +}; + // Override UserData for data storage struct ModaData : public UserData { @@ -69,22 +102,22 @@ struct ModaData : public UserData virtual void unserialize(messageqcpp::ByteStream& bs); template - std::unordered_map* getMap() + std::unordered_map >* getMap() { if (!fMap) { // Just in time creation - fMap = new std::unordered_map; + fMap = new std::unordered_map >; } - return (std::unordered_map*) fMap; + return (std::unordered_map >*) fMap; } // The const version is only called by serialize() // It shouldn't (and can't) create a new map. template - std::unordered_map* getMap() const + std::unordered_map >* getMap() const { - return (std::unordered_map*) fMap; + return (std::unordered_map >*) fMap; } template @@ -92,7 +125,7 @@ struct ModaData : public UserData { if (fMap) { - delete (std::unordered_map*) fMap; + delete (std::unordered_map >*) fMap; fMap = NULL; } } @@ -123,10 +156,10 @@ private: template void serializeMap(messageqcpp::ByteStream& bs) const { - std::unordered_map* map = getMap(); + std::unordered_map >* map = getMap(); if (map) { - typename std::unordered_map::const_iterator iter; + typename std::unordered_map >::const_iterator iter; bs << (uint64_t)map->size(); for (iter = map->begin(); iter != map->end(); ++iter) { @@ -147,7 +180,7 @@ private: T num; uint64_t sz; bs >> sz; - std::unordered_map* map = getMap(); + std::unordered_map >* map = getMap(); map->clear(); for (uint64_t i = 0; i < sz; ++i) { @@ -234,6 +267,7 @@ protected: Moda_impl_T moda_impl_int16; Moda_impl_T moda_impl_int32; Moda_impl_T moda_impl_int64; + Moda_impl_T moda_impl_int128; Moda_impl_T moda_impl_uint8; Moda_impl_T moda_impl_uint16; Moda_impl_T moda_impl_uint32; diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index f39dea607..491b4ef27 100755 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -479,7 +479,7 @@ inline bool RowAggregation::isNull(const RowGroup* pRowGroup, const Row& row, in else { //@bug 1821 - ret = (row.equals("", col) || row.equals(joblist::CPNULLSTRMARK, col)); + ret = (row.equals(string(""), col) || row.equals(joblist::CPNULLSTRMARK, col)); } break; @@ -577,7 +577,7 @@ inline bool RowAggregation::isNull(const RowGroup* pRowGroup, const Row& row, in case execplan::CalpontSystemCatalog::VARBINARY: case execplan::CalpontSystemCatalog::BLOB: { - ret = (row.equals("", col) || row.equals(joblist::CPNULLSTRMARK, col)); + ret = (row.equals(string(""), col) || row.equals(joblist::CPNULLSTRMARK, col)); break; } @@ -1528,7 +1528,7 @@ void RowAggregation::doSum(const Row& rowIn, int64_t colIn, int64_t colOut, int int128_t* dec = reinterpret_cast(wideValInPtr); if (LIKELY(!isNull(fRowGroupOut, fRow, colOut))) { - int128_t *valOutPtr = fRow.getBinaryField(valOutPtr, colOut); + int128_t *valOutPtr = fRow.getBinaryField(colOut); int128_t sum = *valOutPtr + *dec; fRow.setBinaryField_offset(&sum, sizeof(sum), offset); } @@ -1989,7 +1989,7 @@ void RowAggregation::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, int6 int128_t* dec = reinterpret_cast(wideValInPtr); if (LIKELY(notFirstValue)) { - int128_t *valOutPtr = fRow.getBinaryField(valOutPtr, colOut); + int128_t *valOutPtr = fRow.getBinaryField(colOut); int128_t sum = *valOutPtr + *dec; fRow.setBinaryField_offset(&sum, sizeof(sum), offset); } @@ -4366,7 +4366,7 @@ void RowAggregationUMP2::doAvg(const Row& rowIn, int64_t colIn, int64_t colOut, int128_t* dec = reinterpret_cast(wideValInPtr); if (LIKELY(cnt > 0)) { - int128_t *valOutPtr = fRow.getBinaryField(valOutPtr, colOut); + int128_t *valOutPtr = fRow.getBinaryField(colOut); int128_t sum = *valOutPtr + *dec; fRow.setBinaryField_offset(&sum, sizeof(sum), offset); fRow.setUintField(rowIn.getUintField(colIn + 1) + cnt, colAux); diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index c6a1daed8..31712c2ad 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -380,7 +380,8 @@ public: template inline bool equals(uint64_t val, uint32_t colIndex) const; inline bool equals(long double val, uint32_t colIndex) const; bool equals(const std::string& val, uint32_t colIndex) const; - + inline bool equals(int128_t val, uint32_t colIndex) const; + inline double getDoubleField(uint32_t colIndex) const; inline float getFloatField(uint32_t colIndex) const; inline double getDecimalField(uint32_t colIndex) const @@ -388,6 +389,8 @@ public: return 0.0; // TODO: Do something here } inline long double getLongDoubleField(uint32_t colIndex) const; + inline int128_t getInt128Field(uint32_t colIndex) const; + inline uint128_t getUint128Field(uint32_t colIndex) const; inline uint64_t getBaseRid() const; inline uint64_t getRid() const; @@ -417,6 +420,7 @@ public: inline void setFloatField(float val, uint32_t colIndex); inline void setDecimalField(double val, uint32_t colIndex) { }; // TODO: Do something here inline void setLongDoubleField(long double val, uint32_t colIndex); + inline void setInt128Field(int128_t val, uint32_t colIndex); inline void setRid(uint64_t rid); @@ -674,7 +678,7 @@ inline bool Row::inStringTable(uint32_t col) const template inline bool Row::equals(T* value, uint32_t colIndex) const { - return reinterpret_cast(&data[offsets[colIndex]]) == value; + return *reinterpret_cast(&data[offsets[colIndex]]) == *value; } template @@ -704,6 +708,12 @@ inline bool Row::equals(long double val, uint32_t colIndex) const { return *((long double*) &data[offsets[colIndex]]) == val; } + +inline bool Row::equals(int128_t val, uint32_t colIndex) const +{ + return *((int128_t*) &data[offsets[colIndex]]) == val; +} + template inline uint64_t Row::getUintField(uint32_t colIndex) const { @@ -951,6 +961,16 @@ inline long double Row::getLongDoubleField(uint32_t colIndex) const return *((long double*) &data[offsets[colIndex]]); } +inline int128_t Row::getInt128Field(uint32_t colIndex) const +{ + return *((int128_t*) &data[offsets[colIndex]]); +} + +inline uint128_t Row::getUint128Field(uint32_t colIndex) const +{ + return *((uint128_t*) &data[offsets[colIndex]]); +} + inline uint64_t Row::getRid() const { return baseRid + *((uint16_t*) data); @@ -1158,6 +1178,11 @@ inline void Row::setLongDoubleField(long double val, uint32_t colIndex) } } +inline void Row::setInt128Field(int128_t val, uint32_t colIndex) +{ + *((int128_t*)&data[offsets[colIndex]]) = val; +} + inline void Row::setVarBinaryField(const std::string& val, uint32_t colIndex) { if (inStringTable(colIndex)) diff --git a/utils/udfsdk/mcsv1_udaf.cpp b/utils/udfsdk/mcsv1_udaf.cpp index 326c7fee0..f23d53c52 100755 --- a/utils/udfsdk/mcsv1_udaf.cpp +++ b/utils/udfsdk/mcsv1_udaf.cpp @@ -278,11 +278,13 @@ const static_any::any& mcsv1_UDAF::shortTypeId((short)1); const static_any::any& mcsv1_UDAF::intTypeId((int)1); const static_any::any& mcsv1_UDAF::longTypeId((long)1); const static_any::any& mcsv1_UDAF::llTypeId((long long)1); +const static_any::any& mcsv1_UDAF::int128TypeId((int128_t)1); const static_any::any& mcsv1_UDAF::ucharTypeId((unsigned char)1); const static_any::any& mcsv1_UDAF::ushortTypeId((unsigned short)1); const static_any::any& mcsv1_UDAF::uintTypeId((unsigned int)1); const static_any::any& mcsv1_UDAF::ulongTypeId((unsigned long)1); const static_any::any& mcsv1_UDAF::ullTypeId((unsigned long long)1); +const static_any::any& mcsv1_UDAF::uint128TypeId((uint128_t)1); const static_any::any& mcsv1_UDAF::floatTypeId((float)1); const static_any::any& mcsv1_UDAF::doubleTypeId((double)1); const static_any::any& mcsv1_UDAF::strTypeId(typeStr); diff --git a/utils/udfsdk/mcsv1_udaf.h b/utils/udfsdk/mcsv1_udaf.h index d7096c052..6c0ea7370 100755 --- a/utils/udfsdk/mcsv1_udaf.h +++ b/utils/udfsdk/mcsv1_udaf.h @@ -638,11 +638,13 @@ protected: static const static_any::any& intTypeId; static const static_any::any& longTypeId; static const static_any::any& llTypeId; + static const static_any::any& int128TypeId; static const static_any::any& ucharTypeId; static const static_any::any& ushortTypeId; static const static_any::any& uintTypeId; static const static_any::any& ulongTypeId; static const static_any::any& ullTypeId; + static const static_any::any& uint128TypeId; static const static_any::any& floatTypeId; static const static_any::any& doubleTypeId; static const static_any::any& strTypeId; @@ -1069,6 +1071,14 @@ inline T mcsv1_UDAF::convertAnyTo(static_any::any& valIn) { val = valIn.cast(); } + else if (valIn.compatible(int128TypeId)) + { + val = valIn.cast(); + } + else if (valIn.compatible(uint128TypeId)) + { + val = valIn.cast(); + } else { throw std::runtime_error("mcsv1_UDAF::convertAnyTo(): input param has unrecognized type"); diff --git a/utils/windowfunction/frameboundrange.cpp b/utils/windowfunction/frameboundrange.cpp index 21323f823..e10cb6fbd 100644 --- a/utils/windowfunction/frameboundrange.cpp +++ b/utils/windowfunction/frameboundrange.cpp @@ -285,7 +285,6 @@ void FrameBoundExpressionRange::validate() case execplan::CalpontSystemCatalog::MEDINT: case execplan::CalpontSystemCatalog::INT: case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::DECIMAL: { int64_t tmp = this->fRow.getIntField(this->fIndex[1]); this->fIsZero = (tmp == 0); @@ -299,6 +298,33 @@ void FrameBoundExpressionRange::validate() break; } + case execplan::CalpontSystemCatalog::DECIMAL: + { + if (this->fRow.getColumnWidth(this->fIndex[1]) < 16) + { + int64_t tmp = this->fRow.getIntField(this->fIndex[1]); + this->fIsZero = (tmp == 0); + + if (tmp < 0) + { + invalid = true; + oss << ""; + } + } + else + { + int128_t tmp = this->fRow.getInt128Field(this->fIndex[1]); + this->fIsZero = (tmp == 0); + + if (tmp < 0) + { + invalid = true; + oss << ""; + } + } + break; + } + case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::UDOUBLE: { @@ -343,15 +369,30 @@ void FrameBoundExpressionRange::validate() break; } + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (this->fRow.getColumnWidth(this->fIndex[1]) < 16) + { + uint64_t tmp = this->fRow.getUintField(this->fIndex[1]); + this->fIsZero = (tmp == 0); + break; + } + else + { + uint128_t tmp = this->fRow.getUint128Field(this->fIndex[1]); + this->fIsZero = (tmp == 0); + break; + } + } + case execplan::CalpontSystemCatalog::UTINYINT: case execplan::CalpontSystemCatalog::USMALLINT: case execplan::CalpontSystemCatalog::UMEDINT: case execplan::CalpontSystemCatalog::UINT: case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UDECIMAL: default: { - int64_t tmp = this->fRow.getIntField(this->fIndex[1]); + uint64_t tmp = this->fRow.getUintField(this->fIndex[1]); this->fIsZero = (tmp == 0); break; } diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index 58b01ee38..6814ed4ad 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -610,7 +610,7 @@ void CompareRule::compileRules(const std::vector& spec, const rowgr case datatypes::MAXDECIMALWIDTH: c = new WideDecimalCompare(*i, offsets[i->fIndex]); break; case datatypes::MAXLEGACYWIDTH: - c = new BigIntCompare(*i); + c = new BigIntCompare(*i); break; case 1 : c = new TinyIntCompare(*i); break; case 2 : @@ -837,8 +837,6 @@ bool EqualCompData::operator()(Row::Pointer a, Row::Pointer b) case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::UTINYINT: case CalpontSystemCatalog::USMALLINT: case CalpontSystemCatalog::UMEDINT: @@ -854,6 +852,21 @@ bool EqualCompData::operator()(Row::Pointer a, Row::Pointer b) break; } + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + // equal compare. ignore sign and null + if (fRow1.getColumnWidth(*i) < 16) + { + eq = (fRow1.getUintField(*i) == fRow2.getUintField(*i)); + } + else + { + eq = (fRow1.getUint128Field(*i) == fRow2.getUint128Field(*i)); + } + break; + } + case CalpontSystemCatalog::CHAR: case CalpontSystemCatalog::VARCHAR: { diff --git a/utils/windowfunction/wf_count.cpp b/utils/windowfunction/wf_count.cpp index 7af4acb0a..d7e9b104e 100644 --- a/utils/windowfunction/wf_count.cpp +++ b/utils/windowfunction/wf_count.cpp @@ -55,7 +55,7 @@ namespace windowfunction template -boost::shared_ptr WF_count::makeFunction(int id, const string& name, int ct) +boost::shared_ptr WF_count::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) { boost::shared_ptr func; @@ -70,7 +70,25 @@ boost::shared_ptr WF_count::makeFunction(int id, const st } case CalpontSystemCatalog::BINARY: + { std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; + break; + } + + case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: + { + if (wc->functionParms()[0]->resultType().colWidth < 16) + { + func.reset(new WF_count(id, name)); + } + else + { + func.reset(new WF_count(id, name)); + } + break; + } + default: { func.reset(new WF_count(id, name)); break; @@ -176,7 +194,7 @@ void WF_count::operator()(int64_t b, int64_t e, int64_t c) template -boost::shared_ptr WF_count::makeFunction(int, const string&, int); +boost::shared_ptr WF_count::makeFunction(int, const string&, int, WindowFunctionColumn*); } //namespace diff --git a/utils/windowfunction/wf_count.h b/utils/windowfunction/wf_count.h index cc4b63959..934489815 100644 --- a/utils/windowfunction/wf_count.h +++ b/utils/windowfunction/wf_count.h @@ -43,7 +43,7 @@ public: WindowFunctionType* clone() const; void resetData(); - static boost::shared_ptr makeFunction(int, const string&, int); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); protected: diff --git a/utils/windowfunction/wf_lead_lag.cpp b/utils/windowfunction/wf_lead_lag.cpp index cb15f3490..62fab334b 100644 --- a/utils/windowfunction/wf_lead_lag.cpp +++ b/utils/windowfunction/wf_lead_lag.cpp @@ -53,7 +53,7 @@ namespace windowfunction template -boost::shared_ptr WF_lead_lag::makeFunction(int id, const string& name, int ct) +boost::shared_ptr WF_lead_lag::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) { boost::shared_ptr func; @@ -64,7 +64,6 @@ boost::shared_ptr WF_lead_lag::makeFunction(int id, const case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: { func.reset(new WF_lead_lag(id, name)); break; @@ -75,7 +74,6 @@ boost::shared_ptr WF_lead_lag::makeFunction(int id, const case CalpontSystemCatalog::UMEDINT: case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: case CalpontSystemCatalog::TIMESTAMP: @@ -85,6 +83,32 @@ boost::shared_ptr WF_lead_lag::makeFunction(int id, const break; } + case CalpontSystemCatalog::DECIMAL: + { + if (wc->functionParms()[0]->resultType().colWidth < 16) + { + func.reset(new WF_lead_lag(id, name)); + } + else + { + func.reset(new WF_lead_lag(id, name)); + } + break; + } + + case CalpontSystemCatalog::UDECIMAL: + { + if (wc->functionParms()[0]->resultType().colWidth < 16) + { + func.reset(new WF_lead_lag(id, name)); + } + else + { + func.reset(new WF_lead_lag(id, name)); + } + break; + } + case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { @@ -293,10 +317,12 @@ void WF_lead_lag::operator()(int64_t b, int64_t e, int64_t c) template -boost::shared_ptr WF_lead_lag::makeFunction(int, const string&, int); +boost::shared_ptr WF_lead_lag::makeFunction(int, const string&, int, WindowFunctionColumn*); template void WF_lead_lag::parseParms(const std::vector&); template void WF_lead_lag::parseParms(const std::vector&); +template void WF_lead_lag::parseParms(const std::vector&); +template void WF_lead_lag::parseParms(const std::vector&); template void WF_lead_lag::parseParms(const std::vector&); template void WF_lead_lag::parseParms(const std::vector&); template void WF_lead_lag::parseParms(const std::vector&); diff --git a/utils/windowfunction/wf_lead_lag.h b/utils/windowfunction/wf_lead_lag.h index de14ca702..81b707ae2 100644 --- a/utils/windowfunction/wf_lead_lag.h +++ b/utils/windowfunction/wf_lead_lag.h @@ -43,7 +43,7 @@ public: void resetData(); void parseParms(const std::vector&); - static boost::shared_ptr makeFunction(int, const string&, int); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); protected: T fValue; diff --git a/utils/windowfunction/wf_min_max.cpp b/utils/windowfunction/wf_min_max.cpp index 4ae76e84d..f9c410362 100644 --- a/utils/windowfunction/wf_min_max.cpp +++ b/utils/windowfunction/wf_min_max.cpp @@ -52,7 +52,7 @@ namespace windowfunction template -boost::shared_ptr WF_min_max::makeFunction(int id, const string& name, int ct) +boost::shared_ptr WF_min_max::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) { boost::shared_ptr func; @@ -63,7 +63,6 @@ boost::shared_ptr WF_min_max::makeFunction(int id, const case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: { func.reset(new WF_min_max(id, name)); break; @@ -74,7 +73,6 @@ boost::shared_ptr WF_min_max::makeFunction(int id, const case CalpontSystemCatalog::UMEDINT: case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: case CalpontSystemCatalog::TIMESTAMP: @@ -84,6 +82,32 @@ boost::shared_ptr WF_min_max::makeFunction(int id, const break; } + case CalpontSystemCatalog::DECIMAL: + { + if (wc->functionParms()[0]->resultType().colWidth < 16) + { + func.reset(new WF_min_max(id, name)); + } + else + { + func.reset(new WF_min_max(id, name)); + } + break; + } + + case CalpontSystemCatalog::UDECIMAL: + { + if (wc->functionParms()[0]->resultType().colWidth < 16) + { + func.reset(new WF_min_max(id, name)); + } + else + { + func.reset(new WF_min_max(id, name)); + } + break; + } + case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { @@ -173,7 +197,7 @@ void WF_min_max::operator()(int64_t b, int64_t e, int64_t c) template -boost::shared_ptr WF_min_max::makeFunction(int, const string&, int); +boost::shared_ptr WF_min_max::makeFunction(int, const string&, int, WindowFunctionColumn*); } //namespace diff --git a/utils/windowfunction/wf_min_max.h b/utils/windowfunction/wf_min_max.h index aac8d579c..ebe12df65 100644 --- a/utils/windowfunction/wf_min_max.h +++ b/utils/windowfunction/wf_min_max.h @@ -42,7 +42,7 @@ public: WindowFunctionType* clone() const; void resetData(); - static boost::shared_ptr makeFunction(int, const string&, int); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); protected: T fValue; diff --git a/utils/windowfunction/wf_nth_value.cpp b/utils/windowfunction/wf_nth_value.cpp index c736c0e92..9c566345f 100644 --- a/utils/windowfunction/wf_nth_value.cpp +++ b/utils/windowfunction/wf_nth_value.cpp @@ -53,7 +53,7 @@ namespace windowfunction template -boost::shared_ptr WF_nth_value::makeFunction(int id, const string& name, int ct) +boost::shared_ptr WF_nth_value::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) { boost::shared_ptr func; @@ -64,7 +64,6 @@ boost::shared_ptr WF_nth_value::makeFunction(int id, cons case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: { func.reset(new WF_nth_value(id, name)); break; @@ -75,7 +74,6 @@ boost::shared_ptr WF_nth_value::makeFunction(int id, cons case CalpontSystemCatalog::UMEDINT: case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: case CalpontSystemCatalog::TIMESTAMP: @@ -85,6 +83,32 @@ boost::shared_ptr WF_nth_value::makeFunction(int id, cons break; } + case CalpontSystemCatalog::DECIMAL: + { + if (wc->functionParms()[0]->resultType().colWidth < 16) + { + func.reset(new WF_nth_value(id, name)); + } + else + { + func.reset(new WF_nth_value(id, name)); + } + break; + } + + case CalpontSystemCatalog::UDECIMAL: + { + if (wc->functionParms()[0]->resultType().colWidth < 16) + { + func.reset(new WF_nth_value(id, name)); + } + else + { + func.reset(new WF_nth_value(id, name)); + } + break; + } + case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { @@ -273,7 +297,7 @@ void WF_nth_value::operator()(int64_t b, int64_t e, int64_t c) template -boost::shared_ptr WF_nth_value::makeFunction(int, const string&, int); +boost::shared_ptr WF_nth_value::makeFunction(int, const string&, int, WindowFunctionColumn*); } //namespace diff --git a/utils/windowfunction/wf_nth_value.h b/utils/windowfunction/wf_nth_value.h index 313313210..8ea294b02 100644 --- a/utils/windowfunction/wf_nth_value.h +++ b/utils/windowfunction/wf_nth_value.h @@ -43,7 +43,7 @@ public: void resetData(); void parseParms(const std::vector&); - static boost::shared_ptr makeFunction(int, const string&, int); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); protected: T fValue; diff --git a/utils/windowfunction/wf_ntile.cpp b/utils/windowfunction/wf_ntile.cpp index 3bf2a3c24..9361609a6 100644 --- a/utils/windowfunction/wf_ntile.cpp +++ b/utils/windowfunction/wf_ntile.cpp @@ -54,7 +54,7 @@ namespace windowfunction { -boost::shared_ptr WF_ntile::makeFunction(int id, const string& name, int ct) +boost::shared_ptr WF_ntile::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) { boost::shared_ptr func(new WF_ntile(id, name)); return func; diff --git a/utils/windowfunction/wf_ntile.h b/utils/windowfunction/wf_ntile.h index 783ee1983..45d5ba64a 100644 --- a/utils/windowfunction/wf_ntile.h +++ b/utils/windowfunction/wf_ntile.h @@ -43,7 +43,7 @@ public: void resetData(); void parseParms(const std::vector&); - static boost::shared_ptr makeFunction(int, const string&, int); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); protected: diff --git a/utils/windowfunction/wf_percentile.cpp b/utils/windowfunction/wf_percentile.cpp index 0c93b1434..043e36c08 100644 --- a/utils/windowfunction/wf_percentile.cpp +++ b/utils/windowfunction/wf_percentile.cpp @@ -56,7 +56,7 @@ namespace windowfunction { template -boost::shared_ptr WF_percentile::makeFunction(int id, const string& name, int ct) +boost::shared_ptr WF_percentile::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) { boost::shared_ptr func; @@ -69,7 +69,6 @@ boost::shared_ptr WF_percentile::makeFunction(int id, con case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: { func.reset(new WF_percentile(id, name)); break; @@ -80,7 +79,6 @@ boost::shared_ptr WF_percentile::makeFunction(int id, con case CalpontSystemCatalog::UMEDINT: case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::DATE: case CalpontSystemCatalog::DATETIME: case CalpontSystemCatalog::TIMESTAMP: @@ -90,6 +88,32 @@ boost::shared_ptr WF_percentile::makeFunction(int id, con break; } + case CalpontSystemCatalog::DECIMAL: + { + if (wc->functionParms()[0]->resultType().colWidth < 16) + { + func.reset(new WF_percentile(id, name)); + } + else + { + func.reset(new WF_percentile(id, name)); + } + break; + } + + case CalpontSystemCatalog::UDECIMAL: + { + if (wc->functionParms()[0]->resultType().colWidth < 16) + { + func.reset(new WF_percentile(id, name)); + } + else + { + func.reset(new WF_percentile(id, name)); + } + break; + } + case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { @@ -371,7 +395,7 @@ void WF_percentile::operator()(int64_t b, int64_t e, int64_t c) template -boost::shared_ptr WF_percentile::makeFunction(int, const string&, int); +boost::shared_ptr WF_percentile::makeFunction(int, const string&, int, WindowFunctionColumn*); } //namespace diff --git a/utils/windowfunction/wf_percentile.h b/utils/windowfunction/wf_percentile.h index afc2f69b3..ae2c8db50 100644 --- a/utils/windowfunction/wf_percentile.h +++ b/utils/windowfunction/wf_percentile.h @@ -44,7 +44,7 @@ public: void resetData(); void parseParms(const std::vector&); - static boost::shared_ptr makeFunction(int, const string&, int); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); protected: diff --git a/utils/windowfunction/wf_ranking.cpp b/utils/windowfunction/wf_ranking.cpp index 4e9765630..818820ee5 100644 --- a/utils/windowfunction/wf_ranking.cpp +++ b/utils/windowfunction/wf_ranking.cpp @@ -53,7 +53,7 @@ namespace windowfunction { -boost::shared_ptr WF_ranking::makeFunction(int id, const string& name, int ct) +boost::shared_ptr WF_ranking::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) { boost::shared_ptr func(new WF_ranking(id, name)); return func; diff --git a/utils/windowfunction/wf_ranking.h b/utils/windowfunction/wf_ranking.h index c8be23ba4..ce95bd122 100644 --- a/utils/windowfunction/wf_ranking.h +++ b/utils/windowfunction/wf_ranking.h @@ -42,7 +42,7 @@ public: WindowFunctionType* clone() const; void resetData(); - static boost::shared_ptr makeFunction(int, const string&, int); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); protected: diff --git a/utils/windowfunction/wf_row_number.cpp b/utils/windowfunction/wf_row_number.cpp index cac718317..048422614 100644 --- a/utils/windowfunction/wf_row_number.cpp +++ b/utils/windowfunction/wf_row_number.cpp @@ -53,7 +53,7 @@ namespace windowfunction { -boost::shared_ptr WF_row_number::makeFunction(int id, const string& name, int ct) +boost::shared_ptr WF_row_number::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) { boost::shared_ptr func(new WF_row_number(id, name)); return func; diff --git a/utils/windowfunction/wf_row_number.h b/utils/windowfunction/wf_row_number.h index 65a1b6618..4b6a61c08 100644 --- a/utils/windowfunction/wf_row_number.h +++ b/utils/windowfunction/wf_row_number.h @@ -42,7 +42,7 @@ public: WindowFunctionType* clone() const; void resetData(); - static boost::shared_ptr makeFunction(int, const string&, int); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); protected: diff --git a/utils/windowfunction/wf_stats.cpp b/utils/windowfunction/wf_stats.cpp index 8b06eda61..1804ad432 100644 --- a/utils/windowfunction/wf_stats.cpp +++ b/utils/windowfunction/wf_stats.cpp @@ -55,7 +55,7 @@ namespace windowfunction template -boost::shared_ptr WF_stats::makeFunction(int id, const string& name, int ct) +boost::shared_ptr WF_stats::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) { boost::shared_ptr func; @@ -66,7 +66,6 @@ boost::shared_ptr WF_stats::makeFunction(int id, const st case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: { func.reset(new WF_stats(id, name)); break; @@ -77,12 +76,37 @@ boost::shared_ptr WF_stats::makeFunction(int id, const st case CalpontSystemCatalog::UMEDINT: case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: { func.reset(new WF_stats(id, name)); break; } + case CalpontSystemCatalog::DECIMAL: + { + if (wc->functionParms()[0]->resultType().colWidth < 16) + { + func.reset(new WF_stats(id, name)); + } + else + { + func.reset(new WF_stats(id, name)); + } + break; + } + + case CalpontSystemCatalog::UDECIMAL: + { + if (wc->functionParms()[0]->resultType().colWidth < 16) + { + func.reset(new WF_stats(id, name)); + } + else + { + func.reset(new WF_stats(id, name)); + } + break; + } + case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { @@ -177,17 +201,19 @@ void WF_stats::operator()(int64_t b, int64_t e, int64_t c) { int scale = fRow.getScale(colIn); long double factor = pow(10.0, scale); + long double ldSum1 = fSum1; + long double ldSum2 = fSum2; // adjust the scale if necessary if (scale != 0 && cdt != CalpontSystemCatalog::LONGDOUBLE) { - fSum1 /= factor; - fSum2 /= factor * factor; + ldSum1 /= factor; + ldSum2 /= factor * factor; } - long double stat = fSum1 * fSum1 / fCount; - stat = fSum2 - stat; + long double stat = ldSum1 * ldSum1 / fCount; + stat = ldSum2 - stat; if (fFunctionId == WF__STDDEV_POP) stat = sqrt(stat / fCount); @@ -220,7 +246,7 @@ void WF_stats::operator()(int64_t b, int64_t e, int64_t c) template -boost::shared_ptr WF_stats::makeFunction(int, const string&, int); +boost::shared_ptr WF_stats::makeFunction(int, const string&, int, WindowFunctionColumn*); } //namespace diff --git a/utils/windowfunction/wf_stats.h b/utils/windowfunction/wf_stats.h index 381920482..6bff926a6 100644 --- a/utils/windowfunction/wf_stats.h +++ b/utils/windowfunction/wf_stats.h @@ -42,7 +42,7 @@ public: WindowFunctionType* clone() const; void resetData(); - static boost::shared_ptr makeFunction(int, const string&, int); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); protected: long double fSum1; diff --git a/utils/windowfunction/wf_sum_avg.cpp b/utils/windowfunction/wf_sum_avg.cpp index d0d5f46f7..0ad1c1a30 100644 --- a/utils/windowfunction/wf_sum_avg.cpp +++ b/utils/windowfunction/wf_sum_avg.cpp @@ -1,4 +1,4 @@ -/* Copyright (C) 2014 InfiniDB, Inc. +/* CopyrighT (C) 2014 InfiniDB, Inc. Copyright (c) 2019 MariaDB Corporation This program is free software; you can redistribute it and/or @@ -50,112 +50,116 @@ using namespace joblist; #include "wf_sum_avg.h" -#if 0 -namespace +namespace windowfunction { - -template -void checkSumLimit(T sum, T val) +template +void WF_sum_avg::checkSumLimit(long double val, long double sum) { } - -template<> -void checkSumLimit(int64_t sum, int64_t val) +template +inline void WF_sum_avg::checkSumLimit(int128_t val, int128_t sum) { - if (((sum >= 0) && ((numeric_limits::max() - sum) < val)) || - ((sum < 0) && ((numeric_limits::min() - sum) > val))) + if (((sum >= 0) && ((fMax128 - sum) < val)) || + ((sum < 0) && ((fMin128 - sum) > val))) { - string errStr = "SUM(int):"; + string errStr = "SUM(int128_t)"; + errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_OVERFLOW, errStr); + cerr << errStr << endl; + throw IDBExcept(errStr, ERR_WF_OVERFLOW); + } +} - ostringstream oss; - oss << sum << "+" << val; +template +inline void WF_sum_avg::checkSumLimit(uint128_t val, uint128_t sum) +{ + if ((fMaxu128 - sum) < val) + { + string errStr = "SUM(uint128_t)"; + errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_OVERFLOW, errStr); + cerr << errStr << endl; + throw IDBExcept(errStr, ERR_WF_OVERFLOW); + } +} - if (sum > 0) - oss << " > " << numeric_limits::max(); +template +int128_t WF_sum_avg::calculateAvg(int128_t sum, uint64_t count, int scale) +{ + int128_t factor = pow(10.0, scale); + int128_t avg; + if (scale > 0) + { + if ((sum * factor) / factor == sum) + { + avg = sum * factor; + avg /= count; + } else - oss << " < " << numeric_limits::min(); - - errStr += oss.str(); - - errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_OVERFLOW, errStr); - cerr << errStr << endl; - throw IDBExcept(errStr, ERR_WF_OVERFLOW); + { + // scale won't fit before divide, we're gonna lose precision. + avg = sum / count; + if ((avg * factor) / factor != avg) // Still won't fit + { + string errStr = string("AVG(int)"); + errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_OVERFLOW, errStr); + cerr << errStr << endl; + throw IDBExcept(errStr, ERR_WF_OVERFLOW); + } + avg *= factor; + } } -} - - -template<> -void checkSumLimit(uint64_t sum, uint64_t val) -{ - if ((sum >= 0) && ((numeric_limits::max() - sum) < val)) + else { - string errStr = "SUM(unsigned):"; - - ostringstream oss; - oss << sum << "+" << val << " > " << numeric_limits::max(); - errStr += oss.str(); - - errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_OVERFLOW, errStr); - cerr << errStr << endl; - throw IDBExcept(errStr, ERR_WF_OVERFLOW); + avg = sum / count; } -} - - -template<> -long double calculateAvg(long double sum, uint64_t count, int s) -{ - return sum / count; -} - -long double avgWithLimit(long double sum, uint64_t count, int scale, long double u, long double l) -{ - long double factor = pow(10.0, scale); - long double avg = sum / count; - avg *= factor; + avg += (avg < 0) ? (-0.5) : (0.5); - - if (avg > u || avg < l) - { - string errStr = string("AVG") + (l < 0 ? "(int):" : "(unsign)"); - ostringstream oss; - oss << avg; - errStr += oss.str(); - - errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_OVERFLOW, errStr); - cerr << errStr << endl; - throw IDBExcept(errStr, ERR_WF_OVERFLOW); - } - return avg; } - -template<> -int64_t calculateAvg(int64_t sum, uint64_t count, int scale) +template +uint128_t WF_sum_avg::calculateAvg(uint128_t sum, uint64_t count, int scale) { - int64_t t = (int64_t) avgWithLimit(sum, count, scale, - numeric_limits::max(), numeric_limits::min()); - return t; + uint128_t factor = pow(10.0, scale); + uint128_t avg = sum; + if (scale > 0) + { + if ((sum * factor) / factor == sum) + { + avg = sum * factor; + avg /= count; + } + else + { + // scale won't fit before divide, we're gonna lose precision. + avg = sum / count; + if ((avg * factor) / factor != avg) // Still won't fit + { + string errStr = string("AVG(int)"); + errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_OVERFLOW, errStr); + cerr << errStr << endl; + throw IDBExcept(errStr, ERR_WF_OVERFLOW); + } + avg *= factor; + } + } + else + { + avg = sum / count; + } + avg += 0.5; + return avg; } - -template<> -uint64_t calculateAvg(uint64_t sum, uint64_t count, int scale) +template +inline long double WF_sum_avg::calculateAvg(long double sum, uint64_t count, int scale) { - uint64_t t = (uint64_t) avgWithLimit(sum, count, scale, numeric_limits::max(), 0); - return t; + return sum / count; } -} -#endif - -namespace windowfunction -{ - -template -boost::shared_ptr WF_sum_avg::makeFunction(int id, const string& name, int ct) +// For the static function makeFunction, the template parameters are ignored +template +boost::shared_ptr WF_sum_avg::makeFunction(int id, const string& name, int ct, WindowFunctionColumn* wc) { boost::shared_ptr func; switch (ct) @@ -165,9 +169,9 @@ boost::shared_ptr WF_sum_avg::makeFunction(int id, const case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: { - func.reset(new WF_sum_avg(id, name)); + // Look into using int128_t instead of long double + func.reset(new WF_sum_avg(id, name)); break; } @@ -176,29 +180,54 @@ boost::shared_ptr WF_sum_avg::makeFunction(int id, const case CalpontSystemCatalog::UMEDINT: case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::UBIGINT: + { + func.reset(new WF_sum_avg(id, name)); + break; + } + + case CalpontSystemCatalog::DECIMAL: + { + if (wc->functionParms()[0]->resultType().colWidth < 16) + { + func.reset(new WF_sum_avg(id, name)); + } + else + { + func.reset(new WF_sum_avg(id, name)); + } + break; + } + case CalpontSystemCatalog::UDECIMAL: { - func.reset(new WF_sum_avg(id, name)); + if (wc->functionParms()[0]->resultType().colWidth < 16) + { + func.reset(new WF_sum_avg(id, name)); + } + else + { + func.reset(new WF_sum_avg(id, name)); + } break; } case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { - func.reset(new WF_sum_avg(id, name)); + func.reset(new WF_sum_avg(id, name)); break; } case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: { - func.reset(new WF_sum_avg(id, name)); + func.reset(new WF_sum_avg(id, name)); break; } case CalpontSystemCatalog::LONGDOUBLE: { - func.reset(new WF_sum_avg(id, name)); + func.reset(new WF_sum_avg(id, name)); break; } default: @@ -216,15 +245,15 @@ boost::shared_ptr WF_sum_avg::makeFunction(int id, const } -template -WindowFunctionType* WF_sum_avg::clone() const +template +WindowFunctionType* WF_sum_avg::clone() const { - return new WF_sum_avg(*this); + return new WF_sum_avg(*this); } -template -void WF_sum_avg::resetData() +template +void WF_sum_avg::resetData() { fAvg = 0; fSum = 0; @@ -235,8 +264,8 @@ void WF_sum_avg::resetData() } -template -void WF_sum_avg::operator()(int64_t b, int64_t e, int64_t c) +template +void WF_sum_avg::operator()(int64_t b, int64_t e, int64_t c) { uint64_t colOut = fFieldIndex[0]; @@ -251,8 +280,7 @@ void WF_sum_avg::operator()(int64_t b, int64_t e, int64_t c) e = c; uint64_t colIn = fFieldIndex[1]; - double scale = fRow.getScale(colIn); - + int scale = fRow.getScale(colOut) - fRow.getScale(colIn); for (int64_t i = b; i <= e; i++) { if (i % 1000 == 0 && fStep->cancelled()) @@ -263,34 +291,27 @@ void WF_sum_avg::operator()(int64_t b, int64_t e, int64_t c) if (fRow.isNullValue(colIn) == true) continue; - T valIn; CDT cdt; - getValue(colIn, valIn, &cdt); -// checkSumLimit(fSum, valIn); + getValue(colIn, fVal, &cdt); - if ((!fDistinct) || (fSet.find(valIn) == fSet.end())) + if ((!fDistinct) || (fSet.find(fVal) == fSet.end())) { - long double val = valIn; - if (scale && - cdt != CalpontSystemCatalog::LONGDOUBLE) - { - val /= pow(10.0, scale); - } - fSum += val; + checkSumLimit(fVal, fSum); + fSum += (T_OUT)fVal; fCount++; if (fDistinct) - fSet.insert(valIn); + fSet.insert(fVal); } } if ((fCount > 0) && (fFunctionId == WF__AVG || fFunctionId == WF__AVG_DISTINCT)) { - fAvg = fSum / fCount; + fAvg = calculateAvg(fSum, fCount, scale); } } - long double* v = NULL; + T_OUT* v = NULL; if (fCount > 0) { @@ -307,7 +328,7 @@ void WF_sum_avg::operator()(int64_t b, int64_t e, int64_t c) template -boost::shared_ptr WF_sum_avg::makeFunction(int, const string&, int); +boost::shared_ptr WF_sum_avg::makeFunction(int, const string&, int, WindowFunctionColumn*); } //namespace diff --git a/utils/windowfunction/wf_sum_avg.h b/utils/windowfunction/wf_sum_avg.h index 48d46453b..b86a9c16d 100644 --- a/utils/windowfunction/wf_sum_avg.h +++ b/utils/windowfunction/wf_sum_avg.h @@ -29,8 +29,9 @@ namespace windowfunction { - -template +// T_IN is the data type of the input values. +// T_OUT is the data type we are using for output and internal values +template class WF_sum_avg : public WindowFunctionType { public: @@ -38,6 +39,9 @@ public: WindowFunctionType(id, name), fDistinct(id != WF__SUM && id != WF__AVG) { resetData(); + utils::int128Max(fMax128); + utils::int128Min(fMin128); + utils::uint128Max(fMaxu128); } // pure virtual in base @@ -45,16 +49,27 @@ public: WindowFunctionType* clone() const; void resetData(); - static boost::shared_ptr makeFunction(int, const string&, int); + static boost::shared_ptr makeFunction(int, const string&, int, WindowFunctionColumn*); protected: - long double fAvg; - long double fSum; + T_IN fVal; + T_OUT fAvg; + T_OUT fSum; uint64_t fCount; bool fDistinct; - std::set fSet; -}; + std::set fSet; + int128_t fMax128; + int128_t fMin128; + uint128_t fMaxu128; + void checkSumLimit(long double val, long double sum); + void checkSumLimit(int128_t val, int128_t sum); + void checkSumLimit(uint128_t val, uint128_t sum); + + int128_t calculateAvg(int128_t sum, uint64_t count, int scale); + uint128_t calculateAvg(uint128_t sum, uint64_t count, int scale); + long double calculateAvg(long double sum, uint64_t count, int scale); +}; } // namespace diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index c1dc5a908..fb5f3521a 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -56,7 +56,7 @@ using namespace joblist; namespace windowfunction { -boost::shared_ptr WF_udaf::makeFunction(int id, const string& name, int ct, mcsv1sdk::mcsv1Context& context) +boost::shared_ptr WF_udaf::makeFunction(int id, const string& name, int ct, mcsv1sdk::mcsv1Context& context, WindowFunctionColumn* wc) { boost::shared_ptr func; @@ -544,11 +544,13 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, static const static_any::any& intTypeId = (int)1; static const static_any::any& longTypeId = (long)1; static const static_any::any& llTypeId = (long long)1; + static const static_any::any& int128TypeId = (int128_t)1; static const static_any::any& ucharTypeId = (unsigned char)1; static const static_any::any& ushortTypeId = (unsigned short)1; static const static_any::any& uintTypeId = (unsigned int)1; static const static_any::any& ulongTypeId = (unsigned long)1; static const static_any::any& ullTypeId = (unsigned long long)1; + static const static_any::any& uint128TypeId = (int128_t)1; static const static_any::any& floatTypeId = (float)1; static const static_any::any& doubleTypeId = (double)1; static const std::string typeStr(""); @@ -562,6 +564,8 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, // it to whatever they said to return. int64_t intOut = 0; uint64_t uintOut = 0; + int128_t int128Out = 0; + uint128_t uint128Out = 0; float floatOut = 0.0; double doubleOut = 0.0; long double longdoubleOut = 0.0; @@ -651,6 +655,24 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, intOut = (int64_t)doubleOut; oss << doubleOut; } + else if (valOut.compatible(int128TypeId)) + { + int128Out = valOut.cast(); + uintOut = intOut = int128Out; // may truncate + floatOut = int128Out; + doubleOut = int128Out; + longdoubleOut = int128Out; + oss << longdoubleOut; + } + else if (valOut.compatible(uint128TypeId)) + { + uint128Out = valOut.cast(); + uintOut = intOut = uint128Out; // may truncate + floatOut = uint128Out; + doubleOut = uint128Out; + longdoubleOut = uint128Out; + oss << longdoubleOut; + } if (valOut.compatible(strTypeId)) { @@ -901,33 +923,66 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - int64_t valIn; - - if (cc) + if (fRow.getColumnWidth(colIn) < 16) { - valIn = cc->getDecimalVal(fRow, isNull).value; + int64_t valIn; + + if (cc) + { + valIn = cc->getDecimalVal(fRow, isNull).value; + } + else + { + getValue(colIn, valIn); + } + + // Check for distinct, if turned on. + // Currently, distinct only works on the first parameter. + if (k == 0 && fDistinct) + { + std::pair val = make_pair(valIn, 1); + std::pair distinct; + distinct = fDistinctMap.insert(val); + if (distinct.second == false) + { + ++(*distinct.first).second; + bSkipIt = true; + continue; + } + } + + datum.columnData = valIn; } else { - getValue(colIn, valIn); - } + int128_t valIn; - // Check for distinct, if turned on. - // Currently, distinct only works on the first parameter. - if (k == 0 && fDistinct) - { - std::pair val = make_pair(valIn, 1); - std::pair distinct; - distinct = fDistinctMap.insert(val); - if (distinct.second == false) + if (cc) { - ++(*distinct.first).second; - bSkipIt = true; - continue; + valIn = cc->getDecimalVal(fRow, isNull).s128Value; + } + else + { + getValue(colIn, valIn); } - } - datum.columnData = valIn; + // Check for distinct, if turned on. + // Currently, distinct only works on the first parameter. + if (k == 0 && fDistinct) + { + std::pair val = make_pair(valIn, 1); + std::pair distinct; + distinct = fDistinctMap.insert(val); + if (distinct.second == false) + { + ++(*distinct.first).second; + bSkipIt = true; + continue; + } + } + + datum.columnData = valIn; + } break; } @@ -1154,7 +1209,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) fPrev = c; } -boost::shared_ptr WF_udaf::makeFunction(int id, const string& name, int ct, mcsv1sdk::mcsv1Context& context); +boost::shared_ptr WF_udaf::makeFunction(int id, const string& name, int ct, mcsv1sdk::mcsv1Context& context, WindowFunctionColumn*); } //namespace // vim:ts=4 sw=4: diff --git a/utils/windowfunction/wf_udaf.h b/utils/windowfunction/wf_udaf.h index 38515285f..e87b39759 100644 --- a/utils/windowfunction/wf_udaf.h +++ b/utils/windowfunction/wf_udaf.h @@ -104,7 +104,7 @@ protected: public: static boost::shared_ptr makeFunction(int id, const string& name, - int ct, mcsv1sdk::mcsv1Context& context); + int ct, mcsv1sdk::mcsv1Context& context, WindowFunctionColumn* wc); }; diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index 600f5784f..6356a04bb 100644 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -150,67 +150,67 @@ WindowFunctionType::makeWindowFunction(const string& name, int ct, WindowFunctio int functionId = windowFunctionId[algorithm::to_upper_copy(name)]; // The template parameters here are dummies to execute the static makeFunction - // which sets the real type based on ct. + // which sets the real types based on ct. switch (functionId) { case WF__COUNT_ASTERISK: case WF__COUNT: case WF__COUNT_DISTINCT: - af = WF_count::makeFunction(functionId, name, ct); + af = WF_count::makeFunction(functionId, name, ct, wc); break; case WF__MIN: case WF__MAX: - af = WF_min_max::makeFunction(functionId, name, ct); + af = WF_min_max::makeFunction(functionId, name, ct, wc); break; case WF__SUM: case WF__AVG: case WF__SUM_DISTINCT: case WF__AVG_DISTINCT: - af = WF_sum_avg::makeFunction(functionId, name, ct); + af = WF_sum_avg::makeFunction(functionId, name, ct, wc); break; case WF__STDDEV_POP: case WF__STDDEV_SAMP: case WF__VAR_POP: case WF__VAR_SAMP: - af = WF_stats::makeFunction(functionId, name, ct); + af = WF_stats::makeFunction(functionId, name, ct, wc); break; case WF__ROW_NUMBER: - af = WF_row_number::makeFunction(functionId, name, ct); + af = WF_row_number::makeFunction(functionId, name, ct, wc); break; case WF__RANK: case WF__DENSE_RANK: case WF__PERCENT_RANK: case WF__CUME_DIST: - af = WF_ranking::makeFunction(functionId, name, ct); + af = WF_ranking::makeFunction(functionId, name, ct, wc); break; case WF__FIRST_VALUE: case WF__LAST_VALUE: case WF__NTH_VALUE: - af = WF_nth_value::makeFunction(functionId, name, ct); + af = WF_nth_value::makeFunction(functionId, name, ct, wc); break; case WF__LEAD: case WF__LAG: - af = WF_lead_lag::makeFunction(functionId, name, ct); + af = WF_lead_lag::makeFunction(functionId, name, ct, wc); break; case WF__NTILE: - af = WF_ntile::makeFunction(functionId, name, ct); + af = WF_ntile::makeFunction(functionId, name, ct, wc); break; case WF__PERCENTILE_CONT: case WF__PERCENTILE_DISC: - af = WF_percentile::makeFunction(functionId, name, ct); + af = WF_percentile::makeFunction(functionId, name, ct, wc); break; case WF__UDAF: - af = WF_udaf::makeFunction(functionId, name, ct, wc->getUDAFContext()); + af = WF_udaf::makeFunction(functionId, name, ct, wc->getUDAFContext(), wc); break; case WF__REGR_SLOPE: @@ -229,7 +229,7 @@ WindowFunctionType::makeWindowFunction(const string& name, int ct, WindowFunctio break; } - // Copy the only the constant parameter pointers + // Copy only the constant parameter pointers af->constParms(wc->functionParms()); return af; @@ -308,6 +308,26 @@ template<> void WindowFunctionType::getValue(uint64_t i, string& t, CDT* // By not setting cdt, we let it default to the column's type } +template<> void WindowFunctionType::getValue(uint64_t i, int128_t& t, CDT* cdt) +{ + t = fRow.getInt128Field(i); + + if (cdt) + { + *cdt = execplan::CalpontSystemCatalog::DECIMAL; + } +} + +template<> void WindowFunctionType::getValue(uint64_t i, uint128_t& t, CDT* cdt) +{ + t = fRow.getUint128Field(i); + + if (cdt) + { + *cdt = execplan::CalpontSystemCatalog::DECIMAL; + } +} + template void WindowFunctionType::setValue(uint64_t i, T& t) { } @@ -337,6 +357,11 @@ template<> void WindowFunctionType::setValue(uint64_t i, long doubl fRow.setLongDoubleField(t, i); } +template<> void WindowFunctionType::setValue(uint64_t i, int128_t& t) +{ + fRow.setInt128Field(t, i); +} + template<> void WindowFunctionType::setValue(uint64_t i, string& t) { fRow.setStringField(t, i); @@ -389,7 +414,6 @@ void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, T* v) case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: { int64_t iv = *v; setValue(i, iv); @@ -401,13 +425,26 @@ void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, T* v) case CalpontSystemCatalog::UMEDINT: case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: { uint64_t uv = *v; setValue(i, uv); break; } + case CalpontSystemCatalog::DECIMAL: + { + int128_t iv = *v; + setValue(i, iv); + break; + } + + case CalpontSystemCatalog::UDECIMAL: + { + uint128_t uv = *v; + setValue(i, uv); + break; + } + case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { @@ -452,7 +489,6 @@ void WindowFunctionType::implicit2T(uint64_t i, T& t, int s) case CalpontSystemCatalog::MEDINT: case CalpontSystemCatalog::INT: case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: { t = (T) fRow.getIntField(i); pw = s - fRow.getScale(i); // pw is difference of scales, will be in [-18, 18] @@ -470,7 +506,6 @@ void WindowFunctionType::implicit2T(uint64_t i, T& t, int s) case CalpontSystemCatalog::UMEDINT: case CalpontSystemCatalog::UINT: case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: { t = (T) fRow.getUintField(i); pw = s - fRow.getScale(i); // pw is difference of scales, will be in [-18, 18] @@ -483,6 +518,40 @@ void WindowFunctionType::implicit2T(uint64_t i, T& t, int s) break; } + case CalpontSystemCatalog::DECIMAL: + { + uint32_t w = fRow.getColumnWidth(i); + if (w < 16) + t = (T) fRow.getIntField(i); + else + t = (T) fRow.getInt128Field(i); + pw = s - fRow.getScale(i); // pw is difference of scales, will be in [-18, 18] + + if (pw > 0) + t *= IDB_pow[pw]; + else if (pw < 0) + t /= IDB_pow[-pw]; + + break; + } + + case CalpontSystemCatalog::UDECIMAL: + { + uint32_t w = fRow.getColumnWidth(i); + if (w < 16) + t = (T) fRow.getUintField(i); + else + t = (T) fRow.getUint128Field(i); + pw = s - fRow.getScale(i); // pw is difference of scales, will be in [-18, 18] + + if (pw > 0) + t *= IDB_pow[pw]; + else if (pw < 0) + t /= IDB_pow[-pw]; + + break; + } + case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { @@ -552,6 +621,18 @@ void WindowFunctionType::getConstValue(ConstantColumn* cc, uint64_t& t t = cc->getUintVal(fRow, b); } +template<> +void WindowFunctionType::getConstValue(ConstantColumn* cc, int128_t& t, bool& b) +{ + t = cc->getDecimalVal(fRow, b).s128Value; +} + +template<> +void WindowFunctionType::getConstValue(ConstantColumn* cc, uint128_t& t, bool& b) +{ + t = cc->getDecimalVal(fRow, b).s128Value; +} + template<> void WindowFunctionType::getConstValue(ConstantColumn* cc, double& t, bool& b) { @@ -581,12 +662,16 @@ template void WindowFunctionType::implicit2T(uint64_t, uint64_t&, int) template void WindowFunctionType::implicit2T(uint64_t, float&, int); template void WindowFunctionType::implicit2T(uint64_t, double&, int); template void WindowFunctionType::implicit2T(uint64_t, long double&, int); +template void WindowFunctionType::implicit2T(uint64_t, int128_t&, int); +template void WindowFunctionType::implicit2T(uint64_t, uint128_t&, int); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, int64_t*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, uint64_t*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, float*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, double*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, long double*); +template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, int128_t*); +template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, uint128_t*); void* WindowFunctionType::getNullValueByType(int ct, int pos) { @@ -610,7 +695,8 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos) // static uint64_t char4Null = joblist::CHAR4NULL; // static uint64_t char8Null = joblist::CHAR8NULL; static string stringNull(""); - + static int128_t int128Null; // Set at runtime; + void* v = NULL; switch (ct) @@ -714,9 +800,18 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos) v = &intNull; break; - default: + case 8: v = &bigIntNull; break; + + case 16: + utils::setWideDecimalNullValue(int128Null); + v = &int128Null; + break; + + default: + break; + } break; diff --git a/utils/windowfunction/windowfunctiontype.h b/utils/windowfunction/windowfunctiontype.h index 9260cc4a2..1c7dea375 100644 --- a/utils/windowfunction/windowfunctiontype.h +++ b/utils/windowfunction/windowfunctiontype.h @@ -226,6 +226,10 @@ protected: { return fRow.getIntField(i); } + int128_t getInt128Value(uint64_t i) + { + return fRow.getInt128Field(i); + } double getDoubleValue(uint64_t i) { return fRow.getDoubleField(i); @@ -238,6 +242,10 @@ protected: { fRow.setIntField(v, i); } + void setInt128Value(int64_t i, int128_t v) + { + fRow.setInt128Field(v, i); + } void setDoubleValue(int64_t i, double v) { fRow.setDoubleField(v, i); diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 24407d513..e75ee794e 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -837,7 +837,18 @@ int WriteEngineWrapper::deleteRow(const TxnID& txnid, const vector inline void allocateValArray(void*& valArray, ColTupleList::size_type totalRow, ColType colType, int colWidth) { - valArray = calloc(totalRow, colWidth); + switch (colType) + { + case WriteEngine::WR_VARBINARY : // treat same as char for now + case WriteEngine::WR_CHAR: + case WriteEngine::WR_BLOB: + case WriteEngine::WR_TEXT: + valArray = (char*) calloc(sizeof(char), totalRow * MAX_COLUMN_BOUNDARY); + break; + default: + valArray = calloc(totalRow, colWidth); + break; + } // TODO MCOL-641 is commenting out the switch statement below correct? #if 0 switch (colType) From 1588ebe4393b9b96ed7822558e0a1b49f8343c3f Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Fri, 17 Jul 2020 14:56:59 +0000 Subject: [PATCH 52/78] MCOL-641 Clean up primitives code Add int128_t support into ByteStream Fixed UTs broken after collation patch --- datatypes/mcs_decimal.h | 2 + dbcon/execplan/calpontsystemcatalog.h | 1 - dbcon/joblist/batchprimitiveprocessor-jl.cpp | 54 ++++---- dbcon/joblist/batchprimitiveprocessor-jl.h | 2 +- dbcon/joblist/orderby-tests.cpp | 3 + dbcon/joblist/primitivemsg.h | 6 +- primitives/linux-port/CMakeLists.txt | 6 + primitives/linux-port/column.cpp | 123 ++++++++---------- .../{tdriver.cpp => pp-scan-unittest.cpp} | 0 primitives/linux-port/primitiveprocessor.h | 1 + .../primproc/batchprimitiveprocessor.cpp | 64 ++++----- primitives/primproc/batchprimitiveprocessor.h | 16 +-- primitives/primproc/columncommand.cpp | 45 ++++--- primitives/primproc/columncommand.h | 2 +- primitives/primproc/filtercommand.cpp | 10 +- primitives/primproc/filtercommand.h | 2 +- primitives/primproc/passthrucommand.cpp | 6 +- primitives/primproc/primproc.cpp | 7 +- tests/comparators-tests.cpp | 3 + tests/rowgroup-tests.cpp | 3 + utils/common/columnwidth.h | 1 + utils/messageqcpp/bytestream.cpp | 47 ++++--- utils/messageqcpp/bytestream.h | 18 ++- 23 files changed, 225 insertions(+), 197 deletions(-) rename primitives/linux-port/{tdriver.cpp => pp-scan-unittest.cpp} (100%) diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index fa2ed3849..1fac9485e 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -68,6 +68,8 @@ const uint64_t mcs_pow_10[20] = constexpr uint32_t maxPowOf10 = sizeof(mcs_pow_10)/sizeof(mcs_pow_10[0])-1; constexpr int128_t Decimal128Null = int128_t(0x8000000000000000LL) << 64; +constexpr int128_t Decimal128Empty = (int128_t(0x8000000000000000LL) << 64) + 1; + /** @brief The function to produce scale multiplier/divisor for diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index 3cd285f44..2efd7cef1 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -1023,7 +1023,6 @@ inline bool isUnsigned(const execplan::CalpontSystemCatalog::ColDataType type) case execplan::CalpontSystemCatalog::UMEDINT: case execplan::CalpontSystemCatalog::UINT: case execplan::CalpontSystemCatalog::UBIGINT: - // TODO MCOL-641 add decimal here return true; default: diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.cpp b/dbcon/joblist/batchprimitiveprocessor-jl.cpp index 234c8d4eb..2a874c4e2 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.cpp +++ b/dbcon/joblist/batchprimitiveprocessor-jl.cpp @@ -60,7 +60,7 @@ BatchPrimitiveProcessorJL::BatchPrimitiveProcessorJL(const ResourceManager* rm) baseRid(0), ridCount(0), needStrValues(false), - hasWideDecimalType(false), + wideColumnsWidths(0), filterCount(0), projectCount(0), needRidsAtDelivery(false), @@ -102,7 +102,7 @@ void BatchPrimitiveProcessorJL::addFilterStep(const pColScanStep& scan, vectorgetWidth())) - hasWideDecimalType = true; + wideColumnsWidths |= cc->getWidth(); idbassert(sessionID == scan.sessionId()); } @@ -117,9 +117,6 @@ void BatchPrimitiveProcessorJL::addFilterStep(const PseudoColStep& pcs) cc->setStepUuid(uuid); filterSteps.push_back(cc); filterCount++; - // TODO MCOL-641 How do we get to this execution path? - //if (utils::isWide(cc->getWidth())) - // hasWideDecimalType = true; idbassert(sessionID == pcs.sessionId()); } @@ -135,7 +132,7 @@ void BatchPrimitiveProcessorJL::addFilterStep(const pColStep& step) filterSteps.push_back(cc); filterCount++; if (utils::isWide(cc->getWidth())) - hasWideDecimalType = true; + wideColumnsWidths |= cc->getWidth(); idbassert(sessionID == step.sessionId()); } @@ -190,9 +187,6 @@ void BatchPrimitiveProcessorJL::addProjectStep(const PseudoColStep& step) colWidths.push_back(cc->getWidth()); tupleLength += cc->getWidth(); projectCount++; - // TODO MCOL-641 How do we get to this execution path? - //if (utils::isWide(cc->getWidth())) - // hasWideDecimalType = true; idbassert(sessionID == step.sessionId()); } @@ -210,7 +204,7 @@ void BatchPrimitiveProcessorJL::addProjectStep(const pColStep& step) tupleLength += cc->getWidth(); projectCount++; if (utils::isWide(cc->getWidth())) - hasWideDecimalType = true; + wideColumnsWidths |= cc->getWidth(); idbassert(sessionID == step.sessionId()); } @@ -229,7 +223,7 @@ void BatchPrimitiveProcessorJL::addProjectStep(const PassThruStep& step) projectCount++; if (utils::isWide(cc->getWidth())) - hasWideDecimalType = true; + wideColumnsWidths |= cc->getWidth(); if (filterCount == 0 && !sendRowGroups) sendValues = true; @@ -739,7 +733,7 @@ void BatchPrimitiveProcessorJL::deserializeAggregateResult(ByteStream* in, void BatchPrimitiveProcessorJL::getRowGroupData(ByteStream& in, vector* out, bool* validCPData, uint64_t* lbid, __int128* min, __int128* max, uint32_t* cachedIO, uint32_t* physIO, uint32_t* touchedBlocks, bool* countThis, - uint32_t threadID, bool* hasBinaryColumn, const execplan::CalpontSystemCatalog::ColType& colType) const + uint32_t threadID, bool* hasWideColumn, const execplan::CalpontSystemCatalog::ColType& colType) const { uint64_t tmp64; unsigned __int128 tmp128; @@ -775,21 +769,32 @@ void BatchPrimitiveProcessorJL::getRowGroupData(ByteStream& in, vector* { in >> *lbid; in >> tmp8; - *hasBinaryColumn = (tmp8 > 8); - if (*hasBinaryColumn) + *hasWideColumn = (tmp8 > utils::MAXLEGACYWIDTH); + if (UNLIKELY(*hasWideColumn)) { - idbassert(colType.colWidth > 8); - in >> tmp128; - *min = tmp128; - in >> tmp128; - *max = tmp128; + idbassert(colType.colWidth > utils::MAXLEGACYWIDTH); + if (LIKELY(datatypes::Decimal::isWideDecimalType(colType))) + { + in >> tmp128; + *min = tmp128; + in >> tmp128; + *max = tmp128; + } + else + { + std::ostringstream oss; + oss << __func__ << " WARNING!!! Not implemented for the data type "; + oss << colType.colDataType << std::endl; + std::cout << oss.str(); + idbassert(false); + } } else { in >> tmp64; - *min = static_cast<__int128>(tmp64); + *min = static_cast(tmp64); in >> tmp64; - *max = static_cast<__int128>(tmp64); + *max = static_cast(tmp64); } } else @@ -1010,11 +1015,14 @@ void BatchPrimitiveProcessorJL::createBPP(ByteStream& bs) const if (sendTupleJoinRowGroupData) flags |= JOIN_ROWGROUP_DATA; - if (hasWideDecimalType) - flags |= HAS_WIDE_DECIMAL; + if (wideColumnsWidths) + flags |= HAS_WIDE_COLUMNS; bs << flags; + if (wideColumnsWidths) + bs << wideColumnsWidths; + bs << bop; bs << (uint8_t) (forHJ ? 1 : 0); diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.h b/dbcon/joblist/batchprimitiveprocessor-jl.h index 1cd07d970..0a8f1f841 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.h +++ b/dbcon/joblist/batchprimitiveprocessor-jl.h @@ -288,7 +288,7 @@ private: uint16_t ridCount; bool needStrValues; - bool hasWideDecimalType; + uint16_t wideColumnsWidths; std::vector filterSteps; std::vector projectSteps; diff --git a/dbcon/joblist/orderby-tests.cpp b/dbcon/joblist/orderby-tests.cpp index fa98f0772..792f65596 100644 --- a/dbcon/joblist/orderby-tests.cpp +++ b/dbcon/joblist/orderby-tests.cpp @@ -174,6 +174,7 @@ private: // create two columns RG. 1st is the sorting key, second is the data column std::vector offsets, roids, tkeys, cscale, cprecision; + std::vector charSetNumVec; std::vector types; offsets.push_back(2); offsets.push_back(10); offsets.push_back(18); roids.push_back(oid); roids.push_back(oid); @@ -182,11 +183,13 @@ private: types.push_back(execplan::CalpontSystemCatalog::UBIGINT); cscale.push_back(0); cscale.push_back(0); cprecision.push_back(20); cprecision.push_back(20); + charSetNumVec.push_back(8); charSetNumVec.push_back(8); rowgroup::RowGroup inRG(2, //column count offsets, //oldOffset roids, // column oids tkeys, //keys types, // types + charSetNumVec, // charset numbers cscale, //scale cprecision, // precision 20, // sTableThreshold diff --git a/dbcon/joblist/primitivemsg.h b/dbcon/joblist/primitivemsg.h index f82a5eeb7..7660f38ef 100644 --- a/dbcon/joblist/primitivemsg.h +++ b/dbcon/joblist/primitivemsg.h @@ -198,7 +198,7 @@ const uint16_t HAS_JOINER = 0x10; //16; const uint16_t SEND_RIDS_AT_DELIVERY = 0x20; //32; const uint16_t HAS_ROWGROUP = 0x40; //64; const uint16_t JOIN_ROWGROUP_DATA = 0x80; //128 -const uint16_t HAS_WIDE_DECIMAL = 0x100; //256; +const uint16_t HAS_WIDE_COLUMNS = 0x100; //256; //TODO: put this in a namespace to stop global ns pollution enum PrimFlags @@ -705,8 +705,8 @@ struct NewColResultHeader uint16_t NVALS; uint16_t ValidMinMax; // 1 if Min/Max are valid, otherwise 0 uint32_t OutputType; - __int128 Min; // Minimum value in this block for signed data types - __int128 Max; // Maximum value in this block for signed data types + int128_t Min; // Minimum value in this block for signed data types + int128_t Max; // Maximum value in this block for signed data types uint32_t CacheIO; // I/O count from buffer cache uint32_t PhysicalIO; // Physical I/O count from disk // if OutputType was OT_DATAVALUE, what follows is DataType[NVALS] diff --git a/primitives/linux-port/CMakeLists.txt b/primitives/linux-port/CMakeLists.txt index 08ca261f8..569169d12 100644 --- a/primitives/linux-port/CMakeLists.txt +++ b/primitives/linux-port/CMakeLists.txt @@ -16,3 +16,9 @@ target_link_libraries(processor ${NETSNMP_LIBRARIES}) INSTALL (TARGETS processor DESTINATION ${ENGINE_LIBDIR}) +#if (WITH_PP_SCAN_UT) +# add_executable(pp_scan_unittest pp-scan-unittest.cpp) +# target_link_libraries(pp_scan_unittest ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit) +# install(TARGETS pp_scan_unittest DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-engine) +#endif() + diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 446d81e05..11bbe07ce 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -1,4 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. + Copyright (C) 2016-2020 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -40,7 +41,6 @@ using namespace boost; #include "stats.h" #include "primproc.h" #include "dataconvert.h" -#include "widedecimalutils.h" #include "mcs_decimal.h" using namespace logging; using namespace dbbc; @@ -278,18 +278,17 @@ inline bool isEmptyVal(uint8_t type, const uint8_t* val8); template<> inline bool isEmptyVal<32>(uint8_t type, const uint8_t* ival) // For BINARY { - const uint64_t* val = reinterpret_cast(ival); - return ((val[0] == joblist::BINARYEMPTYVALUELOW) - && (val[1] == joblist::BINARYNULLVALUELOW) - && (val[2] == joblist::BINARYNULLVALUELOW) - && (val[3] == joblist::BINARYEMPTYVALUEHIGH)); + std::cout << __func__ << " WARNING!!! Not implemented for 32 byte data types." << std::endl; + return false; } template<> inline bool isEmptyVal<16>(uint8_t type, const uint8_t* ival) // For BINARY { const int128_t* val = reinterpret_cast(ival); - return utils::isWideDecimalEmptyValue (*val); + // Wide-DECIMAL supplies a universal NULL/EMPTY magics for all 16 byte + // data types. + return *val == datatypes::Decimal128Empty; } template<> @@ -413,22 +412,22 @@ inline bool isEmptyVal<1>(uint8_t type, const uint8_t* ival) template inline bool isNullVal(uint8_t type, const uint8_t* val8); -// WIP This method only works for wide DECIMAL so far. template<> -inline bool isNullVal<16>(uint8_t type, const uint8_t* ival) // For BINARY +inline bool isNullVal<16>(uint8_t type, const uint8_t* ival) { const int128_t* val = reinterpret_cast(ival); - return utils::isWideDecimalNullValue (*val); + // Wide-DECIMAL supplies a universal NULL/EMPTY magics for all 16 byte + // data types. + return *val == datatypes::Decimal128Null; } template<> -inline bool isNullVal<32>(uint8_t type, const uint8_t* ival) // For BINARY +inline bool isNullVal<32>(uint8_t type, const uint8_t* ival) { - const uint64_t* val = reinterpret_cast(ival); - return ((val[0] == joblist::BINARYNULLVALUELOW) - && (val[1] == joblist::BINARYNULLVALUELOW) - && (val[2] == joblist::BINARYNULLVALUELOW) - && (val[3] == joblist::BINARYNULLVALUEHIGH)); + + std::cout << __func__ << " WARNING!!! Not implemented for 32 byte data types." +<< std::endl; + return false; } template<> @@ -558,12 +557,9 @@ inline bool isNullVal(uint32_t length, uint8_t type, const uint8_t* val8) { switch (length) { - case 32: - return isNullVal<32>(type, val8); - case 16: return isNullVal<16>(type, val8); - + case 8: return isNullVal<8>(type, val8); @@ -576,7 +572,8 @@ inline bool isNullVal(uint32_t length, uint8_t type, const uint8_t* val8) case 1: return isNullVal<1>(type, val8); }; - + std::cout << __func__ << " WARNING!!! Not implemented for " << length << " bytes data types." << std::endl; + return false; } @@ -619,7 +616,7 @@ inline bool isMinMaxValid(const NewColRequestHeader* in) case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - return (in->DataSize <= 16 ); + return (in->DataSize <= datatypes::MAXDECIMALWIDTH); default: return false; @@ -703,7 +700,7 @@ inline bool colCompare(int128_t val1, int128_t val2, uint8_t COP, uint8_t rf, in inline bool colCompareUnsigned(uint64_t val1, uint64_t val2, uint8_t COP, uint8_t rf, int type, uint8_t width, const idb_regex_t& regex, bool isNull = false) { -// cout << "comparing unsigned" << hex << val1 << " to " << val2 << endl; + // cout << "comparing unsigned" << hex << val1 << " to " << val2 << endl; if (COMPARE_NIL == COP) return false; @@ -758,8 +755,7 @@ inline void store(const NewColRequestHeader* in, switch (in->DataSize) { case 32: - ptr2 += (rid << 5); - memcpy(ptr1, ptr2, 32); + std::cout << __func__ << " WARNING!!! Not implemented for 32 byte data types." << std::endl; break; case 16: @@ -768,6 +764,7 @@ inline void store(const NewColRequestHeader* in, break; default: + std::cout << __func__ << " WARNING!!! unspecified column width." << std::endl; case 8: ptr2 += (rid << 3); memcpy(ptr1, ptr2, 8); @@ -893,8 +890,8 @@ inline uint8_t* nextBinColValue(int type, { (*index)++; } - - + + if (static_cast(*index) >= itemsPerBlk) { *done = true; @@ -904,14 +901,12 @@ inline uint8_t* nextBinColValue(int type, } else { - //FIXME: not complete nor tested . How make execution flow pass here - // whe is ridArray not NULL ? fidn by id? how? while (*index < NVALS && isEmptyVal(type, &val8[ridArray[*index] * W])) { (*index)++; } - + if (*index >= NVALS) { *done = true; @@ -920,20 +915,22 @@ inline uint8_t* nextBinColValue(int type, *rid = ridArray[(*index)++]; } - *isNull = isNullVal(type, &val8[*rid * W]); - *isEmpty = isEmptyVal(type, &val8[*rid * W]); - //cout << "nextUnsignedColValue index " << *index << " rowid " << *rid << endl; + uint32_t curValueOffset = *rid * W; + + *isNull = isNullVal(type, &val8[curValueOffset]); + *isEmpty = isEmptyVal(type, &val8[curValueOffset]); + //cout << "nextColBinValue " << *index << " rowid " << *rid << endl; // at this point, nextRid is the index to return, and index is... // if RIDs are not specified, nextRid + 1, // if RIDs are specified, it's the next index in the rid array. - return &val8[*rid * W]; + return &val8[curValueOffset]; #ifdef PRIM_DEBUG throw logic_error("PrimitiveProcessor::nextColBinValue() bad width"); #endif return NULL; } -} + template inline int64_t nextColValue(int type, @@ -1549,8 +1546,11 @@ inline void p_Col_ridArray(NewColRequestHeader* in, #endif } -// for BINARY -template +// There are number of hardcoded type-dependant objects +// that effectively makes this template int128-based only. +// Use type based template method for Min,Max values. +// prestored_set_128 must be a template with a type arg. +template inline void p_Col_bin_ridArray(NewColRequestHeader* in, NewColResultHeader* out, unsigned outSize, @@ -1563,11 +1563,6 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, idb_regex_t placeholderRegex; placeholderRegex.used = false; - //FIXME: pCol is setting it to 8192 cause logicalBlockMode is true - /*if(itemsPerBlk == BLOCK_SIZE){ - itemsPerBlk = BLOCK_SIZE/W; - }*/ - if (in->NVALS > 0) ridArray = reinterpret_cast(&in8[sizeof(NewColRequestHeader) + (in->NOPS * filterSize)]); @@ -1590,6 +1585,7 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, if (out->ValidMinMax) { + // Assume that isUnsigned returns true for 8-bytes DTs only if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType)) { out->Min = -1; @@ -1597,8 +1593,8 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, } else { - utils::int128Max(out->Min); - utils::int128Min(out->Max); + out->Min = datatypes::Decimal::maxInt128; + out->Max = datatypes::Decimal::minInt128; } } else @@ -1625,7 +1621,7 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, scoped_array std_regex; idb_regex_t* regex = NULL; -// no pre-parsed column filter is set, parse the filter in the message + // no pre-parsed column filter is set, parse the filter in the message if (parsedColumnFilter.get() == NULL) { std_regex.reset(new idb_regex_t[in->NOPS]); regex = &(std_regex[0]); @@ -1643,7 +1639,6 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, regex[argIndex].used = false; } } - // WIP MCOL-641 // we have a pre-parsed filter, and it's in the form of op and value arrays else if (parsedColumnFilter->columnFilterMode == TWO_ARRAYS) { @@ -1658,11 +1653,11 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, bval = (binWtype*)nextBinColValue(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull, &isEmpty, &rid, in->OutputType, reinterpret_cast(block), itemsPerBlk); - int128_t val; + T val; while (!done) { - val = *reinterpret_cast(bval); + val = *reinterpret_cast(bval); if (cops == NULL) // implies parsedColumnFilter && columnFilterMode == SET { @@ -1695,8 +1690,7 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, { for (argIndex = 0; argIndex < in->NOPS; argIndex++) { - // WIP MCOL-641 - int128_t filterVal = *reinterpret_cast(argVals[argIndex]); + T filterVal = *reinterpret_cast(argVals[argIndex]); cmp = colCompare(val, filterVal, cops[argIndex], rfs[argIndex], in->DataType, W, isNull); @@ -1731,20 +1725,13 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, { if (in->DataType == CalpontSystemCatalog::CHAR || in->DataType == CalpontSystemCatalog::VARCHAR) { + // !!! colCompare is overloaded with int128_t only yet. if (colCompare(out->Min, val, COMPARE_GT, false, in->DataType, W, placeholderRegex)) out->Min = val; if (colCompare(out->Max, val, COMPARE_LT, false, in->DataType, W, placeholderRegex)) out->Max = val; } - else if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType)) - { - if (static_cast(out->Min) > static_cast(val)) - out->Min = val; - - if (static_cast(out->Max) < static_cast(val)) - out->Max = val; - } else { if (out->Min > val) @@ -1767,7 +1754,8 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, #else fStatsPtr->markEvent(in->LBID, pthread_self(), in->hdr.SessionID, 'K'); #endif - +} + } //namespace anon namespace primitives @@ -1818,14 +1806,6 @@ void PrimitiveProcessor::p_Col(NewColRequestHeader* in, NewColResultHeader* out, switch (in->DataSize) { - case 32: - p_Col_bin_ridArray<32>(in, out, outSize, written, block, fStatsPtr, itemsPerBlk, parsedColumnFilter); - break; - - case 16: - p_Col_bin_ridArray<16>(in, out, outSize, written, block, fStatsPtr, itemsPerBlk, parsedColumnFilter); - break; - case 8: p_Col_ridArray<8>(in, out, outSize, written, block, fStatsPtr, itemsPerBlk, parsedColumnFilter); break; @@ -1842,6 +1822,13 @@ void PrimitiveProcessor::p_Col(NewColRequestHeader* in, NewColResultHeader* out, p_Col_ridArray<1>(in, out, outSize, written, block, fStatsPtr, itemsPerBlk, parsedColumnFilter); break; + case 16: + p_Col_bin_ridArray<16, int128_t>(in, out, outSize, written, block, fStatsPtr, itemsPerBlk, parsedColumnFilter); + break; + + case 32: + std::cout << __func__ << " WARNING!!! Not implemented for 32 byte data types." << std::endl; + default: idbassert(0); break; @@ -1932,8 +1919,6 @@ boost::shared_ptr parseColumnFilter case 8: ret->prestored_argVals[argIndex] = *reinterpret_cast(args->val); break; - case 16: - cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; } } else diff --git a/primitives/linux-port/tdriver.cpp b/primitives/linux-port/pp-scan-unittest.cpp similarity index 100% rename from primitives/linux-port/tdriver.cpp rename to primitives/linux-port/pp-scan-unittest.cpp diff --git a/primitives/linux-port/primitiveprocessor.h b/primitives/linux-port/primitiveprocessor.h index af0ea1909..4c674f95e 100644 --- a/primitives/linux-port/primitiveprocessor.h +++ b/primitives/linux-port/primitiveprocessor.h @@ -329,6 +329,7 @@ private: int dict_OffsetIndex, currentOffsetIndex; // used by p_dictionary int fDebugLevel; dbbc::Stats* fStatsPtr; // pointer for pmstats + // To be removed b/c always true bool logicalBlockMode; boost::shared_ptr parsedColumnFilter; diff --git a/primitives/primproc/batchprimitiveprocessor.cpp b/primitives/primproc/batchprimitiveprocessor.cpp index 999f95518..869c9758f 100644 --- a/primitives/primproc/batchprimitiveprocessor.cpp +++ b/primitives/primproc/batchprimitiveprocessor.cpp @@ -1,5 +1,5 @@ /* Copyright (C) 2014 InfiniDB, Inc. - Copyright (C) 2016, 2017 MariaDB Corporation + Copyright (C) 2016-2020 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -101,7 +101,7 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor() : baseRid(0), ridCount(0), needStrValues(false), - hasWideDecimalType(false), + wideColumnsWidths(0), filterCount(0), projectCount(0), sendRidsAtDelivery(false), @@ -113,7 +113,7 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor() : minVal(MAX64), maxVal(MIN64), lbidForCP(0), - hasBinaryColumn(false), + hasWideColumnOut(false), busyLoaderCount(0), physIO(0), cachedIO(0), @@ -147,7 +147,7 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor(ByteStream& b, double prefetch, baseRid(0), ridCount(0), needStrValues(false), - hasWideDecimalType(false), + wideColumnsWidths(0), filterCount(0), projectCount(0), sendRidsAtDelivery(false), @@ -159,7 +159,7 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor(ByteStream& b, double prefetch, minVal(MAX64), maxVal(MIN64), lbidForCP(0), - hasBinaryColumn(false), + hasWideColumnOut(false), busyLoaderCount(0), physIO(0), cachedIO(0), @@ -242,13 +242,16 @@ void BatchPrimitiveProcessor::initBPP(ByteStream& bs) doJoin = tmp16 & HAS_JOINER; hasRowGroup = tmp16 & HAS_ROWGROUP; getTupleJoinRowGroupData = tmp16 & JOIN_ROWGROUP_DATA; - hasWideDecimalType = tmp16 & HAS_WIDE_DECIMAL; + bool hasWideColumnsIn = tmp16 & HAS_WIDE_COLUMNS; // This used to signify that there was input row data from previous jobsteps, and // it never quite worked right. No need to fix it or update it; all BPP's have started // with a scan for years. Took it out. assert(!hasRowGroup); + if (hasWideColumnsIn) + bs >> wideColumnsWidths; + bs >> bop; bs >> forHJ; @@ -581,7 +584,6 @@ void BatchPrimitiveProcessor::addToJoiner(ByteStream& bs) /* skip the header */ bs.advance(sizeof(ISMPacketHeader) + 3 * sizeof(uint32_t)); -// TODO MCOL-641 bs >> count; bs >> startPos; @@ -1024,7 +1026,7 @@ void BatchPrimitiveProcessor::initProcessor() fFiltRidCount[i] = 0; fFiltCmdRids[i].reset(new uint16_t[LOGICAL_BLOCK_RIDS]); fFiltCmdValues[i].reset(new int64_t[LOGICAL_BLOCK_RIDS]); - if (hasWideDecimalType) + if (wideColumnsWidths | datatypes::MAXDECIMALWIDTH) fFiltCmdBinaryValues[i].reset(new int128_t[LOGICAL_BLOCK_RIDS]); if (filtOnString) fFiltStrValues[i].reset(new string[LOGICAL_BLOCK_RIDS]); @@ -1092,15 +1094,15 @@ void BatchPrimitiveProcessor::initProcessor() fAggregator->setInputOutput(fe2 ? fe2Output : outputRG, &fAggregateRG); } - if (!hasBinaryColumn) + if (LIKELY(!hasWideColumnOut)) { minVal = MAX64; maxVal = MIN64; } else { - utils::int128Min(bigMaxVal); - utils::int128Max(bigMinVal); + max128Val = datatypes::Decimal::minInt128; + min128Val = datatypes::Decimal::maxInt128; } // @bug 1269, initialize data used by execute() for async loading blocks @@ -1546,11 +1548,6 @@ void BatchPrimitiveProcessor::execute() projectSteps[j]->projectIntoRowGroup(fe1Input, projectForFE1[j]); for (j = 0; j < ridCount; j++, fe1In.nextRow()) - // TODO MCOL-641 - // WHERE clause on a numeric and a non-numeric column - // leads to this execution path: - // SELECT a, b from t1 where a!=b - // Here, a is e.g., decimal(38), b is varchar(15) if (fe1->evaluate(&fe1In)) { applyMapping(fe1ToProjection, fe1In, &fe1Out); @@ -1993,15 +1990,16 @@ void BatchPrimitiveProcessor::writeProjectionPreamble() { *serialized << (uint8_t) 1; *serialized << lbidForCP; - if (hasBinaryColumn) + if (UNLIKELY(hasWideColumnOut)) { - *serialized << (uint8_t) 16; // width of min/max value - *serialized << (unsigned __int128) bigMinVal; - *serialized << (unsigned __int128) bigMaxVal; + // PSA width + *serialized << (uint8_t) wideColumnWidthOut; + *serialized << min128Val; + *serialized << max128Val; } else { - *serialized << (uint8_t) 8; // width of min/max value + *serialized << (uint8_t) utils::MAXLEGACYWIDTH; // width of min/max value *serialized << (uint64_t) minVal; *serialized << (uint64_t) maxVal; } @@ -2093,15 +2091,19 @@ void BatchPrimitiveProcessor::makeResponse() { *serialized << (uint8_t) 1; *serialized << lbidForCP; - if (hasBinaryColumn) + + if (UNLIKELY(hasWideColumnOut)) { - *serialized << (uint8_t) 16; // width of min/max value - *serialized << (unsigned __int128) bigMinVal; - *serialized << (unsigned __int128) bigMaxVal; + // PSA width + // Remove the assert for >16 bytes DTs. + assert(wideColumnWidthOut == datatypes::MAXDECIMALWIDTH); + *serialized << (uint8_t) wideColumnWidthOut; + *serialized << min128Val; + *serialized << max128Val; } else { - *serialized << (uint8_t) 8; // width of min/max value + *serialized << (uint8_t) utils::MAXLEGACYWIDTH; // width of min/max value *serialized << (uint64_t) minVal; *serialized << (uint64_t) maxVal; } @@ -2207,16 +2209,18 @@ int BatchPrimitiveProcessor::operator()() } allocLargeBuffers(); - if (!hasBinaryColumn) + + if (LIKELY(!hasWideColumnOut)) { minVal = MAX64; maxVal = MIN64; } else { - utils::int128Min(bigMaxVal); - utils::int128Max(bigMinVal); + max128Val = datatypes::Decimal::minInt128; + min128Val = datatypes::Decimal::maxInt128; } + validCPData = false; #ifdef PRIMPROC_STOPWATCH stopwatch->start("BPP() execute"); @@ -2351,7 +2355,7 @@ SBPP BatchPrimitiveProcessor::duplicate() bpp->stepID = stepID; bpp->uniqueID = uniqueID; bpp->needStrValues = needStrValues; - bpp->hasWideDecimalType = hasWideDecimalType; + bpp->wideColumnsWidths = wideColumnsWidths; bpp->gotAbsRids = gotAbsRids; bpp->gotValues = gotValues; bpp->LBIDTrace = LBIDTrace; diff --git a/primitives/primproc/batchprimitiveprocessor.h b/primitives/primproc/batchprimitiveprocessor.h index 0ffec0044..fd1b3992e 100644 --- a/primitives/primproc/batchprimitiveprocessor.h +++ b/primitives/primproc/batchprimitiveprocessor.h @@ -52,6 +52,7 @@ #include "rowaggregation.h" #include "funcexpwrapper.h" #include "bppsendthread.h" +#include "columnwidth.h" namespace primitiveprocessor { @@ -205,16 +206,15 @@ private: uint16_t relRids[LOGICAL_BLOCK_RIDS]; int64_t values[LOGICAL_BLOCK_RIDS]; - int128_t binaryValues[LOGICAL_BLOCK_RIDS]; + int128_t wide128Values[LOGICAL_BLOCK_RIDS]; boost::scoped_array absRids; boost::scoped_array strValues; uint16_t ridCount; bool needStrValues; - bool hasWideDecimalType; + uint16_t wideColumnsWidths; /* Common space for primitive data */ - static const uint32_t BUFFER_SIZE = 131072; - uint8_t blockData[BLOCK_SIZE * 16]; + uint8_t blockData[BLOCK_SIZE * utils::MAXCOLUMNWIDTH]; boost::scoped_array outputMsg; uint32_t outMsgSize; @@ -233,18 +233,18 @@ private: // CP data from a scanned column union { - __int128 bigMinVal; + int128_t min128Val; int64_t minVal; }; union { - __int128 bigMaxVal; + int128_t max128Val; int64_t maxVal; }; uint64_t lbidForCP; - // MCOL-641 - bool hasBinaryColumn; + bool hasWideColumnOut; + uint8_t wideColumnWidthOut; // IO counters boost::mutex counterLock; uint32_t busyLoaderCount; diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index 6ab2929d6..d1b122466 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -40,7 +40,6 @@ using namespace std; #include "primitiveserver.h" #include "primproc.h" #include "stats.h" -#include "widedecimalutils.h" using namespace messageqcpp; using namespace rowgroup; @@ -94,17 +93,17 @@ void ColumnCommand::execute() if (fFilterFeeder == LEFT_FEEDER) { values = bpp->fFiltCmdValues[0].get(); - binaryValues = bpp->fFiltCmdBinaryValues[0].get(); + wide128Values = bpp->fFiltCmdBinaryValues[0].get(); } else if (fFilterFeeder == RIGHT_FEEDER) { values = bpp->fFiltCmdValues[1].get(); - binaryValues = bpp->fFiltCmdBinaryValues[1].get(); + wide128Values = bpp->fFiltCmdBinaryValues[1].get(); } else { values = bpp->values; - binaryValues = bpp->binaryValues; + wide128Values = bpp->wide128Values; } _execute(); @@ -153,9 +152,10 @@ void ColumnCommand::loadData() bool lastBlockReached = false; oidLastLbid = getLastLbid(); uint32_t blocksToLoad = 0; - // WIP MCOL-641 - BRM::LBID_t* lbids = (BRM::LBID_t*) alloca(16 * sizeof(BRM::LBID_t)); - uint8_t** blockPtrs = (uint8_t**) alloca(16 * sizeof(uint8_t*)); + // The number of elements allocated equals to the number of + // iteratations of the first loop here. + BRM::LBID_t* lbids = (BRM::LBID_t*) alloca(colType.colWidth * sizeof(BRM::LBID_t)); + uint8_t** blockPtrs = (uint8_t**) alloca(colType.colWidth * sizeof(uint8_t*)); int i; @@ -282,11 +282,24 @@ void ColumnCommand::issuePrimitive() //if (wasVersioned && outMsg->ValidMinMax) // cout << "CC: versioning overriding min max data\n"; bpp->lbidForCP = lbid; - if (primMsg->DataSize > 8) + if (UNLIKELY(utils::isWide(colType.colWidth))) { - bpp->hasBinaryColumn = true; - bpp->bigMaxVal = outMsg->Max; - bpp->bigMinVal = outMsg->Min; + if (datatypes::Decimal::isWideDecimalType(colType)) + { + bpp->hasWideColumnOut = true; + // colWidth is int32 and wideColumnWidthOut's + // value is expected to be at most uint8. + bpp->wideColumnWidthOut = colType.colWidth; + bpp->max128Val = outMsg->Max; + bpp->min128Val = outMsg->Min; + } + else + { + ostringstream os; + os << " WARNING!!! Not implemented for "; + os << primMsg->DataSize << " column."; + throw PrimitiveColumnProjectResultExcept(os.str()); + } } else { @@ -315,7 +328,7 @@ void ColumnCommand::process_OT_BOTH() bpp->relRids[i] = *((uint16_t*) &bpp->outputMsg[pos]); pos += 2; - binaryValues[i] = *((int128_t*) &bpp->outputMsg[pos]); + wide128Values[i] = *((int128_t*) &bpp->outputMsg[pos]); pos += 16; } @@ -394,8 +407,7 @@ void ColumnCommand::process_OT_DATAVALUE() { case 16: { - memcpy(binaryValues, outMsg + 1, outMsg->NVALS << 4); - cout << " CC: first value is " << values[0] << endl; + memcpy(wide128Values, outMsg + 1, outMsg->NVALS << 4); break; } @@ -586,6 +598,8 @@ void ColumnCommand::prep(int8_t outputType, bool absRids) + // JFYI This switch results are used by index scan code that is unused + // as of 1.5 switch (colType.colWidth) { case 1: @@ -608,8 +622,6 @@ void ColumnCommand::prep(int8_t outputType, bool absRids) mask = 0x01; break; case 16: - // WIP MCOL-641 - cout << __FILE__<< ":" <<__LINE__ << " Set shift and mask for 16 Bytes"<< endl; shift = 1; mask = 0x01; break; @@ -809,7 +821,6 @@ void ColumnCommand::projectResultRG(RowGroup& rg, uint32_t pos) } case 16: { - cout << __FILE__<< ":" <<__LINE__ << " ColumnCommand::projectResultRG " << endl; for (i = 0; i < outMsg->NVALS; ++i, msg8 += gapSize) { r.setBinaryField_offset((int128_t*)msg8, colType.colWidth, offset); diff --git a/primitives/primproc/columncommand.h b/primitives/primproc/columncommand.h index d868ce2f9..eb51db842 100644 --- a/primitives/primproc/columncommand.h +++ b/primitives/primproc/columncommand.h @@ -147,7 +147,7 @@ private: uint16_t filterCount; bool makeAbsRids; int64_t* values; // this is usually bpp->values; RTSCommand needs to use a different container - int128_t* binaryValues; + int128_t* wide128Values; uint8_t mask, shift; // vars for the selective block loader diff --git a/primitives/primproc/filtercommand.cpp b/primitives/primproc/filtercommand.cpp index e90239d05..5bf5d0b05 100644 --- a/primitives/primproc/filtercommand.cpp +++ b/primitives/primproc/filtercommand.cpp @@ -176,7 +176,7 @@ Command* FilterCommand::makeFilterCommand(ByteStream& bs, vector& cmds FilterCommand::FilterCommand() : Command(FILTER_COMMAND), fBOP(0), - hasWideDecimalType(false) + hasWideColumns(false) { } @@ -251,7 +251,7 @@ void FilterCommand::setColTypes(const execplan::CalpontSystemCatalog::ColType& l rightColType = right; if (datatypes::Decimal::isWideDecimalType(left) || datatypes::Decimal::isWideDecimalType(right)) - hasWideDecimalType = true; + hasWideColumns = true; } @@ -262,7 +262,7 @@ void FilterCommand::doFilter() bool (FilterCommand::*compareFunc)(uint64_t, uint64_t); - if (hasWideDecimalType) + if (hasWideColumns) compareFunc = &FilterCommand::binaryCompare; else compareFunc = &FilterCommand::compare; @@ -280,10 +280,8 @@ void FilterCommand::doFilter() if ((this->*compareFunc)(i, j) == true) { bpp->relRids[bpp->ridCount] = bpp->fFiltCmdRids[0][i]; - // WIP MCOL-641 How is bpp->(binary)values used given that - // we are setting the relRids? if (datatypes::Decimal::isWideDecimalType(leftColType)) - bpp->binaryValues[bpp->ridCount] = bpp->fFiltCmdBinaryValues[0][i]; + bpp->wide128Values[bpp->ridCount] = bpp->fFiltCmdBinaryValues[0][i]; else bpp->values[bpp->ridCount] = bpp->fFiltCmdValues[0][i]; bpp->ridMap |= 1 << (bpp->relRids[bpp->ridCount] >> 10); diff --git a/primitives/primproc/filtercommand.h b/primitives/primproc/filtercommand.h index 4d91c925d..263523bef 100644 --- a/primitives/primproc/filtercommand.h +++ b/primitives/primproc/filtercommand.h @@ -82,7 +82,7 @@ protected: // binary operator uint8_t fBOP; - bool hasWideDecimalType; + bool hasWideColumns; // column type for null check execplan::CalpontSystemCatalog::ColType leftColType; diff --git a/primitives/primproc/passthrucommand.cpp b/primitives/primproc/passthrucommand.cpp index 707b11b62..7f345d6d8 100644 --- a/primitives/primproc/passthrucommand.cpp +++ b/primitives/primproc/passthrucommand.cpp @@ -78,8 +78,7 @@ void PassThruCommand::project() switch (colWidth) { case 16: - cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; - bpp->serialized->append((uint8_t*) bpp->binaryValues, bpp->ridCount << 4); + bpp->serialized->append((uint8_t*) bpp->wide128Values, bpp->ridCount << 4); break; case 8: @@ -156,10 +155,9 @@ void PassThruCommand::projectIntoRowGroup(RowGroup& rg, uint32_t col) break; case 16: - cout << __FILE__ << ":" << __LINE__ << " PassThruCommand::projectIntoRowGroup" << " Addition for 16 Bytes" << endl; for (i = 0; i < bpp->ridCount; i++) { - r.setBinaryField_offset(&bpp->binaryValues[i], 16, offset); + r.setBinaryField_offset(&bpp->wide128Values[i], 16, offset); r.nextRow(rowSize); } } diff --git a/primitives/primproc/primproc.cpp b/primitives/primproc/primproc.cpp index a898e8d97..883503bf7 100644 --- a/primitives/primproc/primproc.cpp +++ b/primitives/primproc/primproc.cpp @@ -512,10 +512,9 @@ int ServicePrimProc::Child() // set to smallest extent size // do not allow to read beyond the end of an extent const int MaxReadAheadSz = (extentRows) / BLOCK_SIZE; - //defaultBufferSize = 512 * 1024; // @bug 2627 - changed default dict buffer from 256K to 512K, allows for cols w/ length of 61. - // WIP MCOL-641 Check with Patrick on this. Changed it from 100*1024 to 128*1024 - // to match with BatchPrimitiveProcessor::BUFFER_SIZE - defaultBufferSize = 128 * 1024; // 1/17/12 - made the dict buffer dynamic, max size for a numeric col is 80k + ovrhd + // Max size for a primitive message column buffer. Must be big enough + // to accomodate 8192 values of the widest non-dict column + 8192 RIDs + ohead. + defaultBufferSize = 150 * 1024; // This parm controls whether we rotate through the output sockets diff --git a/tests/comparators-tests.cpp b/tests/comparators-tests.cpp index 83627fe56..4d6f8aab2 100644 --- a/tests/comparators-tests.cpp +++ b/tests/comparators-tests.cpp @@ -141,6 +141,7 @@ private: std::cout << std::endl << "------------------------------------------------------------" << std::endl; uint32_t oid =3001; std::vector offsets, roids, tkeys, cscale, cprecision; + std::vector charSetNumVec; std::vector types; offsets.push_back(2); offsets.push_back(2+width); roids.push_back(oid); @@ -148,11 +149,13 @@ private: types.push_back(cscDt); cscale.push_back(0); cprecision.push_back(precision); + charSetNumVec.push_back(8); rowgroup::RowGroup inRG(1, //column count offsets, //oldOffset roids, // column oids tkeys, //keys types, // types + charSetNumVec, // charset numbers cscale, //scale cprecision, // precision 20, // sTableThreshold diff --git a/tests/rowgroup-tests.cpp b/tests/rowgroup-tests.cpp index 8beec7ace..d53356d3a 100644 --- a/tests/rowgroup-tests.cpp +++ b/tests/rowgroup-tests.cpp @@ -43,6 +43,7 @@ protected: std::vector precisionVec; std::vector roids, tkeys, cscale; std::vector widthVec; + std::vector charSetNumVec; types.push_back(execplan::CalpontSystemCatalog::DECIMAL); types.push_back(execplan::CalpontSystemCatalog::UDECIMAL); @@ -68,6 +69,7 @@ protected: roids.push_back(oid + i); tkeys.push_back(i + 1); cscale.push_back(0); + charSetNumVec.push_back(8); } rowgroup::RowGroup inRG(roids.size(), // column count @@ -75,6 +77,7 @@ protected: roids, // column oids tkeys, // keys types, // types + charSetNumVec, // charset numbers cscale, // scale precisionVec, // precision 20, // sTableThreshold diff --git a/utils/common/columnwidth.h b/utils/common/columnwidth.h index d21e15230..9d02a7312 100644 --- a/utils/common/columnwidth.h +++ b/utils/common/columnwidth.h @@ -23,6 +23,7 @@ namespace utils { const uint8_t MAXLEGACYWIDTH = 8ULL; + const uint8_t MAXCOLUMNWIDTH = 16ULL; inline bool isWide(uint8_t width) { diff --git a/utils/messageqcpp/bytestream.cpp b/utils/messageqcpp/bytestream.cpp index ec6872dcb..c0fad8554 100644 --- a/utils/messageqcpp/bytestream.cpp +++ b/utils/messageqcpp/bytestream.cpp @@ -235,18 +235,6 @@ ByteStream& ByteStream::operator<<(const uint64_t o) return *this; } -// WIP MCOL-641 -ByteStream& ByteStream::operator<<(const int128_t& o) -{ - if (fBuf == 0 || (fCurInPtr - fBuf + 16U > fMaxLen + ISSOverhead)) - growBuf(fMaxLen + BlockSize); - - *((int128_t*) fCurInPtr) = o; - fCurInPtr += 16; - - return *this; -} - ByteStream& ByteStream::operator<<(const uint128_t& o) { if (fBuf == 0 || (fCurInPtr - fBuf + 16U > fMaxLen + ISSOverhead)) @@ -258,6 +246,17 @@ ByteStream& ByteStream::operator<<(const uint128_t& o) return *this; } +ByteStream& ByteStream::operator<<(const int128_t& o) +{ + if (fBuf == 0 || (fCurInPtr - fBuf + 16U > fMaxLen + ISSOverhead)) + growBuf(fMaxLen + BlockSize); + + *((int128_t*) fCurInPtr) = o; + fCurInPtr += 16; + + return *this; +} + ByteStream& ByteStream::operator<<(const string& s) { int32_t len = s.size(); @@ -342,15 +341,14 @@ ByteStream& ByteStream::operator>>(uint64_t& o) return *this; } -// WIP MCOL-641 -ByteStream& ByteStream::operator>>(int128_t& o) +ByteStream& ByteStream::operator>>(uint128_t& o) { peek(o); fCurOutPtr += 16; return *this; } -ByteStream& ByteStream::operator>>(uint128_t& o) +ByteStream& ByteStream::operator>>(int128_t& o) { peek(o); fCurOutPtr += 16; @@ -437,16 +435,6 @@ void ByteStream::peek(uint64_t& o) const o = *((uint64_t*) fCurOutPtr); } -// WIP MCOL-641 -void ByteStream::peek(int128_t& o) const -{ - - if (length() < 16) - throw underflow_error("ByteStream>int128_t: not enough data in stream to fill datatype"); - - o = *((int128_t*) fCurOutPtr); -} - void ByteStream::peek(uint128_t& o) const { @@ -456,6 +444,15 @@ void ByteStream::peek(uint128_t& o) const o = *((uint128_t*) fCurOutPtr); } +void ByteStream::peek(int128_t& o) const +{ + + if (length() < 16) + throw underflow_error("ByteStream>int128_t: not enough data in stream to fill datatype"); + + o = *((int128_t*) fCurOutPtr); +} + void ByteStream::peek(string& s) const { int32_t len; diff --git a/utils/messageqcpp/bytestream.h b/utils/messageqcpp/bytestream.h index 59aa1679c..35a85e3bc 100644 --- a/utils/messageqcpp/bytestream.h +++ b/utils/messageqcpp/bytestream.h @@ -47,6 +47,7 @@ class ByteStreamTestSuite; using int128_t = __int128; using uint128_t = unsigned __int128; +using int128_t = __int128; namespace messageqcpp { @@ -77,7 +78,7 @@ public: typedef uint16_t doublebyte; typedef uint32_t quadbyte; typedef uint64_t octbyte; - typedef uint128_t hexbyte; + typedef int128_t hexbyte; typedef boost::uuids::uuid uuid; /** @@ -147,7 +148,6 @@ public: * push an uint64_t onto the end of the stream. The byte order is whatever the native byte order is. */ EXPORT ByteStream& operator<<(const uint64_t o); - // WIP MCOL-641 /** * push an uint128_t onto the end of the stream. The byte order is whatever the native byte order is. */ @@ -156,6 +156,10 @@ public: * push an uint128_t onto the end of the stream. The byte order is whatever the native byte order is. */ EXPORT ByteStream& operator<<(const uint128_t& o); + /** + * push an int128_t onto the end of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator<<(const int128_t& o); /** * push a float onto the end of the stream. The byte order is * whatever the native byte order is. @@ -220,7 +224,6 @@ public: * extract an uint64_t from the front of the stream. The byte order is whatever the native byte order is. */ EXPORT ByteStream& operator>>(uint64_t& o); - // WIP MCOL-641 /** * extract an uint128_t from the front of the stream. The byte order is whatever the native byte order is. */ @@ -229,6 +232,10 @@ public: * extract an uint128_t from the front of the stream. The byte order is whatever the native byte order is. */ EXPORT ByteStream& operator>>(uint128_t& o); + /** + * extract an int128_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT ByteStream& operator>>(int128_t& o); /** * extract a float from the front of the stream. The byte * order is whatever the native byte order is. @@ -299,7 +306,6 @@ public: * Peek at an uint64_t from the front of the stream. The byte order is whatever the native byte order is. */ EXPORT void peek(uint64_t& o) const; - // WIP MCOL-641 /** * Peek at an uint128_t from the front of the stream. The byte order is whatever the native byte order is. */ @@ -308,6 +314,10 @@ public: * Peek at an uint128_t from the front of the stream. The byte order is whatever the native byte order is. */ EXPORT void peek(uint128_t& o) const; + /** + * Peek at an int128_t from the front of the stream. The byte order is whatever the native byte order is. + */ + EXPORT void peek(int128_t& o) const; /** * Peek at a float from the front of the stream. The byte order * is whatever the native byte order is. From 62c1c1e0e2134667882cb3c2e148eb12e6f4feef Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Mon, 27 Jul 2020 19:22:13 -0400 Subject: [PATCH 53/78] Remove hi_val/lo_val data members from EMCasualPartition_struct and use the union members instead. --- dbcon/joblist/lbidlist.cpp | 40 +++---- dbcon/joblist/pcolstep.cpp | 8 +- dbcon/joblist/pseudocc-jl.cpp | 8 +- dbcon/joblist/rowestimator.cpp | 25 ++-- dbcon/joblist/tuple-bps.cpp | 10 +- dbcon/mysql/ha_mcs_partition.cpp | 28 ++--- dbcon/mysql/is_columnstore_extents.cpp | 10 +- tools/editem/editem.cpp | 4 +- versioning/BRM/extentmap.cpp | 151 ++++++++++++------------- versioning/BRM/extentmap.h | 4 +- 10 files changed, 135 insertions(+), 153 deletions(-) diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index bc7ca9303..79d053d9a 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -335,8 +335,8 @@ bool LBIDList::GetMinMax(T* min, T* max, int64_t* seq, } else { - *min = entry.partition.cprange.lo_val; - *max = entry.partition.cprange.hi_val; + *min = entry.partition.cprange.loVal; + *max = entry.partition.cprange.hiVal; } *seq = entry.partition.cprange.sequenceNum; @@ -364,8 +364,8 @@ int LBIDList::getMinMaxFromEntries(T& min, T& max, int32_t& seq, } else { - min = EMEntries[i].partition.cprange.lo_val; - max = EMEntries[i].partition.cprange.hi_val; + min = EMEntries[i].partition.cprange.loVal; + max = EMEntries[i].partition.cprange.hiVal; } seq = EMEntries[i].partition.cprange.sequenceNum; return EMEntries[i].partition.cprange.isValid; @@ -757,7 +757,6 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, bool scan = true; int64_t value = 0; __int128 bigValue = 0; - uint64_t* int128Ptr = reinterpret_cast(&bigValue); bool bIsUnsigned = execplan::isUnsigned(ct.colDataType); bool bIsChar = execplan::isCharType(ct.colDataType); @@ -806,15 +805,14 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, { uint64_t val = *(int64_t*)MsgDataPtr; value = static_cast(val); + break; } case 16: { - unsigned __int128 val; - int128Ptr = reinterpret_cast(&val); - int128Ptr[0] = *reinterpret_cast(MsgDataPtr); - int128Ptr[1] = *(reinterpret_cast(MsgDataPtr) + 1); - bigValue = static_cast<__int128>(val); + uint128_t val = *(int128_t*)MsgDataPtr; + bigValue = static_cast(val); + break; } } } @@ -847,12 +845,14 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, { int64_t val = *(int64_t*)MsgDataPtr; value = val; + break; } case 16: { - int128Ptr[0] = *reinterpret_cast(MsgDataPtr); - int128Ptr[1] = *(reinterpret_cast(MsgDataPtr) + 1); + int128_t val = *(int128_t*)MsgDataPtr; + bigValue = val; + break; } } } @@ -875,8 +875,8 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, { // MCOL-1246 Trim trailing whitespace for matching so that we have // the same as InnoDB behaviour - int64_t tMin = cpRange.lo_val; - int64_t tMax = cpRange.hi_val; + int64_t tMin = cpRange.loVal; + int64_t tMax = cpRange.hiVal; dataconvert::DataConvert::trimWhitespace(tMin); dataconvert::DataConvert::trimWhitespace(tMax); @@ -886,22 +886,22 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, } else if (bIsUnsigned) { - if (ct.colWidth <= 8) + if (ct.colWidth != datatypes::MAXDECIMALWIDTH) { - scan = compareVal(static_cast(cpRange.lo_val), static_cast(cpRange.hi_val), static_cast(value), op, lcf); + scan = compareVal(static_cast(cpRange.loVal), static_cast(cpRange.hiVal), static_cast(value), op, lcf); } - else if (ct.colWidth == 16) + else { scan = compareVal(static_cast(cpRange.bigLoVal), static_cast(cpRange.bigHiVal), static_cast(bigValue), op, lcf); } } else { - if (ct.colWidth <= 8) + if (ct.colWidth != datatypes::MAXDECIMALWIDTH) { - scan = compareVal(cpRange.lo_val, cpRange.hi_val, value, op, lcf); + scan = compareVal(cpRange.loVal, cpRange.hiVal, value, op, lcf); } - else if (ct.colWidth == 16) + else { scan = compareVal(cpRange.bigLoVal, cpRange.bigHiVal, bigValue, op, lcf); } diff --git a/dbcon/joblist/pcolstep.cpp b/dbcon/joblist/pcolstep.cpp index fb8c1c209..722fb57ce 100644 --- a/dbcon/joblist/pcolstep.cpp +++ b/dbcon/joblist/pcolstep.cpp @@ -739,8 +739,8 @@ void pColStep::sendPrimitiveMessages() // { // // bool flag = lbidList->CasualPartitionPredicate( -// extents[idx].partition.cprange.lo_val, -// extents[idx].partition.cprange.hi_val, +// extents[idx].partition.cprange.loVal, +// extents[idx].partition.cprange.hiVal, // &fFilterString, // fFilterCount, // fColType, @@ -749,8 +749,8 @@ void pColStep::sendPrimitiveMessages() //#ifdef DEBUG // if (fOid >= 3000 && flushInterval == 0) // cout << (flag ? " will scan " : " will not scan ") -// << "extent with range " << extents[idx].partition.cprange.lo_val -// << "-" << extents[idx].partition.cprange.hi_val << endl; +// << "extent with range " << extents[idx].partition.cprange.loVal +// << "-" << extents[idx].partition.cprange.hiVal << endl; //#endif // // } diff --git a/dbcon/joblist/pseudocc-jl.cpp b/dbcon/joblist/pseudocc-jl.cpp index 65b5a2317..94e7cb574 100644 --- a/dbcon/joblist/pseudocc-jl.cpp +++ b/dbcon/joblist/pseudocc-jl.cpp @@ -48,8 +48,8 @@ void PseudoCCJL::runCommand(ByteStream& bs) const { if (!datatypes::Decimal::isWideDecimalType(colType)) { - int64_t max = extents[currentExtentIndex].partition.cprange.hi_val; - int64_t min = extents[currentExtentIndex].partition.cprange.lo_val; + int64_t max = extents[currentExtentIndex].partition.cprange.hiVal; + int64_t min = extents[currentExtentIndex].partition.cprange.loVal; if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) bs << max; @@ -75,8 +75,8 @@ void PseudoCCJL::runCommand(ByteStream& bs) const { if (!datatypes::Decimal::isWideDecimalType(colType)) { - int64_t max = extents[currentExtentIndex].partition.cprange.hi_val; - int64_t min = extents[currentExtentIndex].partition.cprange.lo_val; + int64_t max = extents[currentExtentIndex].partition.cprange.hiVal; + int64_t min = extents[currentExtentIndex].partition.cprange.loVal; if (extents[currentExtentIndex].partition.cprange.isValid == BRM::CP_VALID && max >= min) bs << min; diff --git a/dbcon/joblist/rowestimator.cpp b/dbcon/joblist/rowestimator.cpp index 1a5e5f73d..c28e00781 100644 --- a/dbcon/joblist/rowestimator.cpp +++ b/dbcon/joblist/rowestimator.cpp @@ -200,7 +200,7 @@ uint32_t RowEstimator::estimateDistinctValues(const execplan::CalpontSystemCatal ret = max - min + 1; } - if (ret > (T) fRowsPerExtent) + if (ret > fRowsPerExtent) { ret = fRowsPerExtent; } @@ -251,7 +251,7 @@ float RowEstimator::estimateOpFactor(const T& min, const T& max, const T& value, if (!datatypes::Decimal::isWideDecimalType(ct)) factor = (1.0 * max - value) / (1.0 * max - min + 1); else - factor = (1.0 * max - value) / (1.0 * max - min + 1); + factor = ((__float128) max - value) / (max - min + 1); } break; @@ -264,7 +264,7 @@ float RowEstimator::estimateOpFactor(const T& min, const T& max, const T& value, if (!datatypes::Decimal::isWideDecimalType(ct)) factor = (1.0 * max - value + 1) / (max - min + 1); else - factor = (1.0 * max - value + 1) / (max - min + 1); + factor = ((__float128) max - value + 1) / (max - min + 1); } break; @@ -310,8 +310,8 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, // Adjust values based on column type and estimate the if (!datatypes::Decimal::isWideDecimalType(ct)) { - adjustedMin = adjustValue(ct, emEntry.partition.cprange.lo_val); - adjustedMax = adjustValue(ct, emEntry.partition.cprange.hi_val); + adjustedMin = adjustValue(ct, emEntry.partition.cprange.loVal); + adjustedMax = adjustValue(ct, emEntry.partition.cprange.hiVal); distinctValuesEstimate = estimateDistinctValues( ct, adjustedMin, adjustedMax, emEntry.partition.cprange.isValid); } @@ -452,8 +452,8 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, } #if ROW_EST_DEBUG - cout << " Min-" << emEntry.partition.cprange.lo_val << - ", Max-" << emEntry.partition.cprange.hi_val << + cout << " Min-" << emEntry.partition.cprange.loVal << + ", Max-" << emEntry.partition.cprange.hiVal << ", Val-" << value; #endif @@ -685,16 +685,5 @@ uint64_t RowEstimator::estimateRowsForNonCPColumn(ColumnCommandJL& colCmd) return estimatedRows; } -template -uint32_t RowEstimator::estimateDistinctValues(const execplan::CalpontSystemCatalog::ColType& ct, - const int128_t& min, - const int128_t& max, - const char cpStatus); - -template -uint32_t RowEstimator::estimateDistinctValues(const execplan::CalpontSystemCatalog::ColType& ct, - const int64_t& min, - const int64_t& max, - const char cpStatus); } //namespace joblist diff --git a/dbcon/joblist/tuple-bps.cpp b/dbcon/joblist/tuple-bps.cpp index 7df74cc54..dcdc2f5f1 100644 --- a/dbcon/joblist/tuple-bps.cpp +++ b/dbcon/joblist/tuple-bps.cpp @@ -1584,7 +1584,7 @@ bool TupleBPS::processSingleFilterString(int8_t BOP, int8_t colWidth, T val, con throw logic_error("invalid column width"); } - if (colWidth < 16) + if (colWidth < datatypes::MAXDECIMALWIDTH) thisPredicate = compareSingleValue(COP, (int64_t) val, val2); else thisPredicate = compareSingleValue(COP, (int128_t) val, bigVal2); @@ -1694,11 +1694,11 @@ bool TupleBPS::processPseudoColFilters(uint32_t extentIndex, boost::shared_ptr::max(); @@ -1334,7 +1334,7 @@ extern "C" if (!datatypes::Decimal::isWideDecimalType(ct)) output << setw(30) << "N/A" << setw(30) << "N/A"; else - output << setw(42) << "N/A" << setw(42) << "N/A"; + output << setw(utils::MAXLENGTH16BYTES) << "N/A" << setw(utils::MAXLENGTH16BYTES) << "N/A"; } else { @@ -1352,9 +1352,9 @@ extern "C" { if (static_cast(partIt->second.bigMin) == ubigMaxLimit && static_cast(partIt->second.bigMax) == ubigMinLimit) - output << setw(42) << "Empty/Null" << setw(42) << "Empty/Null"; + output << setw(utils::MAXLENGTH16BYTES) << "Empty/Null" << setw(utils::MAXLENGTH16BYTES) << "Empty/Null"; else - output << setw(42) << format(partIt->second.bigMin, ct) << setw(42) << format(partIt->second.bigMax, ct); + output << setw(utils::MAXLENGTH16BYTES) << format(partIt->second.bigMin, ct) << setw(utils::MAXLENGTH16BYTES) << format(partIt->second.bigMax, ct); } } else @@ -1369,9 +1369,9 @@ extern "C" else { if (partIt->second.bigMin == bigMaxLimit && partIt->second.bigMax == bigMinLimit) - output << setw(42) << "Empty/Null" << setw(42) << "Empty/Null"; + output << setw(utils::MAXLENGTH16BYTES) << "Empty/Null" << setw(utils::MAXLENGTH16BYTES) << "Empty/Null"; else - output << setw(42) << format(partIt->second.bigMin, ct) << setw(42) << format(partIt->second.bigMax, ct); + output << setw(utils::MAXLENGTH16BYTES) << format(partIt->second.bigMin, ct) << setw(utils::MAXLENGTH16BYTES) << format(partIt->second.bigMax, ct); } } } @@ -2308,8 +2308,8 @@ extern "C" { output.setf(ios::left, ios::adjustfield); output << setw(10) << "Part#" - << setw(42) << "Min" - << setw(42) << "Max" << "Status"; + << setw(utils::MAXLENGTH16BYTES) << "Min" + << setw(utils::MAXLENGTH16BYTES) << "Max" << "Status"; } noPartFound = false; @@ -2321,23 +2321,23 @@ extern "C" if (mapit->second.status & CPINVALID) { - output << setw(42) << "N/A" << setw(42) << "N/A"; + output << setw(utils::MAXLENGTH16BYTES) << "N/A" << setw(utils::MAXLENGTH16BYTES) << "N/A"; } else { if ((isUnsigned(ct.colDataType))) { if (static_cast(mapit->second.bigMin) > static_cast(mapit->second.bigMax)) - output << setw(42) << "Empty/Null" << setw(42) << "Empty/Null"; + output << setw(utils::MAXLENGTH16BYTES) << "Empty/Null" << setw(utils::MAXLENGTH16BYTES) << "Empty/Null"; else - output << setw(42) << format(mapit->second.bigMin, ct) << setw(42) << format(mapit->second.bigMax, ct); + output << setw(utils::MAXLENGTH16BYTES) << format(mapit->second.bigMin, ct) << setw(utils::MAXLENGTH16BYTES) << format(mapit->second.bigMax, ct); } else { if (mapit->second.bigMin > mapit->second.bigMax) - output << setw(42) << "Empty/Null" << setw(42) << "Empty/Null"; + output << setw(utils::MAXLENGTH16BYTES) << "Empty/Null" << setw(utils::MAXLENGTH16BYTES) << "Empty/Null"; else - output << setw(42) << format(mapit->second.bigMin, ct) << setw(42) << format(mapit->second.bigMax, ct); + output << setw(utils::MAXLENGTH16BYTES) << format(mapit->second.bigMin, ct) << setw(utils::MAXLENGTH16BYTES) << format(mapit->second.bigMax, ct); } } diff --git a/dbcon/mysql/is_columnstore_extents.cpp b/dbcon/mysql/is_columnstore_extents.cpp index d93152a52..7f48b0d41 100644 --- a/dbcon/mysql/is_columnstore_extents.cpp +++ b/dbcon/mysql/is_columnstore_extents.cpp @@ -82,25 +82,25 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM* emp, TABLE* table, THD* th if (iter->colWid != datatypes::MAXDECIMALWIDTH) { - if (iter->partition.cprange.lo_val == std::numeric_limits::max() || - iter->partition.cprange.lo_val <= (std::numeric_limits::min() + 1)) + if (iter->partition.cprange.loVal == std::numeric_limits::max() || + iter->partition.cprange.loVal <= (std::numeric_limits::min() + 1)) { table->field[4]->set_null(); } else { table->field[4]->set_notnull(); - table->field[4]->store(iter->partition.cprange.lo_val); + table->field[4]->store(iter->partition.cprange.loVal); } - if (iter->partition.cprange.hi_val <= (std::numeric_limits::min() + 1)) + if (iter->partition.cprange.hiVal <= (std::numeric_limits::min() + 1)) { table->field[5]->set_null(); } else { table->field[5]->set_notnull(); - table->field[5]->store(iter->partition.cprange.hi_val); + table->field[5]->store(iter->partition.cprange.hiVal); } } else diff --git a/tools/editem/editem.cpp b/tools/editem/editem.cpp index c8cc31683..9ed40b2aa 100644 --- a/tools/editem/editem.cpp +++ b/tools/editem/editem.cpp @@ -349,8 +349,8 @@ int dumpone(OID_t oid, unsigned int sortOrder) if (iter->colWid != datatypes::MAXDECIMALWIDTH) { - max = iter->partition.cprange.hi_val; - min = iter->partition.cprange.lo_val; + max = iter->partition.cprange.hiVal; + min = iter->partition.cprange.loVal; cout << iter->range.start << " - " << (iter->range.start + lbidRangeSize - 1) << diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index e5f84ea53..fb054551c 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -56,6 +56,7 @@ namespace bi = boost::interprocess; #include "blocksize.h" #include "dataconvert.h" #include "widedecimalutils.h" +#include "mcs_decimal.h" #include "oamcache.h" #include "IDBDataFile.h" #include "IDBPolicy.h" @@ -118,8 +119,6 @@ namespace BRM EMCasualPartition_struct::EMCasualPartition_struct() { - lo_val = numeric_limits::max(); - hi_val = numeric_limits::min(); utils::int128Max(bigLoVal); utils::int128Min(bigHiVal); sequenceNum = 0; @@ -128,8 +127,8 @@ EMCasualPartition_struct::EMCasualPartition_struct() EMCasualPartition_struct::EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum) { - lo_val = lo; - hi_val = hi; + loVal = lo; + hiVal = hi; sequenceNum = seqNum; isValid = CP_INVALID; } @@ -144,8 +143,6 @@ EMCasualPartition_struct::EMCasualPartition_struct(const __int128 bigLo, const _ EMCasualPartition_struct::EMCasualPartition_struct(const EMCasualPartition_struct& em) { - lo_val = em.lo_val; - hi_val = em.hi_val; bigLoVal = em.bigLoVal; bigHiVal = em.bigHiVal; sequenceNum = em.sequenceNum; @@ -154,8 +151,6 @@ EMCasualPartition_struct::EMCasualPartition_struct(const EMCasualPartition_struc EMCasualPartition_struct& EMCasualPartition_struct::operator= (const EMCasualPartition_struct& em) { - lo_val = em.lo_val; - hi_val = em.hi_val; bigLoVal = em.bigLoVal; bigHiVal = em.bigHiVal; sequenceNum = em.sequenceNum; @@ -353,17 +348,21 @@ int ExtentMap::_markInvalid(const LBID_t lbid, const execplan::CalpontSystemCata if (isUnsigned(colDataType)) { - fExtentMap[i].partition.cprange.lo_val = numeric_limits::max(); - fExtentMap[i].partition.cprange.hi_val = 0; fExtentMap[i].partition.cprange.bigLoVal = -1; fExtentMap[i].partition.cprange.bigHiVal = 0; } else { - fExtentMap[i].partition.cprange.lo_val = numeric_limits::max(); - fExtentMap[i].partition.cprange.hi_val = numeric_limits::min(); - utils::int128Max(fExtentMap[i].partition.cprange.bigLoVal); - utils::int128Min(fExtentMap[i].partition.cprange.bigHiVal); + if (fExtentMap[i].colWid != datatypes::MAXDECIMALWIDTH) + { + fExtentMap[i].partition.cprange.loVal = numeric_limits::max(); + fExtentMap[i].partition.cprange.hiVal = numeric_limits::min(); + } + else + { + utils::int128Max(fExtentMap[i].partition.cprange.bigLoVal); + utils::int128Min(fExtentMap[i].partition.cprange.bigHiVal); + } } incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); @@ -372,8 +371,8 @@ int ExtentMap::_markInvalid(const LBID_t lbid, const execplan::CalpontSystemCata os << "ExtentMap::_markInvalid(): casual partitioning update: firstLBID=" << fExtentMap[i].range.start << " lastLBID=" << fExtentMap[i].range.start + fExtentMap[i].range.size * 1024 - 1 << " OID=" << fExtentMap[i].fileID << - " min=" << fExtentMap[i].partition.cprange.lo_val << - " max=" << fExtentMap[i].partition.cprange.hi_val << + " min=" << fExtentMap[i].partition.cprange.loVal << + " max=" << fExtentMap[i].partition.cprange.hiVal << "seq=" << fExtentMap[i].partition.cprange.sequenceNum; log(os.str(), logging::LOG_TYPE_DEBUG); #endif @@ -472,7 +471,7 @@ int ExtentMap::markInvalid(const vector& lbids, /** * @brief set the max/min values for the extent if the seqNum matches the extents sequenceNum * -* reset the lbid's hi_val to max and lo_val to min +* reset the lbid's hiVal to max and loVal to min * the seqNum matches the ExtentMap.sequenceNum. Then increments * the current sequenceNum value by 1. If the sequenceNum does not * match the seqNum value do not update the lbid's max/min values @@ -543,8 +542,8 @@ int ExtentMap::setMaxMin(const LBID_t lbid, if (curSequence == seqNum) { makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); - fExtentMap[i].partition.cprange.hi_val = max; - fExtentMap[i].partition.cprange.lo_val = min; + fExtentMap[i].partition.cprange.hiVal = max; + fExtentMap[i].partition.cprange.loVal = min; fExtentMap[i].partition.cprange.isValid = CP_VALID; incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); return 0; @@ -554,7 +553,7 @@ int ExtentMap::setMaxMin(const LBID_t lbid, else if (seqNum == -1) { makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); - // We set hi_val and lo_val to correct values for signed or unsigned + // We set hiVal and loVal to correct values for signed or unsigned // during the markinvalid step, which sets the invalid variable to CP_UPDATING. // During this step (seqNum == -1), the min and max passed in are not reliable // and should not be used. @@ -655,8 +654,8 @@ void ExtentMap::setExtentsMaxMin(const CPMaxMinMap_t& cpMap, bool firstNode, boo } else { - fExtentMap[i].partition.cprange.hi_val = it->second.max; - fExtentMap[i].partition.cprange.lo_val = it->second.min; + fExtentMap[i].partition.cprange.hiVal = it->second.max; + fExtentMap[i].partition.cprange.loVal = it->second.min; } fExtentMap[i].partition.cprange.isValid = CP_VALID; incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); @@ -681,7 +680,7 @@ void ExtentMap::setExtentsMaxMin(const CPMaxMinMap_t& cpMap, bool firstNode, boo else if (it->second.seqNum == -1) { makeUndoRecord(&fExtentMap[i], sizeof(struct EMEntry)); - // We set hi_val and lo_val to correct values for signed or unsigned + // We set hiVal and loVal to correct values for signed or unsigned // during the markinvalid step, which sets the invalid variable to CP_UPDATING. // During this step (seqNum == -1), the min and max passed in are not reliable // and should not be used. @@ -700,8 +699,8 @@ void ExtentMap::setExtentsMaxMin(const CPMaxMinMap_t& cpMap, bool firstNode, boo } else { - fExtentMap[i].partition.cprange.hi_val = it->second.max; - fExtentMap[i].partition.cprange.lo_val = it->second.min; + fExtentMap[i].partition.cprange.hiVal = it->second.max; + fExtentMap[i].partition.cprange.loVal = it->second.min; } fExtentMap[i].partition.cprange.isValid = CP_INVALID; incSeqNum(fExtentMap[i].partition.cprange.sequenceNum); @@ -816,8 +815,8 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) os << "ExtentMap::mergeExtentsMaxMin(): casual partitioning update: firstLBID=" << fExtentMap[i].range.start << " lastLBID=" << fExtentMap[i].range.start + fExtentMap[i].range.size * 1024 - 1 << " OID=" << fExtentMap[i].fileID << - " hi_val=" << fExtentMap[i].partition.cprange.hi_val << - " lo_val=" << fExtentMap[i].partition.cprange.lo_val << + " hiVal=" << fExtentMap[i].partition.cprange.hiVal << + " loVal=" << fExtentMap[i].partition.cprange.loVal << " min=" << it->second.min << " max=" << it->second.max << " seq=" << it->second.seqNum; log(os.str(), logging::LOG_TYPE_DEBUG); @@ -845,8 +844,8 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) // min/max needs to be set instead of merged. if (isValidCPRange( - !isBinaryColumn ? fExtentMap[i].partition.cprange.hi_val : fExtentMap[i].partition.cprange.bigHiVal, - !isBinaryColumn ? fExtentMap[i].partition.cprange.lo_val : fExtentMap[i].partition.cprange.bigLoVal, + !isBinaryColumn ? fExtentMap[i].partition.cprange.hiVal : fExtentMap[i].partition.cprange.bigHiVal, + !isBinaryColumn ? fExtentMap[i].partition.cprange.loVal : fExtentMap[i].partition.cprange.bigLoVal, it->second.type)) { // Swap byte order to do binary string comparison @@ -861,18 +860,18 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) int64_t oldMinVal = static_cast( uint64ToStr( static_cast( - fExtentMap[i].partition.cprange.lo_val)) ); + fExtentMap[i].partition.cprange.loVal)) ); int64_t oldMaxVal = static_cast( uint64ToStr( static_cast( - fExtentMap[i].partition.cprange.hi_val)) ); + fExtentMap[i].partition.cprange.hiVal)) ); if (newMinVal < oldMinVal) - fExtentMap[i].partition.cprange.lo_val = + fExtentMap[i].partition.cprange.loVal = it->second.min; if (newMaxVal > oldMaxVal) - fExtentMap[i].partition.cprange.hi_val = + fExtentMap[i].partition.cprange.hiVal = it->second.max; } else if (isUnsigned(it->second.type)) @@ -880,16 +879,16 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) if (!isBinaryColumn) { if (static_cast(it->second.min) < - static_cast(fExtentMap[i].partition.cprange.lo_val)) + static_cast(fExtentMap[i].partition.cprange.loVal)) { - fExtentMap[i].partition.cprange.lo_val = + fExtentMap[i].partition.cprange.loVal = it->second.min; } if (static_cast(it->second.max) > - static_cast(fExtentMap[i].partition.cprange.hi_val)) + static_cast(fExtentMap[i].partition.cprange.hiVal)) { - fExtentMap[i].partition.cprange.hi_val = + fExtentMap[i].partition.cprange.hiVal = it->second.max; } } @@ -915,13 +914,13 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) if (!isBinaryColumn) { if (it->second.min < - fExtentMap[i].partition.cprange.lo_val) - fExtentMap[i].partition.cprange.lo_val = + fExtentMap[i].partition.cprange.loVal) + fExtentMap[i].partition.cprange.loVal = it->second.min; if (it->second.max > - fExtentMap[i].partition.cprange.hi_val) - fExtentMap[i].partition.cprange.hi_val = + fExtentMap[i].partition.cprange.hiVal) + fExtentMap[i].partition.cprange.hiVal = it->second.max; } else @@ -942,9 +941,9 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) { if (!isBinaryColumn) { - fExtentMap[i].partition.cprange.lo_val = + fExtentMap[i].partition.cprange.loVal = it->second.min; - fExtentMap[i].partition.cprange.hi_val = + fExtentMap[i].partition.cprange.hiVal = it->second.max; } else @@ -988,9 +987,9 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) { if (!isBinaryColumn) { - fExtentMap[i].partition.cprange.lo_val = + fExtentMap[i].partition.cprange.loVal = it->second.min; - fExtentMap[i].partition.cprange.hi_val = + fExtentMap[i].partition.cprange.hiVal = it->second.max; } else @@ -1086,7 +1085,7 @@ bool ExtentMap::isValidCPRange(const T& max, const T& min, execplan::CalpontSyst } /** -* @brief retrieve the hi_val and lo_val or sequenceNum of the extent containing the LBID lbid. +* @brief retrieve the hiVal and loVal or sequenceNum of the extent containing the LBID lbid. * * For the extent containing the LBID lbid, return the max/min values if the extent range values * are valid and a -1 in the seqNum parameter. If the range values are flaged as invalid @@ -1156,8 +1155,8 @@ int ExtentMap::getMaxMin(const LBID_t lbid, } else { - max = fExtentMap[i].partition.cprange.hi_val; - min = fExtentMap[i].partition.cprange.lo_val; + max = fExtentMap[i].partition.cprange.hiVal; + min = fExtentMap[i].partition.cprange.loVal; } seqNum = fExtentMap[i].partition.cprange.sequenceNum; isValid = fExtentMap[i].partition.cprange.isValid; @@ -1369,12 +1368,10 @@ void ExtentMap::loadVersion4or5(IDBDataFile* in, bool upgradeV4ToV5) fExtentMap[i].dbRoot = emEntryV4.dbRoot; fExtentMap[i].colWid = emEntryV4.colWid; fExtentMap[i].status = emEntryV4.status; - fExtentMap[i].partition.cprange.hi_val = emEntryV4.partition.cprange.hi_val; - fExtentMap[i].partition.cprange.lo_val = emEntryV4.partition.cprange.lo_val; + fExtentMap[i].partition.cprange.hiVal = emEntryV4.partition.cprange.hi_val; + fExtentMap[i].partition.cprange.loVal = emEntryV4.partition.cprange.lo_val; fExtentMap[i].partition.cprange.sequenceNum = emEntryV4.partition.cprange.sequenceNum; fExtentMap[i].partition.cprange.isValid = emEntryV4.partition.cprange.isValid; - utils::int128Max(fExtentMap[i].partition.cprange.bigLoVal); - utils::int128Min(fExtentMap[i].partition.cprange.bigHiVal); } std::cout<partition.cprange.lo_val = numeric_limits::max(); - e->partition.cprange.hi_val = 0; + e->partition.cprange.loVal = numeric_limits::max(); + e->partition.cprange.hiVal = 0; } else { @@ -2776,10 +2773,10 @@ LBID_t ExtentMap::_createColumnExtent_DBroot(uint32_t size, int OID, } else { - if (colWidth <= 8) + if (colWidth != datatypes::MAXDECIMALWIDTH) { - e->partition.cprange.lo_val = numeric_limits::max(); - e->partition.cprange.hi_val = numeric_limits::min(); + e->partition.cprange.loVal = numeric_limits::max(); + e->partition.cprange.hiVal = numeric_limits::min(); } else { @@ -2974,10 +2971,10 @@ LBID_t ExtentMap::_createColumnExtentExactFile(uint32_t size, int OID, if (isUnsigned(colDataType)) { - if (colWidth <= 8) + if (colWidth != datatypes::MAXDECIMALWIDTH) { - e->partition.cprange.lo_val = numeric_limits::max(); - e->partition.cprange.hi_val = 0; + e->partition.cprange.loVal = numeric_limits::max(); + e->partition.cprange.hiVal = 0; } else { @@ -2987,10 +2984,10 @@ LBID_t ExtentMap::_createColumnExtentExactFile(uint32_t size, int OID, } else { - if (colWidth <= 8) + if (colWidth != datatypes::MAXDECIMALWIDTH) { - e->partition.cprange.lo_val = numeric_limits::max(); - e->partition.cprange.hi_val = numeric_limits::min(); + e->partition.cprange.loVal = numeric_limits::max(); + e->partition.cprange.hiVal = numeric_limits::min(); } else { @@ -3179,8 +3176,6 @@ LBID_t ExtentMap::_createDictStoreExtent(uint32_t size, int OID, e->range.size = size; e->fileID = OID; e->status = EXTENTUNAVAILABLE;// @bug 1911 mark extent as in process - e->partition.cprange.lo_val = numeric_limits::max(); - e->partition.cprange.hi_val = numeric_limits::min(); utils::int128Max(e->partition.cprange.bigLoVal); utils::int128Min(e->partition.cprange.bigHiVal); e->partition.cprange.sequenceNum = 0; @@ -3271,8 +3266,8 @@ void ExtentMap::printEM(const EMEntry& em) const << (long) em.range.size << " OID " << (long) em.fileID << " offset " << (long) em.blockOffset - << " LV " << em.partition.cprange.lo_val - << " HV " << em.partition.cprange.hi_val; + << " LV " << em.partition.cprange.loVal + << " HV " << em.partition.cprange.hiVal; cout << endl; } @@ -4668,8 +4663,8 @@ void ExtentMap::setLocalHWM(int OID, uint32_t partitionNum, os << "ExtentMap::setLocalHWM(): firstLBID=" << fExtentMap[lastExtentIndex].range.start << " lastLBID=" << fExtentMap[lastExtentIndex].range.start + fExtentMap[lastExtentIndex].range.size * 1024 - 1 << " newHWM=" << fExtentMap[lastExtentIndex].HWM - << " min=" << fExtentMap[lastExtentIndex].partition.cprange.lo_val << " max=" << - fExtentMap[lastExtentIndex].partition.cprange.hi_val << " seq=" << + << " min=" << fExtentMap[lastExtentIndex].partition.cprange.loVal << " max=" << + fExtentMap[lastExtentIndex].partition.cprange.hiVal << " seq=" << fExtentMap[lastExtentIndex].partition.cprange.sequenceNum << " status="; switch (fExtentMap[lastExtentIndex].partition.cprange.isValid) @@ -4830,8 +4825,8 @@ void ExtentMap::getExtents_dbroot(int OID, vector& entries, cons fakeEntry.dbRoot = 1; fakeEntry.colWid = 4; fakeEntry.status = EXTENTAVAILABLE; - fakeEntry.partition.cprange.hi_val = numeric_limits::min() + 2; - fakeEntry.partition.cprange.lo_val = numeric_limits::max(); + fakeEntry.partition.cprange.hiVal = numeric_limits::min() + 2; + fakeEntry.partition.cprange.loVal = numeric_limits::max(); fakeEntry.partition.cprange.sequenceNum = 0; fakeEntry.partition.cprange.isValid = CP_INVALID; entries.push_back(fakeEntry); @@ -5475,8 +5470,8 @@ void ExtentMap::lookup(OID_t OID, LBIDRange_v& ranges) fakeEntry.dbRoot = 1; fakeEntry.colWid = 4; fakeEntry.status = EXTENTAVAILABLE; - fakeEntry.partition.cprange.hi_val = numeric_limits::min() + 2; - fakeEntry.partition.cprange.lo_val = numeric_limits::max(); + fakeEntry.partition.cprange.hiVal = numeric_limits::min() + 2; + fakeEntry.partition.cprange.loVal = numeric_limits::max(); fakeEntry.partition.cprange.sequenceNum = 0; fakeEntry.partition.cprange.isValid = CP_INVALID; #endif @@ -6011,8 +6006,8 @@ void ExtentMap::dumpTo(ostream& os) << fExtentMap[i].dbRoot << '|' << fExtentMap[i].colWid << '|' << fExtentMap[i].status << '|' - << fExtentMap[i].partition.cprange.hi_val << '|' - << fExtentMap[i].partition.cprange.lo_val << '|' + << fExtentMap[i].partition.cprange.hiVal << '|' + << fExtentMap[i].partition.cprange.loVal << '|' << fExtentMap[i].partition.cprange.sequenceNum << '|' << (int)fExtentMap[i].partition.cprange.isValid << '|' << endl; diff --git a/versioning/BRM/extentmap.h b/versioning/BRM/extentmap.h index 3aa0da0ca..7a5425935 100644 --- a/versioning/BRM/extentmap.h +++ b/versioning/BRM/extentmap.h @@ -133,13 +133,11 @@ struct EMEntry_v4 // and max values for casual partitioning. struct EMCasualPartition_struct { - RangePartitionData_t hi_val; // This needs to be reinterpreted as unsigned for uint64_t column types. - RangePartitionData_t lo_val; int32_t sequenceNum; char isValid; //CP_INVALID - No min/max and no DML in progress. CP_UPDATING - Update in progress. CP_VALID- min/max is valid union { - __int128 bigLoVal; + __int128 bigLoVal; // These need to be reinterpreted as unsigned for uint64_t/uint128_t column types. int64_t loVal; }; union From c4d8516a4739fb469ca8c1be156e590880db2584 Mon Sep 17 00:00:00 2001 From: David Hall Date: Mon, 10 Aug 2020 13:59:49 -0500 Subject: [PATCH 54/78] MCOL-4171 Window functions with decimal(38) --- datatypes/mcs_decimal.h | 89 ++++++++++++++++++--- dbcon/execplan/windowfunctioncolumn.cpp | 7 +- utils/rowgroup/rowgroup.h | 4 +- utils/windowfunction/windowfunctiontype.cpp | 59 ++++---------- 4 files changed, 97 insertions(+), 62 deletions(-) diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index 1fac9485e..11d463f95 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -24,6 +24,7 @@ #include "calpontsystemcatalog.h" using int128_t = __int128; +using uint128_t = unsigned __int128; using ColTypeAlias = execplan::CalpontSystemCatalog::ColType; using ColDataTypeAlias = execplan::CalpontSystemCatalog::ColDataType; @@ -32,6 +33,45 @@ namespace execplan struct IDB_Decimal; } +// A class by Fabio Fernandes pulled off of stackoverflow +// Creates a type _xxl that can be used to create 128bit constant values +// Ex: int128_t i128 = 12345678901234567890123456789_xxl +namespace detail_xxl +{ + constexpr uint8_t hexval(char c) + { return c>='a' ? (10+c-'a') : c>='A' ? (10+c-'A') : c-'0'; } + + template + constexpr uint128_t lit_eval() { return V; } + + template + constexpr uint128_t lit_eval() { + static_assert( BASE!=16 || sizeof...(Cs) <= 32-1, "Literal too large for BASE=16"); + static_assert( BASE!=10 || sizeof...(Cs) <= 39-1, "Literal too large for BASE=10"); + static_assert( BASE!=8 || sizeof...(Cs) <= 44-1, "Literal too large for BASE=8"); + static_assert( BASE!=2 || sizeof...(Cs) <= 128-1, "Literal too large for BASE=2"); + return lit_eval(); + } + + template struct LitEval + {static constexpr uint128_t eval() {return lit_eval<10,0,Cs...>();} }; + + template struct LitEval<'0','x',Cs...> + {static constexpr uint128_t eval() {return lit_eval<16,0,Cs...>();} }; + + template struct LitEval<'0','b',Cs...> + {static constexpr uint128_t eval() {return lit_eval<2,0,Cs...>();} }; + + template struct LitEval<'0',Cs...> + {static constexpr uint128_t eval() {return lit_eval<8,0,Cs...>();} }; + + template + constexpr uint128_t operator "" _xxl() {return LitEval::eval();} +} + +template +constexpr uint128_t operator "" _xxl() {return ::detail_xxl::operator "" _xxl();} + namespace datatypes { @@ -65,6 +105,29 @@ const uint64_t mcs_pow_10[20] = 1000000000000000000ULL, 10000000000000000000ULL }; +const uint128_t mcs_pow_10_128[20] = +{ + 10000000000000000000_xxl, + 100000000000000000000_xxl, + 1000000000000000000000_xxl, + 10000000000000000000000_xxl, + 100000000000000000000000_xxl, + 1000000000000000000000000_xxl, + 10000000000000000000000000_xxl, + 100000000000000000000000000_xxl, + 1000000000000000000000000000_xxl, + 10000000000000000000000000000_xxl, + 100000000000000000000000000000_xxl, + 1000000000000000000000000000000_xxl, + 10000000000000000000000000000000_xxl, + 100000000000000000000000000000000_xxl, + 1000000000000000000000000000000000_xxl, + 10000000000000000000000000000000000_xxl, + 100000000000000000000000000000000000_xxl, + 1000000000000000000000000000000000000_xxl, + 10000000000000000000000000000000000000_xxl, + 100000000000000000000000000000000000000_xxl +}; constexpr uint32_t maxPowOf10 = sizeof(mcs_pow_10)/sizeof(mcs_pow_10[0])-1; constexpr int128_t Decimal128Null = int128_t(0x8000000000000000LL) << 64; @@ -75,21 +138,21 @@ constexpr int128_t Decimal128Empty = (int128_t(0x8000000000000000LL) << 64) + 1; @brief The function to produce scale multiplier/divisor for wide decimals. */ -inline void getScaleDivisor(int128_t& divisor, const int8_t scale) +template +inline void getScaleDivisor(T& divisor, const int8_t scale) { - divisor = 1; - switch (scale/maxPowOf10) + if (scale < 0) { - case 2: - divisor *= mcs_pow_10[maxPowOf10]; - divisor *= mcs_pow_10[maxPowOf10]; - break; - case 1: - divisor *= mcs_pow_10[maxPowOf10]; - case 0: - divisor *= mcs_pow_10[scale%maxPowOf10]; - default: - break; + std::string msg = "getScaleDivisor called with negative scale: " + std::to_string(scale); + throw std::invalid_argument(msg); + } + if (scale < 19) + { + divisor = mcs_pow_10[scale]; + } + else + { + divisor = mcs_pow_10_128[scale-18]; } } diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index d32e41651..21f09aa06 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -361,7 +361,9 @@ bool WindowFunctionColumn::hasWindowFunc() void WindowFunctionColumn::adjustResultType() { - if (fResultType.colDataType == CalpontSystemCatalog::DECIMAL && + if ((fResultType.colDataType == CalpontSystemCatalog::DECIMAL || + fResultType.colDataType == CalpontSystemCatalog::UDECIMAL) + && !boost::iequals(fFunctionName, "COUNT") && !boost::iequals(fFunctionName, "COUNT(*)") && !boost::iequals(fFunctionName, "ROW_NUMBER") && @@ -389,7 +391,8 @@ void WindowFunctionColumn::adjustResultType() boost::iequals(fFunctionName, "AVG") || boost::iequals(fFunctionName, "AVG_DISTINCT")) { - if (fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::DECIMAL) + if (fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::DECIMAL || + fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::UDECIMAL) { fResultType.colWidth = sizeof(int128_t); } diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 31712c2ad..049520b95 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -380,7 +380,7 @@ public: template inline bool equals(uint64_t val, uint32_t colIndex) const; inline bool equals(long double val, uint32_t colIndex) const; bool equals(const std::string& val, uint32_t colIndex) const; - inline bool equals(int128_t val, uint32_t colIndex) const; + inline bool equals(const int128_t& val, uint32_t colIndex) const; inline double getDoubleField(uint32_t colIndex) const; inline float getFloatField(uint32_t colIndex) const; @@ -709,7 +709,7 @@ inline bool Row::equals(long double val, uint32_t colIndex) const return *((long double*) &data[offsets[colIndex]]) == val; } -inline bool Row::equals(int128_t val, uint32_t colIndex) const +inline bool Row::equals(const int128_t& val, uint32_t colIndex) const { return *((int128_t*) &data[offsets[colIndex]]) == val; } diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index 6356a04bb..7d7329e48 100644 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -40,7 +40,6 @@ using namespace logging; using namespace ordering; #include "calpontsystemcatalog.h" -#include "dataconvert.h" // int64_t IDB_pow[19] using namespace execplan; #include "windowfunctionstep.h" @@ -59,6 +58,7 @@ using namespace joblist; #include "wf_stats.h" #include "wf_sum_avg.h" #include "wf_udaf.h" +#include "mcs_decimal.h" namespace windowfunction { @@ -480,7 +480,7 @@ template void WindowFunctionType::implicit2T(uint64_t i, T& t, int s) { int ct = fRow.getColType(i); - int pw = 0; + int64_t divisor = 1; switch (ct) { @@ -491,13 +491,6 @@ void WindowFunctionType::implicit2T(uint64_t i, T& t, int s) case CalpontSystemCatalog::BIGINT: { t = (T) fRow.getIntField(i); - pw = s - fRow.getScale(i); // pw is difference of scales, will be in [-18, 18] - - if (pw > 0) - t *= IDB_pow[pw]; - else if (pw < 0) - t /= IDB_pow[-pw]; - break; } @@ -508,13 +501,6 @@ void WindowFunctionType::implicit2T(uint64_t i, T& t, int s) case CalpontSystemCatalog::UBIGINT: { t = (T) fRow.getUintField(i); - pw = s - fRow.getScale(i); // pw is difference of scales, will be in [-18, 18] - - if (pw > 0) - t *= IDB_pow[pw]; - else if (pw < 0) - t /= IDB_pow[-pw]; - break; } @@ -525,13 +511,6 @@ void WindowFunctionType::implicit2T(uint64_t i, T& t, int s) t = (T) fRow.getIntField(i); else t = (T) fRow.getInt128Field(i); - pw = s - fRow.getScale(i); // pw is difference of scales, will be in [-18, 18] - - if (pw > 0) - t *= IDB_pow[pw]; - else if (pw < 0) - t /= IDB_pow[-pw]; - break; } @@ -542,45 +521,26 @@ void WindowFunctionType::implicit2T(uint64_t i, T& t, int s) t = (T) fRow.getUintField(i); else t = (T) fRow.getUint128Field(i); - pw = s - fRow.getScale(i); // pw is difference of scales, will be in [-18, 18] - - if (pw > 0) - t *= IDB_pow[pw]; - else if (pw < 0) - t /= IDB_pow[-pw]; - break; } case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { - if (s == 0) - t = (T) fRow.getDoubleField(i); - else - t = (T) (fRow.getDoubleField(i) * IDB_pow[s]); // s is scale, [0, 18] - + t = (T) fRow.getDoubleField(i); break; } case CalpontSystemCatalog::FLOAT: case CalpontSystemCatalog::UFLOAT: { - if (s == 0) - t = (T) fRow.getFloatField(i); - else - t = (T) (fRow.getFloatField(i) * IDB_pow[s]); // s is scale, [0, 18] - + t = (T) fRow.getFloatField(i); break; } case CalpontSystemCatalog::LONGDOUBLE: { - if (s == 0) - t = (T) fRow.getLongDoubleField(i); - else - t = (T) (fRow.getLongDoubleField(i) * IDB_pow[s]); // s is scale, [0, 18] - + t = (T) fRow.getLongDoubleField(i); break; } @@ -596,6 +556,15 @@ void WindowFunctionType::implicit2T(uint64_t i, T& t, int s) break; } } + + s -= fRow.getScale(i); // we scale only the difference of scales + datatypes::getScaleDivisor(divisor, abs(s)); + if (s > 0) + t *= divisor; + else if (s < 0) + t /= divisor; + + } template<> From 2d044fd7f1c43792e27042217694a8c2e80417b1 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 11 Aug 2020 13:16:17 -0500 Subject: [PATCH 55/78] MCOL-4171 Fix comments in bytestream.h --- utils/messageqcpp/bytestream.h | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/utils/messageqcpp/bytestream.h b/utils/messageqcpp/bytestream.h index 35a85e3bc..3eccb7b0f 100644 --- a/utils/messageqcpp/bytestream.h +++ b/utils/messageqcpp/bytestream.h @@ -47,7 +47,6 @@ class ByteStreamTestSuite; using int128_t = __int128; using uint128_t = unsigned __int128; -using int128_t = __int128; namespace messageqcpp { @@ -149,17 +148,13 @@ public: */ EXPORT ByteStream& operator<<(const uint64_t o); /** - * push an uint128_t onto the end of the stream. The byte order is whatever the native byte order is. + * push an int128_t onto the end of the stream. The byte order is whatever the native byte order is. */ EXPORT ByteStream& operator<<(const int128_t& o); /** * push an uint128_t onto the end of the stream. The byte order is whatever the native byte order is. */ EXPORT ByteStream& operator<<(const uint128_t& o); - /** - * push an int128_t onto the end of the stream. The byte order is whatever the native byte order is. - */ - EXPORT ByteStream& operator<<(const int128_t& o); /** * push a float onto the end of the stream. The byte order is * whatever the native byte order is. @@ -225,17 +220,13 @@ public: */ EXPORT ByteStream& operator>>(uint64_t& o); /** - * extract an uint128_t from the front of the stream. The byte order is whatever the native byte order is. + * extract an int128_t from the front of the stream. The byte order is whatever the native byte order is. */ EXPORT ByteStream& operator>>(int128_t& o); /** * extract an uint128_t from the front of the stream. The byte order is whatever the native byte order is. */ EXPORT ByteStream& operator>>(uint128_t& o); - /** - * extract an int128_t from the front of the stream. The byte order is whatever the native byte order is. - */ - EXPORT ByteStream& operator>>(int128_t& o); /** * extract a float from the front of the stream. The byte * order is whatever the native byte order is. @@ -307,17 +298,13 @@ public: */ EXPORT void peek(uint64_t& o) const; /** - * Peek at an uint128_t from the front of the stream. The byte order is whatever the native byte order is. + * Peek at an int128_t from the front of the stream. The byte order is whatever the native byte order is. */ EXPORT void peek(int128_t& o) const; /** * Peek at an uint128_t from the front of the stream. The byte order is whatever the native byte order is. */ EXPORT void peek(uint128_t& o) const; - /** - * Peek at an int128_t from the front of the stream. The byte order is whatever the native byte order is. - */ - EXPORT void peek(int128_t& o) const; /** * Peek at a float from the front of the stream. The byte order * is whatever the native byte order is. From 5b8aba0005785d68698b70f961394c3447f434c3 Mon Sep 17 00:00:00 2001 From: David Hall Date: Tue, 11 Aug 2020 17:06:20 -0500 Subject: [PATCH 56/78] MCOL-4171 use const ref when passing int128 It saves a pico second or two. --- utils/windowfunction/windowfunctiontype.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/windowfunction/windowfunctiontype.h b/utils/windowfunction/windowfunctiontype.h index 1c7dea375..995ca5718 100644 --- a/utils/windowfunction/windowfunctiontype.h +++ b/utils/windowfunction/windowfunctiontype.h @@ -242,7 +242,7 @@ protected: { fRow.setIntField(v, i); } - void setInt128Value(int64_t i, int128_t v) + void setInt128Value(int64_t i, const int128_t& v) { fRow.setInt128Field(v, i); } From af80081c94c74b303a9a1a7196984d62432abb81 Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 13 Aug 2020 09:54:01 -0500 Subject: [PATCH 57/78] MCOL-4171 Some fixes --- datatypes/mcs_decimal.h | 11 ++++--- utils/common/hasher.h | 1 + utils/rowgroup/rowgroup.h | 14 ++++++--- utils/windowfunction/wf_lead_lag.cpp | 17 +++++------ utils/windowfunction/wf_nth_value.cpp | 17 ++--------- utils/windowfunction/wf_ntile.cpp | 5 +--- utils/windowfunction/wf_percentile.cpp | 4 +-- utils/windowfunction/wf_sum_avg.cpp | 10 ++++--- utils/windowfunction/wf_udaf.cpp | 2 +- utils/windowfunction/windowfunctiontype.cpp | 33 ++++++++++++++++----- utils/windowfunction/windowfunctiontype.h | 2 +- writeengine/wrapper/writeengine.cpp | 3 ++ 12 files changed, 65 insertions(+), 54 deletions(-) diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index 11d463f95..b87db7ebf 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -82,7 +82,7 @@ constexpr uint8_t MAXLEGACYWIDTH = 8U; constexpr uint8_t MAXSCALEINC4AVG = 4U; constexpr int8_t IGNOREPRECISION = -1; -const uint64_t mcs_pow_10[20] = +const int64_t mcs_pow_10[19] = { 1ULL, 10ULL, @@ -103,9 +103,8 @@ const uint64_t mcs_pow_10[20] = 10000000000000000ULL, 100000000000000000ULL, 1000000000000000000ULL, - 10000000000000000000ULL }; -const uint128_t mcs_pow_10_128[20] = +const int128_t mcs_pow_10_128[20] = { 10000000000000000000_xxl, 100000000000000000000_xxl, @@ -126,7 +125,7 @@ const uint128_t mcs_pow_10_128[20] = 100000000000000000000000000000000000_xxl, 1000000000000000000000000000000000000_xxl, 10000000000000000000000000000000000000_xxl, - 100000000000000000000000000000000000000_xxl + 100000000000000000000000000000000000000_xxl, }; constexpr uint32_t maxPowOf10 = sizeof(mcs_pow_10)/sizeof(mcs_pow_10[0])-1; @@ -146,13 +145,13 @@ inline void getScaleDivisor(T& divisor, const int8_t scale) std::string msg = "getScaleDivisor called with negative scale: " + std::to_string(scale); throw std::invalid_argument(msg); } - if (scale < 19) + else if (scale < 19) { divisor = mcs_pow_10[scale]; } else { - divisor = mcs_pow_10_128[scale-18]; + divisor = mcs_pow_10_128[scale-19]; } } diff --git a/utils/common/hasher.h b/utils/common/hasher.h index 27d7fbd01..08b6b6ac1 100644 --- a/utils/common/hasher.h +++ b/utils/common/hasher.h @@ -31,6 +31,7 @@ #include using int128_t = __int128; +using uint128_t = unsigned __int128; namespace utils { diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 049520b95..669220c21 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -419,8 +419,9 @@ public: inline void setDoubleField(double val, uint32_t colIndex); inline void setFloatField(float val, uint32_t colIndex); inline void setDecimalField(double val, uint32_t colIndex) { }; // TODO: Do something here - inline void setLongDoubleField(long double val, uint32_t colIndex); - inline void setInt128Field(int128_t val, uint32_t colIndex); + inline void setLongDoubleField(const long double& val, uint32_t colIndex); + inline void setInt128Field(const int128_t& val, uint32_t colIndex); + inline void setUint128Field(const uint128_t& val, uint32_t colIndex); inline void setRid(uint64_t rid); @@ -1167,7 +1168,7 @@ inline void Row::setFloatField(float val, uint32_t colIndex) *((float*) &data[offsets[colIndex]]) = val; } -inline void Row::setLongDoubleField(long double val, uint32_t colIndex) +inline void Row::setLongDoubleField(const long double& val, uint32_t colIndex) { uint8_t* p = &data[offsets[colIndex]]; *((long double*)p) = val; @@ -1178,11 +1179,16 @@ inline void Row::setLongDoubleField(long double val, uint32_t colIndex) } } -inline void Row::setInt128Field(int128_t val, uint32_t colIndex) +inline void Row::setInt128Field(const int128_t& val, uint32_t colIndex) { *((int128_t*)&data[offsets[colIndex]]) = val; } +inline void Row::setUint128Field(const uint128_t& val, uint32_t colIndex) +{ + *((uint128_t*)&data[offsets[colIndex]]) = val; +} + inline void Row::setVarBinaryField(const std::string& val, uint32_t colIndex) { if (inStringTable(colIndex)) diff --git a/utils/windowfunction/wf_lead_lag.cpp b/utils/windowfunction/wf_lead_lag.cpp index 62fab334b..f8aa32ca1 100644 --- a/utils/windowfunction/wf_lead_lag.cpp +++ b/utils/windowfunction/wf_lead_lag.cpp @@ -191,12 +191,15 @@ void WF_lead_lag::parseParms(const std::vector& parms) // parms[3]: respect null | ignore null cc = dynamic_cast(parms[3].get()); - idbassert(cc != NULL); - bool isNull = false; // dummy, harded coded - fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); + if (cc != NULL) + { + bool isNull = false; // dummy. Return not used + fRespectNulls = (cc->getIntVal(fRow, isNull) > 0); + } } + template void WF_lead_lag::operator()(int64_t b, int64_t e, int64_t c) { @@ -220,13 +223,7 @@ void WF_lead_lag::operator()(int64_t b, int64_t e, int64_t c) if (!fOffsetNull) { implicit2T(idx, tmp, 0); - - if (tmp > e) // prevent integer overflow - tmp = e + 1; - else if (tmp + e < 0) - tmp += e - 1; - - fOffset = (int64_t) tmp; + fOffset = round(tmp); fOffset *= fLead; } } diff --git a/utils/windowfunction/wf_nth_value.cpp b/utils/windowfunction/wf_nth_value.cpp index 9c566345f..73b0bf1be 100644 --- a/utils/windowfunction/wf_nth_value.cpp +++ b/utils/windowfunction/wf_nth_value.cpp @@ -215,23 +215,10 @@ void WF_nth_value::operator()(int64_t b, int64_t e, int64_t c) if (!fNthNull) { implicit2T(idx, tmp, 0); - - if (tmp <= 0) - { - ostringstream oss; - oss << tmp; - throw IDBExcept(IDBErrorInfo::instance()->errorMsg(ERR_WF_ARG_OUT_OF_RANGE, - oss.str()), ERR_WF_ARG_OUT_OF_RANGE); - } - - if (tmp > e) // prevent integer overflow - tmp = e + 1; - - fNth = (int64_t) tmp; + fNth = round(tmp); } } - bool isNull = true; if ((!fNthNull) && ((b + fNth - 1) <= e)) @@ -256,7 +243,7 @@ void WF_nth_value::operator()(int64_t b, int64_t e, int64_t c) int64_t n = k + fNth - 1; - if (n <= e) + if (n <= e && n >= 0) { fRow.setData(getPointer(fRowData->at(n))); getValue(colIn, fValue); diff --git a/utils/windowfunction/wf_ntile.cpp b/utils/windowfunction/wf_ntile.cpp index 9361609a6..716b10a2c 100644 --- a/utils/windowfunction/wf_ntile.cpp +++ b/utils/windowfunction/wf_ntile.cpp @@ -118,10 +118,7 @@ void WF_ntile::operator()(int64_t b, int64_t e, int64_t c) oss.str()), ERR_WF_ARG_OUT_OF_RANGE); } - if (tmp > e) // prevent integer overflow - tmp = e + 1; - - fNtile = (int64_t) tmp; + fNtile = round(tmp); } } diff --git a/utils/windowfunction/wf_percentile.cpp b/utils/windowfunction/wf_percentile.cpp index 043e36c08..8aaeba9f6 100644 --- a/utils/windowfunction/wf_percentile.cpp +++ b/utils/windowfunction/wf_percentile.cpp @@ -92,7 +92,7 @@ boost::shared_ptr WF_percentile::makeFunction(int id, con { if (wc->functionParms()[0]->resultType().colWidth < 16) { - func.reset(new WF_percentile(id, name)); + func.reset(new WF_percentile(id, name)); } else { @@ -105,7 +105,7 @@ boost::shared_ptr WF_percentile::makeFunction(int id, con { if (wc->functionParms()[0]->resultType().colWidth < 16) { - func.reset(new WF_percentile(id, name)); + func.reset(new WF_percentile(id, name)); } else { diff --git a/utils/windowfunction/wf_sum_avg.cpp b/utils/windowfunction/wf_sum_avg.cpp index 0ad1c1a30..b61fb9928 100644 --- a/utils/windowfunction/wf_sum_avg.cpp +++ b/utils/windowfunction/wf_sum_avg.cpp @@ -85,8 +85,9 @@ inline void WF_sum_avg::checkSumLimit(uint128_t val, uint128_t sum) template int128_t WF_sum_avg::calculateAvg(int128_t sum, uint64_t count, int scale) { - int128_t factor = pow(10.0, scale); - int128_t avg; + int128_t avg = 0; + int128_t factor; + datatypes::getScaleDivisor(factor, scale); if (scale > 0) { if ((sum * factor) / factor == sum) @@ -120,8 +121,9 @@ int128_t WF_sum_avg::calculateAvg(int128_t sum, uint64_t count, int template uint128_t WF_sum_avg::calculateAvg(uint128_t sum, uint64_t count, int scale) { - uint128_t factor = pow(10.0, scale); - uint128_t avg = sum; + uint128_t avg = 0; + uint128_t factor; + datatypes::getScaleDivisor(factor, scale); if (scale > 0) { if ((sum * factor) / factor == sum) diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index fb5f3521a..62d4a2714 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -550,7 +550,7 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, static const static_any::any& uintTypeId = (unsigned int)1; static const static_any::any& ulongTypeId = (unsigned long)1; static const static_any::any& ullTypeId = (unsigned long long)1; - static const static_any::any& uint128TypeId = (int128_t)1; + static const static_any::any& uint128TypeId = (uint128_t)1; static const static_any::any& floatTypeId = (float)1; static const static_any::any& doubleTypeId = (double)1; static const std::string typeStr(""); diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index 7d7329e48..17aa122a9 100644 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -362,6 +362,11 @@ template<> void WindowFunctionType::setValue(uint64_t i, int128_t& t) fRow.setInt128Field(t, i); } +template<> void WindowFunctionType::setValue(uint64_t i, uint128_t& t) +{ + fRow.setUint128Field(t, i); +} + template<> void WindowFunctionType::setValue(uint64_t i, string& t) { fRow.setStringField(t, i); @@ -433,15 +438,31 @@ void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, T* v) case CalpontSystemCatalog::DECIMAL: { - int128_t iv = *v; - setValue(i, iv); + if (sizeof(T) == 8) + { + int64_t iv = *v; + setValue(i, iv); + } + else + { + int128_t iv = *v; + setValue(i, iv); + } break; } case CalpontSystemCatalog::UDECIMAL: { - uint128_t uv = *v; - setValue(i, uv); + if (sizeof(T) == 8) + { + uint64_t iv = *v; + setValue(i, iv); + } + else + { + uint128_t iv = *v; + setValue(i, iv); + } break; } @@ -480,7 +501,6 @@ template void WindowFunctionType::implicit2T(uint64_t i, T& t, int s) { int ct = fRow.getColType(i); - int64_t divisor = 1; switch (ct) { @@ -557,14 +577,13 @@ void WindowFunctionType::implicit2T(uint64_t i, T& t, int s) } } + T divisor = 1; s -= fRow.getScale(i); // we scale only the difference of scales datatypes::getScaleDivisor(divisor, abs(s)); if (s > 0) t *= divisor; else if (s < 0) t /= divisor; - - } template<> diff --git a/utils/windowfunction/windowfunctiontype.h b/utils/windowfunction/windowfunctiontype.h index 995ca5718..e08bc0611 100644 --- a/utils/windowfunction/windowfunctiontype.h +++ b/utils/windowfunction/windowfunctiontype.h @@ -250,7 +250,7 @@ protected: { fRow.setDoubleField(v, i); } - void setLongDoubleValue(int64_t i, long double v) + void setLongDoubleValue(int64_t i, const long double& v) { fRow.setLongDoubleField(v, i); } diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index e75ee794e..163f64aa3 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -845,6 +845,9 @@ inline void allocateValArray(void*& valArray, ColTupleList::size_type totalRow, case WriteEngine::WR_TEXT: valArray = (char*) calloc(sizeof(char), totalRow * MAX_COLUMN_BOUNDARY); break; + case WriteEngine::WR_TOKEN: + valArray = (Token*) calloc(sizeof(Token), totalRow); + break; default: valArray = calloc(totalRow, colWidth); break; From 1c3a34a3d0e336f767742e1d89614fd2e7f74c0c Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Thu, 27 Aug 2020 17:04:29 +0000 Subject: [PATCH 58/78] Dataconvert::decimalToString badly fails w/o 20th member of mcs_pow_10 so I returned it WF::percentile runtime threw an exception b/c of wrong DT deduced from its argument Replaced literals with constants Tought WF_sum_avg::checkSumLimit to use refs instead of values --- datatypes/mcs_decimal.h | 3 +- dbcon/execplan/windowfunctioncolumn.cpp | 7 +- utils/regr/moda.cpp | 3 +- utils/rowgroup/rowgroup.h | 4 + utils/windowfunction/idborderby.cpp | 4 +- utils/windowfunction/wf_count.cpp | 4 +- utils/windowfunction/wf_lead_lag.cpp | 4 +- utils/windowfunction/wf_nth_value.cpp | 18 +--- utils/windowfunction/wf_percentile.cpp | 20 +--- utils/windowfunction/wf_stats.cpp | 18 +--- utils/windowfunction/wf_sum_avg.cpp | 115 ++++++++--------------- utils/windowfunction/wf_sum_avg.h | 17 ++-- writeengine/server/we_dmlcommandproc.cpp | 2 +- 13 files changed, 76 insertions(+), 143 deletions(-) diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index b87db7ebf..94aa2a50b 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -82,7 +82,7 @@ constexpr uint8_t MAXLEGACYWIDTH = 8U; constexpr uint8_t MAXSCALEINC4AVG = 4U; constexpr int8_t IGNOREPRECISION = -1; -const int64_t mcs_pow_10[19] = +const uint64_t mcs_pow_10[20] = { 1ULL, 10ULL, @@ -103,6 +103,7 @@ const int64_t mcs_pow_10[19] = 10000000000000000ULL, 100000000000000000ULL, 1000000000000000000ULL, + 10000000000000000000ULL, }; const int128_t mcs_pow_10_128[20] = { diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index 21f09aa06..a6ba127fa 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -52,8 +52,6 @@ using namespace rowgroup; #include "joblisttypes.h" using namespace joblist; -#include "widedecimalutils.h" - #ifdef _MSC_VER #define strcasecmp stricmp #endif @@ -389,7 +387,8 @@ void WindowFunctionColumn::adjustResultType() if (boost::iequals(fFunctionName, "SUM") || boost::iequals(fFunctionName, "AVG") || - boost::iequals(fFunctionName, "AVG_DISTINCT")) + boost::iequals(fFunctionName, "AVG_DISTINCT") || + boost::iequals(fFunctionName, "PERCENTILE")) { if (fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::DECIMAL || fFunctionParms[0]->resultType().colDataType == CalpontSystemCatalog::UDECIMAL) @@ -689,7 +688,7 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull) case 16: { int128_t dec = row.getInt128Field(fInputIndex); - if (utils::isWideDecimalNullValue(dec)) + if (dec == datatypes::Decimal128Null) isNull = true; else { diff --git a/utils/regr/moda.cpp b/utils/regr/moda.cpp index e1bc44a9a..7bc5b427d 100644 --- a/utils/regr/moda.cpp +++ b/utils/regr/moda.cpp @@ -21,6 +21,7 @@ #include "moda.h" #include "bytestream.h" #include "objectreader.h" +#include "columnwidth.h" using namespace mcsv1sdk; @@ -186,7 +187,7 @@ mcsv1_UDAF::ReturnCode moda::init(mcsv1Context* context, { context->setColWidth(8); } - else + else if (utils::widthByPrecision(colTypes[0].precision)) { context->setColWidth(16); } diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 669220c21..748bf3dfa 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -962,11 +962,15 @@ inline long double Row::getLongDoubleField(uint32_t colIndex) const return *((long double*) &data[offsets[colIndex]]); } +// !!! Never ever try to remove inline from this f() b/c it returns +// non-integral 16 byte DT inline int128_t Row::getInt128Field(uint32_t colIndex) const { return *((int128_t*) &data[offsets[colIndex]]); } +// !!! Never ever try to remove inline from this f() b/c it returns +// non-integral 16 byte DT inline uint128_t Row::getUint128Field(uint32_t colIndex) const { return *((uint128_t*) &data[offsets[colIndex]]); diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index 6814ed4ad..1028d3e1d 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -856,11 +856,11 @@ bool EqualCompData::operator()(Row::Pointer a, Row::Pointer b) case CalpontSystemCatalog::UDECIMAL: { // equal compare. ignore sign and null - if (fRow1.getColumnWidth(*i) < 16) + if (fRow1.getColumnWidth(*i) < datatypes::MAXDECIMALWIDTH) { eq = (fRow1.getUintField(*i) == fRow2.getUintField(*i)); } - else + else if (fRow1.getColumnWidth(*i) == datatypes::MAXDECIMALWIDTH) { eq = (fRow1.getUint128Field(*i) == fRow2.getUint128Field(*i)); } diff --git a/utils/windowfunction/wf_count.cpp b/utils/windowfunction/wf_count.cpp index d7e9b104e..29987c3b4 100644 --- a/utils/windowfunction/wf_count.cpp +++ b/utils/windowfunction/wf_count.cpp @@ -78,11 +78,11 @@ boost::shared_ptr WF_count::makeFunction(int id, const st case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - if (wc->functionParms()[0]->resultType().colWidth < 16) + if (wc->functionParms()[0]->resultType().colWidth < datatypes::MAXDECIMALWIDTH) { func.reset(new WF_count(id, name)); } - else + else if (wc->functionParms()[0]->resultType().colWidth == datatypes::MAXDECIMALWIDTH) { func.reset(new WF_count(id, name)); } diff --git a/utils/windowfunction/wf_lead_lag.cpp b/utils/windowfunction/wf_lead_lag.cpp index f8aa32ca1..1a1b453cd 100644 --- a/utils/windowfunction/wf_lead_lag.cpp +++ b/utils/windowfunction/wf_lead_lag.cpp @@ -85,11 +85,11 @@ boost::shared_ptr WF_lead_lag::makeFunction(int id, const case CalpontSystemCatalog::DECIMAL: { - if (wc->functionParms()[0]->resultType().colWidth < 16) + if (wc->functionParms()[0]->resultType().colWidth < datatypes::MAXDECIMALWIDTH) { func.reset(new WF_lead_lag(id, name)); } - else + else if (wc->functionParms()[0]->resultType().colWidth == datatypes::MAXDECIMALWIDTH) { func.reset(new WF_lead_lag(id, name)); } diff --git a/utils/windowfunction/wf_nth_value.cpp b/utils/windowfunction/wf_nth_value.cpp index 73b0bf1be..41dde88fb 100644 --- a/utils/windowfunction/wf_nth_value.cpp +++ b/utils/windowfunction/wf_nth_value.cpp @@ -84,31 +84,19 @@ boost::shared_ptr WF_nth_value::makeFunction(int id, cons } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { - if (wc->functionParms()[0]->resultType().colWidth < 16) + if (wc->functionParms()[0]->resultType().colWidth < datatypes::MAXDECIMALWIDTH) { func.reset(new WF_nth_value(id, name)); } - else + else if (wc->functionParms()[0]->resultType().colWidth == datatypes::MAXDECIMALWIDTH) { func.reset(new WF_nth_value(id, name)); } break; } - case CalpontSystemCatalog::UDECIMAL: - { - if (wc->functionParms()[0]->resultType().colWidth < 16) - { - func.reset(new WF_nth_value(id, name)); - } - else - { - func.reset(new WF_nth_value(id, name)); - } - break; - } - case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { diff --git a/utils/windowfunction/wf_percentile.cpp b/utils/windowfunction/wf_percentile.cpp index 8aaeba9f6..a407d41e8 100644 --- a/utils/windowfunction/wf_percentile.cpp +++ b/utils/windowfunction/wf_percentile.cpp @@ -46,6 +46,8 @@ using namespace ordering; #include "constantcolumn.h" using namespace execplan; +#include "mcs_decimal.h" + #include "windowfunctionstep.h" using namespace joblist; @@ -89,31 +91,19 @@ boost::shared_ptr WF_percentile::makeFunction(int id, con } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { - if (wc->functionParms()[0]->resultType().colWidth < 16) + if (wc->resultType().colWidth < datatypes::MAXDECIMALWIDTH) { func.reset(new WF_percentile(id, name)); } - else + else if (wc->resultType().colWidth == datatypes::MAXDECIMALWIDTH) { func.reset(new WF_percentile(id, name)); } break; } - case CalpontSystemCatalog::UDECIMAL: - { - if (wc->functionParms()[0]->resultType().colWidth < 16) - { - func.reset(new WF_percentile(id, name)); - } - else - { - func.reset(new WF_percentile(id, name)); - } - break; - } - case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { diff --git a/utils/windowfunction/wf_stats.cpp b/utils/windowfunction/wf_stats.cpp index 1804ad432..0dee4fa9e 100644 --- a/utils/windowfunction/wf_stats.cpp +++ b/utils/windowfunction/wf_stats.cpp @@ -82,30 +82,18 @@ boost::shared_ptr WF_stats::makeFunction(int id, const st } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { - if (wc->functionParms()[0]->resultType().colWidth < 16) + if (wc->functionParms()[0]->resultType().colWidth < datatypes::MAXDECIMALWIDTH) { func.reset(new WF_stats(id, name)); } - else + else if (wc->functionParms()[0]->resultType().colWidth == datatypes::MAXDECIMALWIDTH) { func.reset(new WF_stats(id, name)); } break; } - - case CalpontSystemCatalog::UDECIMAL: - { - if (wc->functionParms()[0]->resultType().colWidth < 16) - { - func.reset(new WF_stats(id, name)); - } - else - { - func.reset(new WF_stats(id, name)); - } - break; - } case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: diff --git a/utils/windowfunction/wf_sum_avg.cpp b/utils/windowfunction/wf_sum_avg.cpp index b61fb9928..007bb1fc7 100644 --- a/utils/windowfunction/wf_sum_avg.cpp +++ b/utils/windowfunction/wf_sum_avg.cpp @@ -52,35 +52,49 @@ using namespace joblist; namespace windowfunction { -template -void WF_sum_avg::checkSumLimit(long double val, long double sum) -{ -} template -inline void WF_sum_avg::checkSumLimit(int128_t val, int128_t sum) +inline void WF_sum_avg::checkSumLimit(T_IN& val, T_OUT& sum) +{ } + +template<> +inline void WF_sum_avg::checkSumLimit(int128_t& val, int128_t& sum) { - if (((sum >= 0) && ((fMax128 - sum) < val)) || - ((sum < 0) && ((fMin128 - sum) > val))) - { - string errStr = "SUM(int128_t)"; - errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_OVERFLOW, errStr); - cerr << errStr << endl; - throw IDBExcept(errStr, ERR_WF_OVERFLOW); - } + datatypes::AdditionOverflowCheck ofCheckOp; + ofCheckOp(sum, val); } -template -inline void WF_sum_avg::checkSumLimit(uint128_t val, uint128_t sum) -{ - if ((fMaxu128 - sum) < val) - { - string errStr = "SUM(uint128_t)"; - errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_OVERFLOW, errStr); - cerr << errStr << endl; - throw IDBExcept(errStr, ERR_WF_OVERFLOW); - } -} +template<> +inline void WF_sum_avg::checkSumLimit(long double& val, long double& sum) +{ } + +template<> +inline void WF_sum_avg::checkSumLimit(float&, long double&) +{ } + +template<> +inline void WF_sum_avg::checkSumLimit(long&, long double&) +{ } + +template<> +inline void WF_sum_avg::checkSumLimit(unsigned long&, long double&) +{ } +template<> +inline void WF_sum_avg::checkSumLimit(double&, long double&) +{ } + +template<> +void WF_sum_avg::checkSumLimit(int128_t& val, int128_t& sum); +template<> +void WF_sum_avg::checkSumLimit(long double& val, long double& sum); +template<> +void WF_sum_avg::checkSumLimit(float&, long double&); +template<> +void WF_sum_avg::checkSumLimit(long&, long double&); +template<> +void WF_sum_avg::checkSumLimit(unsigned long&, long double&); +template<> +void WF_sum_avg::checkSumLimit(double&, long double&); template int128_t WF_sum_avg::calculateAvg(int128_t sum, uint64_t count, int scale) @@ -118,41 +132,6 @@ int128_t WF_sum_avg::calculateAvg(int128_t sum, uint64_t count, int return avg; } -template -uint128_t WF_sum_avg::calculateAvg(uint128_t sum, uint64_t count, int scale) -{ - uint128_t avg = 0; - uint128_t factor; - datatypes::getScaleDivisor(factor, scale); - if (scale > 0) - { - if ((sum * factor) / factor == sum) - { - avg = sum * factor; - avg /= count; - } - else - { - // scale won't fit before divide, we're gonna lose precision. - avg = sum / count; - if ((avg * factor) / factor != avg) // Still won't fit - { - string errStr = string("AVG(int)"); - errStr = IDBErrorInfo::instance()->errorMsg(ERR_WF_OVERFLOW, errStr); - cerr << errStr << endl; - throw IDBExcept(errStr, ERR_WF_OVERFLOW); - } - avg *= factor; - } - } - else - { - avg = sum / count; - } - avg += 0.5; - return avg; -} - template inline long double WF_sum_avg::calculateAvg(long double sum, uint64_t count, int scale) { @@ -188,31 +167,19 @@ boost::shared_ptr WF_sum_avg::makeFunction(int } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { - if (wc->functionParms()[0]->resultType().colWidth < 16) + if (wc->functionParms()[0]->resultType().colWidth < datatypes::MAXDECIMALWIDTH) { func.reset(new WF_sum_avg(id, name)); } - else + else if (wc->functionParms()[0]->resultType().colWidth == datatypes::MAXDECIMALWIDTH) { func.reset(new WF_sum_avg(id, name)); } break; } - case CalpontSystemCatalog::UDECIMAL: - { - if (wc->functionParms()[0]->resultType().colWidth < 16) - { - func.reset(new WF_sum_avg(id, name)); - } - else - { - func.reset(new WF_sum_avg(id, name)); - } - break; - } - case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { diff --git a/utils/windowfunction/wf_sum_avg.h b/utils/windowfunction/wf_sum_avg.h index b86a9c16d..6ecc37314 100644 --- a/utils/windowfunction/wf_sum_avg.h +++ b/utils/windowfunction/wf_sum_avg.h @@ -39,9 +39,6 @@ public: WindowFunctionType(id, name), fDistinct(id != WF__SUM && id != WF__AVG) { resetData(); - utils::int128Max(fMax128); - utils::int128Min(fMin128); - utils::uint128Max(fMaxu128); } // pure virtual in base @@ -58,19 +55,17 @@ protected: uint64_t fCount; bool fDistinct; std::set fSet; - int128_t fMax128; - int128_t fMin128; - uint128_t fMaxu128; - void checkSumLimit(long double val, long double sum); - void checkSumLimit(int128_t val, int128_t sum); - void checkSumLimit(uint128_t val, uint128_t sum); - + void checkSumLimit(T_IN& val, T_OUT& sum); + int128_t calculateAvg(int128_t sum, uint64_t count, int scale); - uint128_t calculateAvg(uint128_t sum, uint64_t count, int scale); long double calculateAvg(long double sum, uint64_t count, int scale); }; +template<> +void WF_sum_avg::checkSumLimit(long double& val, long double& sum); + + } // namespace #endif // UTILS_WF_SUM_AVG_H diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index dbbc5162c..0a17d5128 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -423,7 +423,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: // call the write engine to write the rows int error = NO_ERROR; - // WIP + // MCOL-641 WIP fWEWrapper.setDebugLevel(WriteEngine::DEBUG_3); cout << "inserting a row with transaction id " << txnid.id << endl; fWEWrapper.setIsInsert(true); From 88e106f0180caeb160fa7be794cfaefe40177657 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Thu, 27 Aug 2020 12:51:00 -0400 Subject: [PATCH 59/78] MCOL-641 Ternary operation with different datatype of the two resulting expressions cannot be used as a template argument. This is because the template instantiation happens at compile time and cannot depend on the runtime value of the evaluating expression. --- datatypes/mcs_decimal.h | 10 ---------- versioning/BRM/extentmap.cpp | 16 ++++++---------- 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index 94aa2a50b..5e30cfbaa 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -156,16 +156,6 @@ inline void getScaleDivisor(T& divisor, const int8_t scale) } } -/** - @brief The template to generalise common math operation - execution path using struct from . -*/ -template -void execute(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, - execplan::IDB_Decimal& result, - BinaryOperation op, - OverflowCheck overflowCheck); /** @brief Contains subset of decimal related operations. diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index fb054551c..42b452c4f 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -829,9 +829,8 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) // Merge input min/max with current min/max case CP_VALID: { - if (!isValidCPRange( !isBinaryColumn ? it->second.max : it->second.bigMax, - !isBinaryColumn ? it->second.min : it->second.bigMin, - it->second.type )) + if ((!isBinaryColumn && !isValidCPRange(it->second.max, it->second.min, it->second.type)) || + (isBinaryColumn && !isValidCPRange(it->second.bigMax, it->second.bigMin, it->second.type))) { break; } @@ -843,10 +842,8 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) // having all NULL values, in which case the current // min/max needs to be set instead of merged. - if (isValidCPRange( - !isBinaryColumn ? fExtentMap[i].partition.cprange.hiVal : fExtentMap[i].partition.cprange.bigHiVal, - !isBinaryColumn ? fExtentMap[i].partition.cprange.loVal : fExtentMap[i].partition.cprange.bigLoVal, - it->second.type)) + if ((!isBinaryColumn && isValidCPRange(fExtentMap[i].partition.cprange.hiVal, fExtentMap[i].partition.cprange.loVal, it->second.type)) || + (isBinaryColumn && isValidCPRange(fExtentMap[i].partition.cprange.bigHiVal, fExtentMap[i].partition.cprange.bigLoVal, it->second.type))) { // Swap byte order to do binary string comparison if (isCharType(it->second.type)) @@ -981,9 +978,8 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) if (it->second.newExtent) { - if (isValidCPRange( !isBinaryColumn ? it->second.max : it->second.bigMax, - !isBinaryColumn ? it->second.min : it->second.bigMin, - it->second.type )) + if ((!isBinaryColumn && isValidCPRange(it->second.max, it->second.min, it->second.type)) || + (isBinaryColumn && isValidCPRange(it->second.bigMax, it->second.bigMin, it->second.type))) { if (!isBinaryColumn) { From f7002e20b507967ec51255f3dba055a08d7902cb Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Fri, 28 Aug 2020 12:50:29 +0000 Subject: [PATCH 60/78] ::writeRow now treats WR_BINARY as int128 for 16 bytes DT only WF avg uses const & as arguments types Removed BINARY from DDL parser --- dbcon/ddlpackage/ddl.l | 1 - dbcon/ddlpackage/ddl.y | 12 +---- utils/windowfunction/wf_sum_avg.cpp | 42 +++++++++++------- utils/windowfunction/wf_sum_avg.h | 10 ++--- writeengine/shared/we_define.h | 2 + writeengine/wrapper/we_colop.cpp | 9 ++-- writeengine/wrapper/writeengine.cpp | 68 ++--------------------------- 7 files changed, 42 insertions(+), 102 deletions(-) diff --git a/dbcon/ddlpackage/ddl.l b/dbcon/ddlpackage/ddl.l index 61741cb6b..87739ba21 100644 --- a/dbcon/ddlpackage/ddl.l +++ b/dbcon/ddlpackage/ddl.l @@ -189,7 +189,6 @@ LONGTEXT {return LONGTEXT;} BOOL {return BOOL;} BOOLEAN {return BOOLEAN;} MEDIUMINT {return MEDIUMINT;} -BINARY {return BINARY;} ZEROFILL {return ZEROFILL;} \n { lineno++;} diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index 974dc6ad8..bb1b1327a 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -104,7 +104,7 @@ char* copy_string(const char *str); %token ACTION ADD ALTER AUTO_INCREMENT BIGINT BIT BLOB IDB_BLOB CASCADE IDB_CHAR CHARACTER CHECK CLOB COLUMN -BOOL BOOLEAN BINARY +BOOL BOOLEAN COLUMNS COMMENT CONSTRAINT CONSTRAINTS CREATE CURRENT_USER DATETIME DEC DECIMAL DEFAULT DEFERRABLE DEFERRED IDB_DELETE DROP ENGINE FOREIGN FULL IMMEDIATE INDEX INITIALLY IDB_INT INTEGER KEY LONGBLOB LONGTEXT @@ -133,7 +133,6 @@ ZEROFILL %type ata_rename_table %type character_string_type %type binary_string_type -%type fixed_binary_string_type %type blob_type %type text_type %type check_constraint_def @@ -752,7 +751,6 @@ opt_column_collate: data_type: character_string_type opt_column_charset opt_column_collate | binary_string_type - | fixed_binary_string_type | numeric_type | datetime_type | blob_type @@ -924,14 +922,6 @@ binary_string_type: } ; -fixed_binary_string_type: - BINARY '(' ICONST ')' - { - $$ = new ColumnType(DDL_BINARY); - $$->fLength = atoi($3); - } - ; - blob_type: BLOB '(' ICONST ')' { diff --git a/utils/windowfunction/wf_sum_avg.cpp b/utils/windowfunction/wf_sum_avg.cpp index 007bb1fc7..c008436bf 100644 --- a/utils/windowfunction/wf_sum_avg.cpp +++ b/utils/windowfunction/wf_sum_avg.cpp @@ -54,50 +54,60 @@ namespace windowfunction { template -inline void WF_sum_avg::checkSumLimit(T_IN& val, T_OUT& sum) +inline void WF_sum_avg::checkSumLimit(const T_IN& val, + const T_OUT& sum) { } template<> -inline void WF_sum_avg::checkSumLimit(int128_t& val, int128_t& sum) +inline void WF_sum_avg::checkSumLimit(const int128_t& val, + const int128_t& sum) { datatypes::AdditionOverflowCheck ofCheckOp; ofCheckOp(sum, val); } template<> -inline void WF_sum_avg::checkSumLimit(long double& val, long double& sum) +inline void WF_sum_avg::checkSumLimit(const long double& val, + const long double& sum) { } template<> -inline void WF_sum_avg::checkSumLimit(float&, long double&) +inline void WF_sum_avg::checkSumLimit(const float&, + const long double&) { } template<> -inline void WF_sum_avg::checkSumLimit(long&, long double&) +inline void WF_sum_avg::checkSumLimit(const long&, + const long double&) { } template<> -inline void WF_sum_avg::checkSumLimit(unsigned long&, long double&) +inline void WF_sum_avg::checkSumLimit(const unsigned long&, + const long double&) { } template<> -inline void WF_sum_avg::checkSumLimit(double&, long double&) +inline void WF_sum_avg::checkSumLimit(const double&, + const long double&) { } template<> -void WF_sum_avg::checkSumLimit(int128_t& val, int128_t& sum); +void WF_sum_avg::checkSumLimit(const int128_t&, const int128_t&); template<> -void WF_sum_avg::checkSumLimit(long double& val, long double& sum); +void WF_sum_avg::checkSumLimit(const long double& val, const long double&); template<> -void WF_sum_avg::checkSumLimit(float&, long double&); +void WF_sum_avg::checkSumLimit(const float&, const long double&); template<> -void WF_sum_avg::checkSumLimit(long&, long double&); +void WF_sum_avg::checkSumLimit(const long&, const long double&); template<> -void WF_sum_avg::checkSumLimit(unsigned long&, long double&); +void WF_sum_avg::checkSumLimit(const unsigned long&, + const long double&); template<> -void WF_sum_avg::checkSumLimit(double&, long double&); +void WF_sum_avg::checkSumLimit(const double&, const long double&); template -int128_t WF_sum_avg::calculateAvg(int128_t sum, uint64_t count, int scale) +int128_t WF_sum_avg::calculateAvg(const int128_t& sum, + const uint64_t count, + const int scale) { int128_t avg = 0; int128_t factor; @@ -133,7 +143,9 @@ int128_t WF_sum_avg::calculateAvg(int128_t sum, uint64_t count, int } template -inline long double WF_sum_avg::calculateAvg(long double sum, uint64_t count, int scale) +inline long double WF_sum_avg::calculateAvg(const long double& sum, + const uint64_t count, + const int scale) { return sum / count; } diff --git a/utils/windowfunction/wf_sum_avg.h b/utils/windowfunction/wf_sum_avg.h index 6ecc37314..0e2a4ffe2 100644 --- a/utils/windowfunction/wf_sum_avg.h +++ b/utils/windowfunction/wf_sum_avg.h @@ -56,16 +56,12 @@ protected: bool fDistinct; std::set fSet; - void checkSumLimit(T_IN& val, T_OUT& sum); + void checkSumLimit(const T_IN& val, const T_OUT& sum); - int128_t calculateAvg(int128_t sum, uint64_t count, int scale); - long double calculateAvg(long double sum, uint64_t count, int scale); + int128_t calculateAvg(const int128_t& sum, const uint64_t count, const int scale); + long double calculateAvg(const long double& sum, const uint64_t count, const int scale); }; -template<> -void WF_sum_avg::checkSumLimit(long double& val, long double& sum); - - } // namespace #endif // UTILS_WF_SUM_AVG_H diff --git a/writeengine/shared/we_define.h b/writeengine/shared/we_define.h index 9a16b7444..03c7783ff 100644 --- a/writeengine/shared/we_define.h +++ b/writeengine/shared/we_define.h @@ -36,6 +36,8 @@ /** Namespace WriteEngine */ namespace WriteEngine { +// Max column size is 16 bytes since MCOL-641. However left this value +// for backward compatibility const short MAX_COLUMN_BOUNDARY = 8; // Max bytes for one column const int MAX_SIGNATURE_SIZE = 8000; // Max len of dict sig val const int MAX_FIELD_SIZE = 1000; // Max len non-dict fld val diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index 7eda7fc7c..bc9f0666e 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -47,6 +47,7 @@ using namespace execplan; using namespace idbdatafile; #include "emptyvaluemanip.h" +#include "mcs_decimal.h" namespace WriteEngine { @@ -1670,8 +1671,6 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, bDataDirty = true; } - // This is a awkward way to convert void* and get its element, I just don't have a good solution for that - // How about pVal = valArray + i*curCol.colWidth? switch (curCol.colType) { case WriteEngine::WR_FLOAT : @@ -1735,7 +1734,11 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, break; case WriteEngine::WR_BINARY: - if (!bDelete) pVal = &((int128_t*) valArray)[i]; + if (!bDelete) + { + if (curCol.colWidth == datatypes::MAXDECIMALWIDTH) + pVal = &((int128_t*) valArray)[i]; + } break; default : diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 163f64aa3..da4c25bf4 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -837,84 +837,22 @@ int WriteEngineWrapper::deleteRow(const TxnID& txnid, const vector inline void allocateValArray(void*& valArray, ColTupleList::size_type totalRow, ColType colType, int colWidth) { + // MCS allocates 8 bytes even for CHARs smaller then 8 bytes. switch (colType) { case WriteEngine::WR_VARBINARY : // treat same as char for now case WriteEngine::WR_CHAR: case WriteEngine::WR_BLOB: case WriteEngine::WR_TEXT: - valArray = (char*) calloc(sizeof(char), totalRow * MAX_COLUMN_BOUNDARY); + valArray = calloc(sizeof(char), totalRow * MAX_COLUMN_BOUNDARY); break; case WriteEngine::WR_TOKEN: - valArray = (Token*) calloc(sizeof(Token), totalRow); + valArray = calloc(sizeof(Token), totalRow); break; default: valArray = calloc(totalRow, colWidth); break; } - // TODO MCOL-641 is commenting out the switch statement below correct? -#if 0 - switch (colType) - { - case WriteEngine::WR_INT: - case WriteEngine::WR_MEDINT: - valArray = (int*) calloc(sizeof(int), totalRow); - break; - - case WriteEngine::WR_UINT: - case WriteEngine::WR_UMEDINT: - valArray = (uint32_t*) calloc(sizeof(uint32_t), totalRow); - break; - - case WriteEngine::WR_VARBINARY : // treat same as char for now - case WriteEngine::WR_CHAR: - case WriteEngine::WR_BLOB: - case WriteEngine::WR_TEXT: - valArray = (char*) calloc(sizeof(char), totalRow * MAX_COLUMN_BOUNDARY); - break; - - case WriteEngine::WR_FLOAT: - valArray = (float*) calloc(sizeof(float), totalRow); - break; - - case WriteEngine::WR_DOUBLE: - valArray = (double*) calloc(sizeof(double), totalRow); - break; - - case WriteEngine::WR_BYTE: - valArray = (char*) calloc(sizeof(char), totalRow); - break; - - case WriteEngine::WR_UBYTE: - valArray = (uint8_t*) calloc(sizeof(uint8_t), totalRow); - break; - - case WriteEngine::WR_SHORT: - valArray = (short*) calloc(sizeof(short), totalRow); - break; - - case WriteEngine::WR_USHORT: - valArray = (uint16_t*) calloc(sizeof(uint16_t), totalRow); - break; - - case WriteEngine::WR_LONGLONG: - valArray = (long long*) calloc(sizeof(long long), totalRow); - break; - - case WriteEngine::WR_ULONGLONG: - valArray = (uint64_t*) calloc(sizeof(uint64_t), totalRow); - break; - - case WriteEngine::WR_TOKEN: - valArray = (Token*) calloc(sizeof(Token), totalRow); - break; - - case WriteEngine::WR_BINARY: - valArray = calloc(totalRow, colWidth); - break; - - } -#endif } int WriteEngineWrapper::deleteBadRows(const TxnID& txnid, ColStructList& colStructs, From 8de9764f84ee6516dafae5b9fc2dba0ecf44d1b3 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Fri, 11 Sep 2020 12:56:47 +0000 Subject: [PATCH 61/78] MCOL-4172 Add support for wide-DECIMAL into statistical aggregate and regr_* UDAF functions The patch fixes wrong results returned when multiple UDAF exist in projection aggregate over wide decimal literals now works --- datatypes/mcs_decimal.cpp | 15 ++ datatypes/mcs_decimal.h | 9 + dbcon/joblist/tupleaggregatestep.cpp | 31 ++- dbcon/mysql/ha_mcs_execplan.cpp | 28 ++- dbcon/mysql/ha_mcs_partition.cpp | 201 +++++++---------- utils/common/any.hpp | 11 +- utils/funcexp/func_char.cpp | 2 +- utils/rowgroup/rowaggregation.cpp | 310 ++++++++++++++++++--------- utils/rowgroup/rowaggregation.h | 4 + utils/udfsdk/mcsv1_udaf.cpp | 4 +- utils/windowfunction/wf_udaf.cpp | 4 +- 11 files changed, 375 insertions(+), 244 deletions(-) diff --git a/datatypes/mcs_decimal.cpp b/datatypes/mcs_decimal.cpp index c5b61b5fa..4b7c4e7c6 100644 --- a/datatypes/mcs_decimal.cpp +++ b/datatypes/mcs_decimal.cpp @@ -208,6 +208,21 @@ namespace datatypes return toString(const_cast(value)); } + // Compare perf with f(string&, ColTypeAlias&, int128_t&) + int128_t Decimal::int128FromString(const std::string& value, + ColTypeAlias& colType) + { + int128_t result = 0; + bool pushWarning = false; + bool noRoundup = false; + dataconvert::number_int_value(value, + colType, + pushWarning, + noRoundup, + result); + return result; + } + int Decimal::compare(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r) { int128_t divisorL, divisorR; diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index 5e30cfbaa..329487743 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -81,6 +81,10 @@ constexpr uint8_t INT128MAXPRECISION = 38U; constexpr uint8_t MAXLEGACYWIDTH = 8U; constexpr uint8_t MAXSCALEINC4AVG = 4U; constexpr int8_t IGNOREPRECISION = -1; +constexpr uint8_t MAXLENGTH16BYTES = 42; +constexpr uint8_t MAXLENGTH8BYTES = 23; + + const uint64_t mcs_pow_10[20] = { @@ -220,6 +224,11 @@ class Decimal static std::string toString(execplan::IDB_Decimal& value); static std::string toString(const execplan::IDB_Decimal& value); + /** + @brief Convenience method to get int128 from a std::string. + */ + static int128_t int128FromString(const std::string& value, ColTypeAlias& colType); + /** @brief The method detects whether decimal type is wide using csc colType. diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index 7ace89be5..50767ab36 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -106,17 +106,14 @@ struct cmpTuple return true; if (pUDAFa == pUDAFb) { - if (pUDAFa == NULL) - return false; std::vector* paramKeysa = boost::get<3>(a); std::vector* paramKeysb = boost::get<3>(b); - + if (paramKeysa == NULL || paramKeysb == NULL) + return false; if (paramKeysa->size() < paramKeysb->size()) return true; if (paramKeysa->size() == paramKeysb->size()) { - if (paramKeysa == NULL) - return false; for (uint64_t i = 0; i < paramKeysa->size(); ++i) { if ((*paramKeysa)[i] < (*paramKeysb)[i]) @@ -1471,7 +1468,7 @@ void TupleAggregateStep::prep1PhaseAggregate( // find if this func is a duplicate AGG_MAP::iterator iter = aggFuncMap.find(boost::make_tuple(key, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)); - if (iter != aggFuncMap.end()) + if (aggOp != ROWAGG_UDAF && aggOp != ROWAGG_MULTI_PARM && iter != aggFuncMap.end()) { if (funct->fAggFunction == ROWAGG_AVG) funct->fAggFunction = ROWAGG_DUP_AVG; @@ -1513,7 +1510,7 @@ void TupleAggregateStep::prep1PhaseAggregate( } // there is avg(k), but no count(k) in the select list - uint64_t lastCol = returnedColVec.size(); + uint64_t lastCol = outIdx; for (map::iterator k = avgFuncMap.begin(); k != avgFuncMap.end(); k++) { @@ -1855,8 +1852,12 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( } // skip if this is a duplicate - if (aggFuncMap.find(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)) != aggFuncMap.end()) + if (aggOp != ROWAGG_UDAF && aggOp != ROWAGG_MULTI_PARM + && aggFuncMap.find(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)) != aggFuncMap.end()) + { + // skip if this is a duplicate continue; + } functionVec1.push_back(funct); aggFuncMap.insert(make_pair(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), colAgg)); @@ -3134,9 +3135,13 @@ void TupleAggregateStep::prep2PhasesAggregate( } // skip if this is a duplicate - if (aggFuncMap.find(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)) != aggFuncMap.end()) + if (aggOp != ROWAGG_UDAF && aggOp != ROWAGG_MULTI_PARM + && aggFuncMap.find(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)) != aggFuncMap.end()) + { + // skip if this is a duplicate continue; - + } + functionVecPm.push_back(funct); aggFuncMap.insert(make_pair(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), colAggPm)); @@ -4010,8 +4015,12 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( } // skip if this is a duplicate - if (aggFuncMap.find(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)) != aggFuncMap.end()) + if (aggOp != ROWAGG_UDAF && aggOp != ROWAGG_MULTI_PARM + && aggFuncMap.find(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL)) != aggFuncMap.end()) + { + // skip if this is a duplicate continue; + } functionVecPm.push_back(funct); aggFuncMap.insert(make_pair(boost::make_tuple(aggKey, aggOp, pUDAFFunc, udafc ? udafc->getContext().getParamKeys() : NULL), colAggPm-multiParm)); diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 2c4c2d775..aa5dd5184 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -4671,6 +4671,9 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) vector selCols; vector orderCols; bool bIsConst = false; + unsigned int constValPrecision = 0; + unsigned int constValScale = 0; + bool hasDecimalConst = false; if (get_fe_conn_info_ptr() == NULL) set_fe_conn_info_ptr((void*)new cal_connection_info()); @@ -4854,6 +4857,13 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) parm.reset(buildReturnedColumn(sfitemp, gwi, gwi.fatalParseError)); ac->constCol(parm); bIsConst = true; + if (sfitemp->cmp_type() == DECIMAL_RESULT) + { + hasDecimalConst = true; + Item_decimal* idp = (Item_decimal*)sfitemp; + constValPrecision = idp->decimal_precision(); + constValScale = idp->decimal_scale(); + } break; } default: @@ -4975,6 +4985,9 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) } } + bool isAvg = (isp->sum_func() == Item_sum::AVG_FUNC || + isp->sum_func() == Item_sum::AVG_DISTINCT_FUNC); + // Get result type // Modified for MCOL-1201 multi-argument aggregate if (!bIsConst && ac->aggParms().size() > 0) @@ -4983,10 +4996,8 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) // use the first parm for result type. parm = ac->aggParms()[0]; - bool isAvg = (isp->sum_func() == Item_sum::AVG_FUNC || - isp->sum_func() == Item_sum::AVG_DISTINCT_FUNC); if (isAvg || isp->sum_func() == Item_sum::SUM_FUNC || - isp->sum_func() == Item_sum::SUM_DISTINCT_FUNC) + isp->sum_func() == Item_sum::SUM_DISTINCT_FUNC) { CalpontSystemCatalog::ColType ct = parm->resultType(); if (datatypes::Decimal::isWideDecimalType(ct)) @@ -5054,6 +5065,17 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) ac->resultType(parm->resultType()); } } + else if (bIsConst && hasDecimalConst && isAvg) + { + CalpontSystemCatalog::ColType ct = parm->resultType(); + if (datatypes::Decimal::isWideDecimalType(constValPrecision)) + { + ct.precision = constValPrecision; + ct.scale = constValScale; + ct.colWidth = datatypes::MAXDECIMALWIDTH; + } + ac->resultType(ct); + } else { ac->resultType(colType_MysqlToIDB(isp)); diff --git a/dbcon/mysql/ha_mcs_partition.cpp b/dbcon/mysql/ha_mcs_partition.cpp index 1b1d1352d..17ba58992 100644 --- a/dbcon/mysql/ha_mcs_partition.cpp +++ b/dbcon/mysql/ha_mcs_partition.cpp @@ -48,7 +48,6 @@ using namespace BRM; #include "dataconvert.h" using namespace dataconvert; -#include "widedecimalutils.h" #include "ddlpkg.h" #include "sqlparser.h" using namespace ddlpackage; @@ -66,6 +65,7 @@ using namespace logging; #include using namespace boost; +#include "mcs_decimal.h" namespace { @@ -238,12 +238,12 @@ struct PartitionInfo int64_t max; union { - __int128 bigMin; + int128_t int128Min; int64_t min_; }; union { - __int128 bigMax; + int128_t int128Max; int64_t max_; }; uint64_t status; @@ -251,8 +251,8 @@ struct PartitionInfo max((uint64_t) - 0x8000000000000001LL), status(0) { - utils::int128Min(bigMin); - utils::int128Max(bigMax); + int128Min = datatypes::Decimal::minInt128; + int128Max = datatypes::Decimal::maxInt128; }; }; @@ -312,8 +312,8 @@ const string format(T v, CalpontSystemCatalog::ColType& ct) } else { - char buf[utils::MAXLENGTH16BYTES]; - DataConvert::decimalToString((__int128*)&v, (unsigned)ct.scale, buf, sizeof(buf), ct.colDataType); + char buf[datatypes::MAXLENGTH16BYTES]; + DataConvert::decimalToString((int128_t*)&v, (unsigned)ct.scale, buf, sizeof(buf), ct.colDataType); oss << buf; } @@ -659,8 +659,8 @@ void partitionByValue_common(UDF_ARGS* args, // input int32_t seqNum; string schema, table, column; CalpontSystemCatalog::ColType ct; - int64_t startVal, endVal; - int128_t bigStartVal, bigEndVal; + int64_t startVal = 0, endVal = 0; + int128_t int128StartVal = 0, int128EndVal = 0; uint8_t rfMin = 0, rfMax = 0; if (args->arg_count == 5) @@ -748,11 +748,11 @@ void partitionByValue_common(UDF_ARGS* args, // input { if (isUnsigned(ct.colDataType)) { - bigStartVal = 0; + int128StartVal = 0; } else { - utils::int128Min(bigStartVal); + int128StartVal = datatypes::Decimal::minInt128; } } } @@ -761,7 +761,7 @@ void partitionByValue_common(UDF_ARGS* args, // input if (!datatypes::Decimal::isWideDecimalType(ct)) startVal = IDB_format((char*) args->args[2], ct, rfMin); else - bigStartVal = IDB_format((char*) args->args[2], ct, rfMin); + int128StartVal = IDB_format((char*) args->args[2], ct, rfMin); } if (!args->args[3]) @@ -781,11 +781,11 @@ void partitionByValue_common(UDF_ARGS* args, // input { if (isUnsigned(ct.colDataType)) { - bigEndVal = -1; + int128EndVal = -1; } else { - utils::int128Max(bigEndVal); + int128EndVal = datatypes::Decimal::maxInt128; } } } @@ -794,7 +794,7 @@ void partitionByValue_common(UDF_ARGS* args, // input if (!datatypes::Decimal::isWideDecimalType(ct)) endVal = IDB_format((char*) args->args[3], ct, rfMax); else - bigEndVal = IDB_format((char*) args->args[3], ct, rfMax); + int128EndVal = IDB_format((char*) args->args[3], ct, rfMax); } } else @@ -816,11 +816,11 @@ void partitionByValue_common(UDF_ARGS* args, // input { if (isUnsigned(ct.colDataType)) { - bigStartVal = 0; + int128StartVal = 0; } else { - utils::int128Min(bigStartVal); + int128StartVal = datatypes::Decimal::minInt128; } } } @@ -829,7 +829,7 @@ void partitionByValue_common(UDF_ARGS* args, // input if (!datatypes::Decimal::isWideDecimalType(ct)) startVal = IDB_format((char*) args->args[3], ct, rfMin); else - bigStartVal = IDB_format((char*) args->args[3], ct, rfMin); + int128StartVal = IDB_format((char*) args->args[3], ct, rfMin); } if (!args->args[4]) @@ -849,11 +849,11 @@ void partitionByValue_common(UDF_ARGS* args, // input { if (isUnsigned(ct.colDataType)) { - bigEndVal = -1; + int128EndVal = -1; } else { - utils::int128Max(bigEndVal); + int128EndVal = datatypes::Decimal::maxInt128; } } } @@ -862,7 +862,7 @@ void partitionByValue_common(UDF_ARGS* args, // input if (!datatypes::Decimal::isWideDecimalType(ct)) endVal = IDB_format((char*) args->args[4], ct, rfMax); else - bigEndVal = IDB_format((char*) args->args[4], ct, rfMax); + int128EndVal = IDB_format((char*) args->args[4], ct, rfMax); } } @@ -889,7 +889,7 @@ void partitionByValue_common(UDF_ARGS* args, // input if (!datatypes::Decimal::isWideDecimalType(ct)) state = em.getExtentMaxMin(iter->range.start, partInfo.max, partInfo.min, seqNum); else - state = em.getExtentMaxMin(iter->range.start, partInfo.bigMax, partInfo.bigMin, seqNum); + state = em.getExtentMaxMin(iter->range.start, partInfo.int128Max, partInfo.int128Min, seqNum); // char column order swap if ((ct.colDataType == CalpontSystemCatalog::CHAR && ct.colWidth <= 8) || @@ -928,18 +928,8 @@ void partitionByValue_common(UDF_ARGS* args, // input } else { - if (isUnsigned(ct.colDataType)) - { - mapit->second.bigMin = - (static_cast(partInfo.bigMin) < static_cast(mapit->second.bigMin) ? partInfo.bigMin : mapit->second.bigMin); - mapit->second.bigMax = - (static_cast(partInfo.bigMax) > static_cast(mapit->second.bigMax) ? partInfo.bigMax : mapit->second.bigMax); - } - else - { - mapit->second.bigMin = (partInfo.bigMin < mapit->second.bigMin ? partInfo.bigMin : mapit->second.bigMin); - mapit->second.bigMax = (partInfo.bigMax > mapit->second.bigMax ? partInfo.bigMax : mapit->second.bigMax); - } + mapit->second.int128Min = (partInfo.int128Min < mapit->second.int128Min ? partInfo.int128Min : mapit->second.int128Min); + mapit->second.int128Max = (partInfo.int128Max > mapit->second.int128Max ? partInfo.int128Max : mapit->second.int128Max); } } } @@ -984,36 +974,16 @@ void partitionByValue_common(UDF_ARGS* args, // input } else { - if (isUnsigned(ct.colDataType)) + if (!(mapit->second.status & CPINVALID) && mapit->second.int128Min >= int128StartVal && mapit->second.int128Max <= int128EndVal && + !(mapit->second.int128Min == datatypes::Decimal::maxInt128 && mapit->second.int128Max == datatypes::Decimal::minInt128)) { - if (!(mapit->second.status & CPINVALID) && - static_cast(mapit->second.bigMin) >= static_cast(bigStartVal) && - static_cast(mapit->second.bigMax) <= static_cast(bigEndVal) && - !(static_cast(mapit->second.bigMin) == static_cast(-1) && - static_cast(mapit->second.bigMax == 0))) - { - if (rfMin == ROUND_POS && mapit->second.bigMin == bigStartVal) - continue; + if (rfMin == ROUND_POS && mapit->second.int128Min == int128StartVal) + continue; - if (rfMax == ROUND_NEG && mapit->second.bigMax == bigEndVal) - continue; + if (rfMax == ROUND_NEG && mapit->second.int128Max == int128EndVal) + continue; - partSet.insert(mapit->first); - } - } - else - { - if (!(mapit->second.status & CPINVALID) && mapit->second.bigMin >= bigStartVal && mapit->second.bigMax <= bigEndVal && - !(mapit->second.bigMin == utils::maxInt128 && mapit->second.bigMax == utils::minInt128)) - { - if (rfMin == ROUND_POS && mapit->second.bigMin == bigStartVal) - continue; - - if (rfMax == ROUND_NEG && mapit->second.bigMax == bigEndVal) - continue; - - partSet.insert(mapit->first); - } + partSet.insert(mapit->first); } } } @@ -1239,7 +1209,7 @@ extern "C" if (!datatypes::Decimal::isWideDecimalType(ct)) state = em.getExtentMaxMin(iter->range.start, partInfo.max, partInfo.min, seqNum); else - state = em.getExtentMaxMin(iter->range.start, partInfo.bigMax, partInfo.bigMin, seqNum); + state = em.getExtentMaxMin(iter->range.start, partInfo.int128Max, partInfo.int128Min, seqNum); // char column order swap for compare if ((ct.colDataType == CalpontSystemCatalog::CHAR && ct.colWidth <= 8) || @@ -1268,8 +1238,8 @@ extern "C" } else { - mapit->second.bigMin = (partInfo.bigMin < mapit->second.bigMin ? partInfo.bigMin : mapit->second.bigMin); - mapit->second.bigMax = (partInfo.bigMax > mapit->second.bigMax ? partInfo.bigMax : mapit->second.bigMax); + mapit->second.int128Min = (partInfo.int128Min < mapit->second.int128Min ? partInfo.int128Min : mapit->second.int128Min); + mapit->second.int128Max = (partInfo.int128Max > mapit->second.int128Max ? partInfo.int128Max : mapit->second.int128Max); } } } @@ -1299,19 +1269,16 @@ extern "C" else { output << setw(10) << "Part#" - << setw(utils::MAXLENGTH16BYTES) << "Min" - << setw(utils::MAXLENGTH16BYTES) << "Max" << "Status"; + << setw(datatypes::MAXLENGTH16BYTES) << "Min" + << setw(datatypes::MAXLENGTH16BYTES) << "Max" << "Status"; } int64_t maxLimit = numeric_limits::max(); int64_t minLimit = numeric_limits::min(); - __int128 bigMaxLimit, bigMinLimit; - utils::int128Max(bigMaxLimit); - utils::int128Min(bigMinLimit); - unsigned __int128 ubigMaxLimit, ubigMinLimit; - utils::uint128Max(ubigMaxLimit); - ubigMinLimit = 0; + int128_t int128MaxLimit, int128MinLimit; + int128MaxLimit = datatypes::Decimal::maxInt128; + int128MinLimit = datatypes::Decimal::minInt128; // char column order swap for compare in subsequent loop if ((ct.colDataType == CalpontSystemCatalog::CHAR && ct.colWidth <= 8) || @@ -1334,7 +1301,7 @@ extern "C" if (!datatypes::Decimal::isWideDecimalType(ct)) output << setw(30) << "N/A" << setw(30) << "N/A"; else - output << setw(utils::MAXLENGTH16BYTES) << "N/A" << setw(utils::MAXLENGTH16BYTES) << "N/A"; + output << setw(datatypes::MAXLENGTH16BYTES) << "N/A" << setw(datatypes::MAXLENGTH16BYTES) << "N/A"; } else { @@ -1350,11 +1317,11 @@ extern "C" } else { - if (static_cast(partIt->second.bigMin) == ubigMaxLimit - && static_cast(partIt->second.bigMax) == ubigMinLimit) - output << setw(utils::MAXLENGTH16BYTES) << "Empty/Null" << setw(utils::MAXLENGTH16BYTES) << "Empty/Null"; + if (partIt->second.int128Min == int128MaxLimit + && partIt->second.int128Max == int128MinLimit) + output << setw(datatypes::MAXLENGTH16BYTES) << "Empty/Null" << setw(datatypes::MAXLENGTH16BYTES) << "Empty/Null"; else - output << setw(utils::MAXLENGTH16BYTES) << format(partIt->second.bigMin, ct) << setw(utils::MAXLENGTH16BYTES) << format(partIt->second.bigMax, ct); + output << setw(datatypes::MAXLENGTH16BYTES) << format(partIt->second.int128Min, ct) << setw(datatypes::MAXLENGTH16BYTES) << format(partIt->second.int128Max, ct); } } else @@ -1368,10 +1335,10 @@ extern "C" } else { - if (partIt->second.bigMin == bigMaxLimit && partIt->second.bigMax == bigMinLimit) - output << setw(utils::MAXLENGTH16BYTES) << "Empty/Null" << setw(utils::MAXLENGTH16BYTES) << "Empty/Null"; + if (partIt->second.int128Min == int128MaxLimit && partIt->second.int128Max == int128MinLimit) + output << setw(datatypes::MAXLENGTH16BYTES) << "Empty/Null" << setw(datatypes::MAXLENGTH16BYTES) << "Empty/Null"; else - output << setw(utils::MAXLENGTH16BYTES) << format(partIt->second.bigMin, ct) << setw(utils::MAXLENGTH16BYTES) << format(partIt->second.bigMax, ct); + output << setw(datatypes::MAXLENGTH16BYTES) << format(partIt->second.int128Min, ct) << setw(datatypes::MAXLENGTH16BYTES) << format(partIt->second.int128Max, ct); } } } @@ -1934,8 +1901,8 @@ extern "C" string schema, table, column; CalpontSystemCatalog::ColType ct; string errMsg; - int64_t startVal, endVal; - int128_t bigStartVal, bigEndVal; + int64_t startVal = 0, endVal = 0; + int128_t int128StartVal = 0, int128EndVal = 0; uint8_t rfMin = 0, rfMax = 0; try @@ -2009,11 +1976,11 @@ extern "C" { if (isUnsigned(ct.colDataType)) { - bigStartVal = 0; + int128StartVal = 0; } else { - utils::int128Min(bigStartVal); + int128StartVal = datatypes::Decimal::minInt128; } } } @@ -2022,7 +1989,7 @@ extern "C" if (!datatypes::Decimal::isWideDecimalType(ct)) startVal = IDB_format((char*) args->args[2], ct, rfMin); else - bigStartVal = IDB_format((char*) args->args[2], ct, rfMin); + int128StartVal = IDB_format((char*) args->args[2], ct, rfMin); } if (!args->args[3]) @@ -2042,11 +2009,11 @@ extern "C" { if (isUnsigned(ct.colDataType)) { - bigEndVal = -1; + int128EndVal = -1; } else { - utils::int128Max(bigEndVal); + int128EndVal = datatypes::Decimal::maxInt128; } } } @@ -2055,7 +2022,7 @@ extern "C" if (!datatypes::Decimal::isWideDecimalType(ct)) endVal = IDB_format((char*) args->args[3], ct, rfMax); else - bigEndVal = IDB_format((char*) args->args[3], ct, rfMax); + int128EndVal = IDB_format((char*) args->args[3], ct, rfMax); } } else @@ -2077,11 +2044,11 @@ extern "C" { if (isUnsigned(ct.colDataType)) { - bigStartVal = 0; + int128StartVal = 0; } else { - utils::int128Min(bigStartVal); + int128StartVal = datatypes::Decimal::minInt128; } } } @@ -2090,7 +2057,7 @@ extern "C" if (!datatypes::Decimal::isWideDecimalType(ct)) startVal = IDB_format((char*) args->args[3], ct, rfMin); else - bigStartVal = IDB_format((char*) args->args[3], ct, rfMin); + int128StartVal = IDB_format((char*) args->args[3], ct, rfMin); } if (!args->args[4]) @@ -2110,11 +2077,11 @@ extern "C" { if (isUnsigned(ct.colDataType)) { - bigEndVal = -1; + int128EndVal = -1; } else { - utils::int128Max(bigEndVal); + int128EndVal = datatypes::Decimal::maxInt128; } } } @@ -2123,7 +2090,7 @@ extern "C" if (!datatypes::Decimal::isWideDecimalType(ct)) endVal = IDB_format((char*) args->args[4], ct, rfMax); else - bigEndVal = IDB_format((char*) args->args[4], ct, rfMax); + int128EndVal = IDB_format((char*) args->args[4], ct, rfMax); } } @@ -2152,7 +2119,7 @@ extern "C" if (!datatypes::Decimal::isWideDecimalType(ct)) state = em.getExtentMaxMin(iter->range.start, partInfo.max, partInfo.min, seqNum); else - state = em.getExtentMaxMin(iter->range.start, partInfo.bigMax, partInfo.bigMin, seqNum); + state = em.getExtentMaxMin(iter->range.start, partInfo.int128Max, partInfo.int128Min, seqNum); // char column order swap if ((ct.colDataType == CalpontSystemCatalog::CHAR && ct.colWidth <= 8) || @@ -2191,18 +2158,8 @@ extern "C" } else { - if (isUnsigned(ct.colDataType)) - { - mapit->second.bigMin = - (static_cast(partInfo.bigMin) < static_cast(mapit->second.bigMin) ? partInfo.bigMin : mapit->second.bigMin); - mapit->second.bigMax = - (static_cast(partInfo.bigMax) > static_cast(mapit->second.bigMax) ? partInfo.bigMax : mapit->second.bigMax); - } - else - { - mapit->second.bigMin = (partInfo.bigMin < mapit->second.bigMin ? partInfo.bigMin : mapit->second.bigMin); - mapit->second.bigMax = (partInfo.bigMax > mapit->second.bigMax ? partInfo.bigMax : mapit->second.bigMax); - } + mapit->second.int128Min = (partInfo.int128Min < mapit->second.int128Min ? partInfo.int128Min : mapit->second.int128Min); + mapit->second.int128Max = (partInfo.int128Max > mapit->second.int128Max ? partInfo.int128Max : mapit->second.int128Max); } } } @@ -2294,13 +2251,13 @@ extern "C" } else { - if (!(mapit->second.status & CPINVALID) && mapit->second.bigMin >= bigStartVal && mapit->second.bigMax <= bigEndVal && - !(mapit->second.bigMin == utils::maxInt128 && mapit->second.bigMax == utils::minInt128)) + if (!(mapit->second.status & CPINVALID) && mapit->second.int128Min >= int128StartVal && mapit->second.int128Max <= int128EndVal && + !(mapit->second.int128Min == datatypes::Decimal::maxInt128 && mapit->second.int128Max == datatypes::Decimal::minInt128)) { - if (rfMin == ROUND_POS && mapit->second.bigMin == bigStartVal) + if (rfMin == ROUND_POS && mapit->second.int128Min == int128StartVal) continue; - if (rfMax == ROUND_NEG && mapit->second.bigMax == bigEndVal) + if (rfMax == ROUND_NEG && mapit->second.int128Max == int128EndVal) continue; // print header @@ -2308,8 +2265,8 @@ extern "C" { output.setf(ios::left, ios::adjustfield); output << setw(10) << "Part#" - << setw(utils::MAXLENGTH16BYTES) << "Min" - << setw(utils::MAXLENGTH16BYTES) << "Max" << "Status"; + << setw(datatypes::MAXLENGTH16BYTES) << "Min" + << setw(datatypes::MAXLENGTH16BYTES) << "Max" << "Status"; } noPartFound = false; @@ -2321,24 +2278,14 @@ extern "C" if (mapit->second.status & CPINVALID) { - output << setw(utils::MAXLENGTH16BYTES) << "N/A" << setw(utils::MAXLENGTH16BYTES) << "N/A"; + output << setw(datatypes::MAXLENGTH16BYTES) << "N/A" << setw(datatypes::MAXLENGTH16BYTES) << "N/A"; } else { - if ((isUnsigned(ct.colDataType))) - { - if (static_cast(mapit->second.bigMin) > static_cast(mapit->second.bigMax)) - output << setw(utils::MAXLENGTH16BYTES) << "Empty/Null" << setw(utils::MAXLENGTH16BYTES) << "Empty/Null"; - else - output << setw(utils::MAXLENGTH16BYTES) << format(mapit->second.bigMin, ct) << setw(utils::MAXLENGTH16BYTES) << format(mapit->second.bigMax, ct); - } + if (mapit->second.int128Min > mapit->second.int128Max) + output << setw(datatypes::MAXLENGTH16BYTES) << "Empty/Null" << setw(datatypes::MAXLENGTH16BYTES) << "Empty/Null"; else - { - if (mapit->second.bigMin > mapit->second.bigMax) - output << setw(utils::MAXLENGTH16BYTES) << "Empty/Null" << setw(utils::MAXLENGTH16BYTES) << "Empty/Null"; - else - output << setw(utils::MAXLENGTH16BYTES) << format(mapit->second.bigMin, ct) << setw(utils::MAXLENGTH16BYTES) << format(mapit->second.bigMax, ct); - } + output << setw(datatypes::MAXLENGTH16BYTES) << format(mapit->second.int128Min, ct) << setw(datatypes::MAXLENGTH16BYTES) << format(mapit->second.int128Max, ct); } if (mapit->second.status & ET_DISABLED) diff --git a/utils/common/any.hpp b/utils/common/any.hpp index 78c65ffb2..7ae5d553e 100755 --- a/utils/common/any.hpp +++ b/utils/common/any.hpp @@ -9,7 +9,7 @@ * http://www.boost.org/LICENSE_1_0.txt */ -#include +#include #include #include @@ -123,7 +123,14 @@ namespace anyimpl typedef void type; }; - /// Specializations for small types. + /// Specializations for big types. +#define BIG_POLICY(TYPE) template<> struct \ + choose_policy { typedef big_any_policy type; }; + + BIG_POLICY(__int128); + BIG_POLICY(unsigned __int128); + + /// Specializations for small types. #define SMALL_POLICY(TYPE) template<> struct \ choose_policy { typedef small_any_policy type; }; diff --git a/utils/funcexp/func_char.cpp b/utils/funcexp/func_char.cpp index f303bb8d3..70320c2b1 100644 --- a/utils/funcexp/func_char.cpp +++ b/utils/funcexp/func_char.cpp @@ -95,7 +95,7 @@ string Func_char::getStrVal(Row& row, buf[0]= 0; char* pBuf = buf; CHARSET_INFO* cs = ct.getCharset(); - int32_t value; + int32_t value = 0; int32_t numBytes = 0; for (uint32_t i = 0; i < parm.size(); ++i) { diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index 491b4ef27..04fe593ef 100755 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -239,16 +239,20 @@ const static_any::any& RowAggregation::shortTypeId((short)1); const static_any::any& RowAggregation::intTypeId((int)1); const static_any::any& RowAggregation::longTypeId((long)1); const static_any::any& RowAggregation::llTypeId((long long)1); +const static_any::any& RowAggregation::int128TypeId((__int128)1); const static_any::any& RowAggregation::ucharTypeId((unsigned char)1); const static_any::any& RowAggregation::ushortTypeId((unsigned short)1); const static_any::any& RowAggregation::uintTypeId((unsigned int)1); const static_any::any& RowAggregation::ulongTypeId((unsigned long)1); const static_any::any& RowAggregation::ullTypeId((unsigned long long)1); +const static_any::any& RowAggregation::uint128TypeId((unsigned __int128)1); const static_any::any& RowAggregation::floatTypeId((float)1); const static_any::any& RowAggregation::doubleTypeId((double)1); const static_any::any& RowAggregation::longdoubleTypeId((long double)1); const static_any::any& RowAggregation::strTypeId(typeStr); +using Dec = datatypes::Decimal; + KeyStorage::KeyStorage(const RowGroup& keys, Row** tRow) : tmpRow(tRow), rg(keys) { RGData data(rg); @@ -619,9 +623,6 @@ RowAggregation::RowAggregation(const RowAggregation& rhs): fSmallSideRGs(NULL), fLargeSideRG(NULL), fSmallSideCount(0), fRGContext(rhs.fRGContext), fOrigFunctionCols(NULL) { - //fGroupByCols.clear(); - //fFunctionCols.clear(); - fGroupByCols.assign(rhs.fGroupByCols.begin(), rhs.fGroupByCols.end()); fFunctionCols.assign(rhs.fFunctionCols.begin(), rhs.fFunctionCols.end()); } @@ -721,31 +722,32 @@ void RowAggregation::setJoinRowGroups(vector* pSmallSideRG, RowGroup* // threads on the PM and by multple threads on the UM. It must remain // thread safe. //------------------------------------------------------------------------------ -void RowAggregation::resetUDAF(RowUDAFFunctionCol* rowUDAF) +void RowAggregation::resetUDAF(RowUDAFFunctionCol* rowUDAF, uint64_t funcColsIdx) { // RowAggregation and it's functions need to be re-entrant which means // each instance (thread) needs its own copy of the context object. // Note: operator=() doesn't copy userData. - fRGContext = rowUDAF->fUDAFContext; + fRGContextColl[funcColsIdx] = rowUDAF->fUDAFContext; // Call the user reset for the group userData. Since, at this point, // context's userData will be NULL, reset will generate a new one. mcsv1sdk::mcsv1_UDAF::ReturnCode rc; - rc = fRGContext.getFunction()->reset(&fRGContext); + rc = fRGContextColl[funcColsIdx].getFunction()->reset(&fRGContextColl[funcColsIdx]); if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) { rowUDAF->bInterrupted = true; - throw logging::QueryDataExcept(fRGContext.getErrorMessage(), logging::aggregateFuncErr); + throw logging::QueryDataExcept(fRGContextColl[funcColsIdx].getErrorMessage(), + logging::aggregateFuncErr); } fRow.setUserDataStore(fRowGroupOut->getRGData()->getUserDataStore()); - fRow.setUserData(fRGContext, - fRGContext.getUserDataSP(), - fRGContext.getUserDataSize(), + fRow.setUserData(fRGContextColl[funcColsIdx], + fRGContextColl[funcColsIdx].getUserDataSP(), + fRGContextColl[funcColsIdx].getUserDataSize(), rowUDAF->fAuxColumnIndex); - - fRGContext.setUserData(NULL); // Prevents calling deleteUserData on the fRGContext. + // Prevents calling deleteUserData on the mcsv1Context. + fRGContextColl[funcColsIdx].setUserData(NULL); } //------------------------------------------------------------------------------ @@ -784,13 +786,15 @@ void RowAggregation::initialize() { fRowGroupOut->setRowCount(1); attachGroupConcatAg(); - + // Lazy approach w/o a mapping b/w fFunctionCols idx and fRGContextColl idx + fRGContextColl.resize(fFunctionCols.size()); // For UDAF, reset the data for (uint64_t i = 0; i < fFunctionCols.size(); i++) { if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) { - resetUDAF(dynamic_cast(fFunctionCols[i].get())); + auto rowUDAFColumnPtr = dynamic_cast(fFunctionCols[i].get()); + resetUDAF(rowUDAFColumnPtr, i); } } } @@ -842,7 +846,8 @@ void RowAggregation::aggReset() { if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) { - resetUDAF(dynamic_cast(fFunctionCols[i].get())); + auto rowUDAFColumnPtr = dynamic_cast(fFunctionCols[i].get()); + resetUDAF(rowUDAFColumnPtr, i); } } } @@ -897,7 +902,8 @@ void RowAggregationUM::aggregateRowWithRemap(Row& row) { if ((*fOrigFunctionCols)[i]->fAggFunction == ROWAGG_UDAF) { - resetUDAF(dynamic_cast((*fOrigFunctionCols)[i].get())); + auto rowUDAFColumnPtr = dynamic_cast((*fOrigFunctionCols)[i].get()); + resetUDAF(rowUDAFColumnPtr, i); } } } @@ -907,7 +913,8 @@ void RowAggregationUM::aggregateRowWithRemap(Row& row) { if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) { - resetUDAF(dynamic_cast(fFunctionCols[i].get())); + auto rowUDAFColumnPtr = dynamic_cast(fFunctionCols[i].get()); + resetUDAF(rowUDAFColumnPtr, i); } } } @@ -972,7 +979,8 @@ void RowAggregation::aggregateRow(Row& row) { if ((*fOrigFunctionCols)[i]->fAggFunction == ROWAGG_UDAF) { - resetUDAF(dynamic_cast((*fOrigFunctionCols)[i].get())); + auto rowUDAFColumnPtr = dynamic_cast((*fOrigFunctionCols)[i].get()); + resetUDAF(rowUDAFColumnPtr, i); } } } @@ -982,7 +990,8 @@ void RowAggregation::aggregateRow(Row& row) { if (fFunctionCols[i]->fAggFunction == ROWAGG_UDAF) { - resetUDAF(dynamic_cast(fFunctionCols[i].get())); + auto rowUDAFColumnPtr = dynamic_cast(fFunctionCols[i].get()); + resetUDAF(rowUDAFColumnPtr, i); } } } @@ -1033,7 +1042,7 @@ void RowAggregation::initMapData(const Row& rowIn) case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - if (LIKELY(fRow.getColumnWidth(colIn) == datatypes::MAXDECIMALWIDTH)) + if (LIKELY(rowIn.getColumnWidth(colIn) == datatypes::MAXDECIMALWIDTH)) { uint32_t colOutOffset = fRow.getOffset(colOut); fRow.setBinaryField_offset( @@ -1041,7 +1050,7 @@ void RowAggregation::initMapData(const Row& rowIn) sizeof(int128_t), colOutOffset); } - else if (fRow.getColumnWidth(colIn) <= datatypes::MAXLEGACYWIDTH) + else if (rowIn.getColumnWidth(colIn) <= datatypes::MAXLEGACYWIDTH) { fRow.setIntField(rowIn.getIntField(colIn), colOut); } @@ -2025,9 +2034,24 @@ void RowAggregation::doStatistics(const Row& rowIn, int64_t colIn, int64_t colOu case execplan::CalpontSystemCatalog::MEDINT: case execplan::CalpontSystemCatalog::INT: case execplan::CalpontSystemCatalog::BIGINT: + valIn = (long double) rowIn.getIntField(colIn); + break; + case execplan::CalpontSystemCatalog::DECIMAL: // handle scale later case execplan::CalpontSystemCatalog::UDECIMAL: // handle scale later - valIn = (long double) rowIn.getIntField(colIn); + if (LIKELY(fRowGroupIn.getColumnWidth(colIn) == datatypes::MAXDECIMALWIDTH)) + { + int128_t* val128InPtr = rowIn.getBinaryField(colIn); + valIn = Dec::getLongDoubleFromWideDecimal(*val128InPtr); + } + else if (fRowGroupIn.getColumnWidth(colIn) <= datatypes::MAXLEGACYWIDTH) + { + valIn = (long double) rowIn.getIntField(colIn); + } + else + { + idbassert(false); + } break; case execplan::CalpontSystemCatalog::UTINYINT: @@ -2068,7 +2092,10 @@ void RowAggregation::doStatistics(const Row& rowIn, int64_t colIn, int64_t colOu void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, int64_t colAux, uint64_t& funcColsIdx) { - uint32_t paramCount = fRGContext.getParameterCount(); + uint32_t paramCount = fRGContextColl[funcColsIdx].getParameterCount(); + // doUDAF changes funcColsIdx to skip UDAF arguments so the real UDAF + // column idx is the initial value of the funcColsIdx + uint64_t origFuncColsIdx = funcColsIdx; // The vector of parameters to be sent to the UDAF utils::VLArray valsIn(paramCount); utils::VLArray dataFlags(paramCount); @@ -2094,7 +2121,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, if ((cc && cc->type() == execplan::ConstantColumn::NULLDATA) || (!cc && isNull(&fRowGroupIn, rowIn, colIn) == true)) { - if (fRGContext.getRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS)) + if (fRGContextColl[origFuncColsIdx].getRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS)) { // When Ignore nulls, if there are multiple parameters and any // one of them is NULL, we ignore the entry. We need to increment @@ -2139,7 +2166,6 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, datum.scale = fRowGroupIn.getScale()[colIn]; datum.precision = fRowGroupIn.getPrecision()[colIn]; } - break; } @@ -2156,7 +2182,19 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, } else { - datum.columnData = rowIn.getIntField(colIn); + if (LIKELY(fRowGroupIn.getColumnWidth(colIn) + == datatypes::MAXDECIMALWIDTH)) + { + datum.columnData = rowIn.getInt128Field(colIn); + } + else if (fRowGroupIn.getColumnWidth(colIn) <= datatypes::MAXLEGACYWIDTH) + { + datum.columnData = rowIn.getIntField(colIn); + } + else + { + idbassert(false); + } datum.scale = fRowGroupIn.getScale()[colIn]; datum.precision = fRowGroupIn.getPrecision()[colIn]; } @@ -2322,7 +2360,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, default: { std::ostringstream errmsg; - errmsg << "RowAggregation " << fRGContext.getName() << + errmsg << "RowAggregation " << fRGContextColl[origFuncColsIdx].getName() << ": No logic for data type: " << colDataType; throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); break; @@ -2348,18 +2386,20 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, } // The intermediate values are stored in userData referenced by colAux. - fRGContext.setDataFlags(dataFlags); - fRGContext.setUserData(fRow.getUserData(colAux)); + fRGContextColl[origFuncColsIdx].setDataFlags(dataFlags); + fRGContextColl[origFuncColsIdx].setUserData(fRow.getUserData(colAux)); mcsv1sdk::mcsv1_UDAF::ReturnCode rc; - rc = fRGContext.getFunction()->nextValue(&fRGContext, valsIn); - fRGContext.setUserData(NULL); + rc = fRGContextColl[origFuncColsIdx].getFunction()->nextValue(&fRGContextColl[origFuncColsIdx], + valsIn); + fRGContextColl[origFuncColsIdx].setUserData(NULL); if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) { - RowUDAFFunctionCol* rowUDAF = dynamic_cast(fFunctionCols[funcColsIdx].get()); + RowUDAFFunctionCol* rowUDAF = dynamic_cast(fFunctionCols[origFuncColsIdx].get()); rowUDAF->bInterrupted = true; - throw logging::QueryDataExcept(fRGContext.getErrorMessage(), logging::aggregateFuncErr); + throw logging::QueryDataExcept(fRGContextColl[origFuncColsIdx].getErrorMessage(), + logging::aggregateFuncErr); } } @@ -2758,11 +2798,11 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) return; } - int64_t intOut = 0; - uint64_t uintOut = 0; - float floatOut = 0.0; - double doubleOut = 0.0; - long double longdoubleOut = 0.0; + int64_t intOut; + uint64_t uintOut; + float floatOut; + double doubleOut; + long double longdoubleOut; ostringstream oss; std::string strOut; @@ -2829,6 +2869,12 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) fRow.setIntField<8>(intOut, colOut); bSetSuccess = true; } + else if (valOut.compatible(int128TypeId)) + { + int128_t int128Out = valOut.cast(); + fRow.setInt128Field(int128Out, colOut); + bSetSuccess = true; + } break; @@ -2952,6 +2998,8 @@ void RowAggregationUM::SetUDAFValue(static_any::any& valOut, int64_t colOut) if (!bSetSuccess) { + // This means the return from the UDAF doesn't match the field + // This handles the mismatch SetUDAFAnyValue(valOut, colOut); } } @@ -2964,104 +3012,102 @@ void RowAggregationUM::SetUDAFAnyValue(static_any::any& valOut, int64_t colOut) // that they didn't set in mcsv1_UDAF::init(), but this // handles whatever return type is given and casts // it to whatever they said to return. - // TODO: Save cpu cycles here. + // TODO: Save cpu cycles here. For one, we don't need to initialize these + int64_t intOut = 0; uint64_t uintOut = 0; - float floatOut = 0.0; double doubleOut = 0.0; long double longdoubleOut = 0.0; + int128_t int128Out = 0; ostringstream oss; std::string strOut; if (valOut.compatible(charTypeId)) { - uintOut = intOut = valOut.cast(); - floatOut = intOut; + int128Out = uintOut = intOut = valOut.cast(); + doubleOut = intOut; oss << intOut; } else if (valOut.compatible(scharTypeId)) { - uintOut = intOut = valOut.cast(); - floatOut = intOut; + int128Out = uintOut = intOut = valOut.cast(); + doubleOut = intOut; oss << intOut; } else if (valOut.compatible(shortTypeId)) { - uintOut = intOut = valOut.cast(); - floatOut = intOut; + int128Out = uintOut = intOut = valOut.cast(); + doubleOut = intOut; oss << intOut; } else if (valOut.compatible(intTypeId)) { - uintOut = intOut = valOut.cast(); - floatOut = intOut; + int128Out = uintOut = intOut = valOut.cast(); + doubleOut = intOut; oss << intOut; } else if (valOut.compatible(longTypeId)) { - uintOut = intOut = valOut.cast(); - floatOut = intOut; + int128Out = uintOut = intOut = valOut.cast(); + doubleOut = intOut; oss << intOut; } else if (valOut.compatible(llTypeId)) { - uintOut = intOut = valOut.cast(); - floatOut = intOut; + int128Out = uintOut = intOut = valOut.cast(); + doubleOut = intOut; oss << intOut; } else if (valOut.compatible(ucharTypeId)) { - intOut = uintOut = valOut.cast(); - floatOut = uintOut; + int128Out = intOut = uintOut = valOut.cast(); + doubleOut = uintOut; oss << uintOut; } else if (valOut.compatible(ushortTypeId)) { - intOut = uintOut = valOut.cast(); - floatOut = uintOut; + int128Out = intOut = uintOut = valOut.cast(); + doubleOut = uintOut; oss << uintOut; } else if (valOut.compatible(uintTypeId)) { - intOut = uintOut = valOut.cast(); - floatOut = uintOut; + int128Out = intOut = uintOut = valOut.cast(); + doubleOut = uintOut; oss << uintOut; } else if (valOut.compatible(ulongTypeId)) { - intOut = uintOut = valOut.cast(); - floatOut = uintOut; + int128Out = intOut = uintOut = valOut.cast(); + doubleOut = uintOut; oss << uintOut; } else if (valOut.compatible(ullTypeId)) { - intOut = uintOut = valOut.cast(); - floatOut = uintOut; + int128Out = intOut = uintOut = valOut.cast(); + doubleOut = uintOut; oss << uintOut; } - else if (valOut.compatible(floatTypeId)) + else if (valOut.compatible(int128TypeId)) { - floatOut = valOut.cast(); - doubleOut = floatOut; - longdoubleOut = doubleOut; - intOut = uintOut = floatOut; - oss << floatOut; + intOut = uintOut = int128Out = valOut.cast(); + doubleOut = uintOut; + oss << uintOut; } - else if (valOut.compatible(doubleTypeId)) + else if (valOut.compatible(floatTypeId) || valOut.compatible(doubleTypeId)) { - doubleOut = valOut.cast(); - longdoubleOut = doubleOut; - floatOut = (float)doubleOut; - uintOut = (uint64_t)doubleOut; - intOut = (int64_t)doubleOut; + // Should look at scale for decimal and adjust + doubleOut = valOut.cast(); + int128Out = doubleOut; + intOut = uintOut = doubleOut; oss << doubleOut; } - else if (valOut.compatible(longdoubleTypeId)) { + // Should look at scale for decimal and adjust longdoubleOut = valOut.cast(); + int128Out = longdoubleOut; doubleOut = (double)longdoubleOut; - floatOut = (float)doubleOut; uintOut = (uint64_t)doubleOut; intOut = (int64_t)doubleOut; oss << doubleOut; @@ -3075,7 +3121,7 @@ void RowAggregationUM::SetUDAFAnyValue(static_any::any& valOut, int64_t colOut) uintOut = strtoul(strOut.c_str(), NULL, 10); doubleOut = strtod(strOut.c_str(), NULL); longdoubleOut = strtold(strOut.c_str(), NULL); - floatOut = (float)doubleOut; + int128Out = longdoubleOut; } else { @@ -3099,11 +3145,20 @@ void RowAggregationUM::SetUDAFAnyValue(static_any::any& valOut, int64_t colOut) break; case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: fRow.setIntField<8>(intOut, colOut); break; + case execplan::CalpontSystemCatalog::DECIMAL: + case execplan::CalpontSystemCatalog::UDECIMAL: + { + uint32_t width = fRowGroupOut->getColumnWidth(colOut); + if (width == datatypes::MAXDECIMALWIDTH) + fRow.setInt128Field(int128Out, colOut); + else + fRow.setIntField<8>(intOut, colOut); + break; + } + case execplan::CalpontSystemCatalog::UTINYINT: fRow.setUintField<1>(uintOut, colOut); break; @@ -3135,8 +3190,11 @@ void RowAggregationUM::SetUDAFAnyValue(static_any::any& valOut, int64_t colOut) case execplan::CalpontSystemCatalog::FLOAT: case execplan::CalpontSystemCatalog::UFLOAT: + { + float floatOut = (float)doubleOut; fRow.setFloatField(floatOut, colOut); break; + } case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::UDOUBLE: @@ -3407,10 +3465,28 @@ void RowAggregationUM::doNullConstantAggregate(const ConstantAggData& aggData, u case execplan::CalpontSystemCatalog::MEDINT: case execplan::CalpontSystemCatalog::INT: case execplan::CalpontSystemCatalog::BIGINT: + { + fRow.setIntField(getIntNullValue(colDataType), colOut); + } + break; + case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - fRow.setIntField(getIntNullValue(colDataType), colOut); + auto width = fRow.getColumnWidth(colOut); + if (fRow.getColumnWidth(colOut) == datatypes::MAXDECIMALWIDTH) + { + fRow.setInt128Field(datatypes::Decimal128Null, colOut); + } + else if (width <= datatypes::MAXLEGACYWIDTH) + { + fRow.setIntField(getIntNullValue(colDataType), colOut); + } + else + { + idbassert(0); + throw std::logic_error("RowAggregationUM::doNullConstantAggregate(): DECIMAL bad length."); + } } break; @@ -3570,7 +3646,7 @@ void RowAggregationUM::doNullConstantAggregate(const ConstantAggData& aggData, u void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData, uint64_t i) { int64_t colOut = fFunctionCols[i]->fOutputColumnIndex; - int colDataType = (fRowGroupOut->getColTypes())[colOut]; + auto colDataType = (fRowGroupOut->getColTypes())[colOut]; int64_t rowCnt = fRow.getIntField(fFunctionCols[i]->fAuxColumnIndex); switch (aggData.fOp) @@ -3592,7 +3668,7 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData { fRow.setIntField(strtol(aggData.fConstValue.c_str(), 0, 10), colOut); } - break; + break; // AVG should not be uint32_t result type. case execplan::CalpontSystemCatalog::UTINYINT: @@ -3608,9 +3684,27 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - double dbl = strtod(aggData.fConstValue.c_str(), 0); - double scale = pow(10.0, (double) fRowGroupOut->getScale()[i]); - fRow.setIntField((int64_t)(scale * dbl), colOut); + auto width = fRow.getColumnWidth(colOut); + if (width == datatypes::MAXDECIMALWIDTH) + { + ColTypeAlias colType; + colType.colWidth = width; + colType.precision = fRow.getPrecision(i); + colType.scale = fRow.getScale(i); + colType.colDataType = colDataType; + fRow.setInt128Field(Dec::int128FromString(aggData.fConstValue, colType), colOut); + } + else if (width <= datatypes::MAXLEGACYWIDTH) + { + double dbl = strtod(aggData.fConstValue.c_str(), 0); + double scale = pow(10.0, (double) fRowGroupOut->getScale()[i]); + fRow.setIntField((int64_t)(scale * dbl), colOut); + } + else + { + idbassert(0); + throw std::logic_error("RowAggregationUM::doNotNullConstantAggregate(): DECIMAL bad length."); + } } break; @@ -3715,15 +3809,39 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - double dbl = strtod(aggData.fConstValue.c_str(), 0); - dbl *= pow(10.0, (double) fRowGroupOut->getScale()[i]); - dbl *= rowCnt; + auto width = fRow.getColumnWidth(colOut); + if (width == datatypes::MAXDECIMALWIDTH) + { + ColTypeAlias colType; + colType.colWidth = width; + colType.precision = fRow.getPrecision(i); + colType.scale = fRow.getScale(i); + colType.colDataType = colDataType; + int128_t constValue = Dec::int128FromString(aggData.fConstValue, + colType); + int128_t sum; - if ((dbl > 0 && dbl > (double) numeric_limits::max()) || - (dbl < 0 && dbl < (double) numeric_limits::min())) - throw logging::QueryDataExcept(overflowMsg, logging::aggregateDataErr); + datatypes::MultiplicationOverflowCheck multOp; + multOp(constValue, rowCnt, sum); + fRow.setInt128Field(sum, colOut); + } + else if (width == datatypes::MAXLEGACYWIDTH) + { + double dbl = strtod(aggData.fConstValue.c_str(), 0); + dbl *= pow(10.0, (double) fRowGroupOut->getScale()[i]); + dbl *= rowCnt; + + if ((dbl > 0 && dbl > (double) numeric_limits::max()) || + (dbl < 0 && dbl < (double) numeric_limits::min())) + throw logging::QueryDataExcept(overflowMsg, logging::aggregateDataErr); + fRow.setIntField((int64_t) dbl, colOut); + } + else + { + idbassert(0); + throw std::logic_error("RowAggregationUM::doNotNullConstantAggregate(): sum() DECIMAL bad length."); + } - fRow.setIntField((int64_t) dbl, colOut); } break; @@ -4459,7 +4577,7 @@ void RowAggregationUMP2::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, if (!userDataIn) { - if (fRGContext.getRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS)) + if (fRGContextColl[funcColsIdx].getRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS)) { return; } @@ -4468,14 +4586,14 @@ void RowAggregationUMP2::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, flags[0] |= mcsv1sdk::PARAM_IS_NULL; } - fRGContext.setDataFlags(flags); + fRGContextColl[funcColsIdx].setDataFlags(flags); // The intermediate values are stored in colAux. - fRGContext.setUserData(fRow.getUserData(colAux)); + fRGContextColl[funcColsIdx].setUserData(fRow.getUserData(colAux)); // Call the UDAF subEvaluate method mcsv1sdk::mcsv1_UDAF::ReturnCode rc; - rc = fRGContext.getFunction()->subEvaluate(&fRGContext, userDataIn.get()); + rc = fRGContextColl[funcColsIdx].getFunction()->subEvaluate(&fRGContextColl[funcColsIdx], userDataIn.get()); fRGContext.setUserData(NULL); if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) diff --git a/utils/rowgroup/rowaggregation.h b/utils/rowgroup/rowaggregation.h index 3c138b1cf..7fb4a10dc 100644 --- a/utils/rowgroup/rowaggregation.h +++ b/utils/rowgroup/rowaggregation.h @@ -645,6 +645,7 @@ protected: } void resetUDAF(RowUDAFFunctionCol* rowUDAF); + void resetUDAF(RowUDAFFunctionCol* rowUDAF, uint64_t funcColIdx); inline bool isNull(const RowGroup* pRowGroup, const Row& row, int64_t col); inline void makeAggFieldsNull(Row& row); @@ -710,6 +711,7 @@ protected: // We need a separate copy for each thread. mcsv1sdk::mcsv1Context fRGContext; + std::vector fRGContextColl; // These are handy for testing the actual type of static_any for UDAF static const static_any::any& charTypeId; @@ -718,10 +720,12 @@ protected: static const static_any::any& intTypeId; static const static_any::any& longTypeId; static const static_any::any& llTypeId; + static const static_any::any& int128TypeId; static const static_any::any& ucharTypeId; static const static_any::any& ushortTypeId; static const static_any::any& uintTypeId; static const static_any::any& ulongTypeId; + static const static_any::any& uint128TypeId; static const static_any::any& ullTypeId; static const static_any::any& floatTypeId; static const static_any::any& doubleTypeId; diff --git a/utils/udfsdk/mcsv1_udaf.cpp b/utils/udfsdk/mcsv1_udaf.cpp index f23d53c52..2ad8905a8 100755 --- a/utils/udfsdk/mcsv1_udaf.cpp +++ b/utils/udfsdk/mcsv1_udaf.cpp @@ -278,13 +278,13 @@ const static_any::any& mcsv1_UDAF::shortTypeId((short)1); const static_any::any& mcsv1_UDAF::intTypeId((int)1); const static_any::any& mcsv1_UDAF::longTypeId((long)1); const static_any::any& mcsv1_UDAF::llTypeId((long long)1); -const static_any::any& mcsv1_UDAF::int128TypeId((int128_t)1); +const static_any::any& mcsv1_UDAF::int128TypeId((__int128)1); const static_any::any& mcsv1_UDAF::ucharTypeId((unsigned char)1); const static_any::any& mcsv1_UDAF::ushortTypeId((unsigned short)1); const static_any::any& mcsv1_UDAF::uintTypeId((unsigned int)1); const static_any::any& mcsv1_UDAF::ulongTypeId((unsigned long)1); const static_any::any& mcsv1_UDAF::ullTypeId((unsigned long long)1); -const static_any::any& mcsv1_UDAF::uint128TypeId((uint128_t)1); +const static_any::any& mcsv1_UDAF::uint128TypeId((unsigned __int128)1); const static_any::any& mcsv1_UDAF::floatTypeId((float)1); const static_any::any& mcsv1_UDAF::doubleTypeId((double)1); const static_any::any& mcsv1_UDAF::strTypeId(typeStr); diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index 62d4a2714..75075b778 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -544,13 +544,13 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, static const static_any::any& intTypeId = (int)1; static const static_any::any& longTypeId = (long)1; static const static_any::any& llTypeId = (long long)1; - static const static_any::any& int128TypeId = (int128_t)1; + static const static_any::any& int128TypeId = (__int128)1; static const static_any::any& ucharTypeId = (unsigned char)1; static const static_any::any& ushortTypeId = (unsigned short)1; static const static_any::any& uintTypeId = (unsigned int)1; static const static_any::any& ulongTypeId = (unsigned long)1; static const static_any::any& ullTypeId = (unsigned long long)1; - static const static_any::any& uint128TypeId = (uint128_t)1; + static const static_any::any& uint128TypeId = (unsigned __int128)1; static const static_any::any& floatTypeId = (float)1; static const static_any::any& doubleTypeId = (double)1; static const std::string typeStr(""); From 7d3d8287900cc618c32873ac7dbabad16c9c57aa Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Wed, 14 Oct 2020 13:12:05 -0400 Subject: [PATCH 62/78] MCOL-641 This commit fixes regression introduced by an earlier commit for group_concat() on a narrow decimal field. --- dbcon/joblist/groupconcat.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index 09b57a325..78fdf24c2 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -471,16 +471,15 @@ void GroupConcator::outputRow(std::ostringstream& oss, const rowgroup::Row& row) else { int64_t intVal = row.getIntField(*i); + if (scale == 0) { oss << intVal; } else { - char buf[utils::MAXLENGTH8BYTES]; - dataconvert::DataConvert::decimalToString(intVal, - scale, buf, sizeof(buf), types[*i]); - oss << fixed << buf; + long double dblVal = intVal / pow(10.0, (double)scale); + oss << fixed << setprecision(scale) << dblVal; } } From 844472d812cb2ccdd980e283167dc360bff2e34b Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 13 Oct 2020 12:43:16 +0000 Subject: [PATCH 63/78] MCOL-4313 Very fragile but high speed approach with inline ASM GCC compiler uses aligned versions of SIMD instructions expecting aligned memory blocks that is hard to implement now --- primitives/primproc/columncommand.cpp | 19 +++++++++++++-- utils/messageqcpp/bytestream.cpp | 35 ++++++++++++++++++++++++--- utils/messageqcpp/bytestream.h | 1 + utils/rowgroup/rowgroup.h | 16 ++++++++++++ 4 files changed, 65 insertions(+), 6 deletions(-) diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index d1b122466..62ac4f66f 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -231,7 +231,11 @@ void ColumnCommand::loadData() { ByteStream::hexbyte h; utils::getEmptyRowValue(colType.colDataType, colType.colWidth, (uint8_t*)&h); - hPtr[idx] = h; + __asm__ volatile("movups %1,%0" + :"=m" ( hPtr[idx] ) // output + :"v"( h ) // input + : "memory" // clobbered + ); } } @@ -328,7 +332,18 @@ void ColumnCommand::process_OT_BOTH() bpp->relRids[i] = *((uint16_t*) &bpp->outputMsg[pos]); pos += 2; - wide128Values[i] = *((int128_t*) &bpp->outputMsg[pos]); + int128_t* int128Ptr = reinterpret_cast(&bpp->outputMsg[pos]); + __asm__ volatile("movdqu %0,%%xmm0;" + : + :"m"( *int128Ptr ) // input + :"xmm0" // clobbered + ); + __asm__ volatile("movups %%xmm0,%0;" + : "=m" (wide128Values[i])// output + : // input + : "memory", "xmm0" // clobbered + ); + pos += 16; } diff --git a/utils/messageqcpp/bytestream.cpp b/utils/messageqcpp/bytestream.cpp index c0fad8554..d42239eb0 100644 --- a/utils/messageqcpp/bytestream.cpp +++ b/utils/messageqcpp/bytestream.cpp @@ -240,7 +240,12 @@ ByteStream& ByteStream::operator<<(const uint128_t& o) if (fBuf == 0 || (fCurInPtr - fBuf + 16U > fMaxLen + ISSOverhead)) growBuf(fMaxLen + BlockSize); - *((uint128_t*) fCurInPtr) = o; + __asm__ volatile("movups %1,%0;" + :"=m" ( *fCurInPtr ) // output + :"v"( o ) // input + : "memory" // clobbered + ); + fCurInPtr += 16; return *this; @@ -251,7 +256,11 @@ ByteStream& ByteStream::operator<<(const int128_t& o) if (fBuf == 0 || (fCurInPtr - fBuf + 16U > fMaxLen + ISSOverhead)) growBuf(fMaxLen + BlockSize); - *((int128_t*) fCurInPtr) = o; + __asm__ volatile("movups %1,%0;" + :"=m" ( *fCurInPtr ) // output + :"v"( o ) // input + : "memory" // clobbered + ); fCurInPtr += 16; return *this; @@ -441,7 +450,16 @@ void ByteStream::peek(uint128_t& o) const if (length() < 16) throw underflow_error("ByteStream>uint128_t: not enough data in stream to fill datatype"); - o = *((uint128_t*) fCurOutPtr); + __asm__ volatile("movdqu %0,%%xmm0;" + : + :"m"( *fCurOutPtr ) // input + :"xmm0" // clobbered + ); + __asm__ volatile("movups %%xmm0,%0;" + : "=m" (o)// output + : // input + : "memory", "xmm0" // clobbered + ); } void ByteStream::peek(int128_t& o) const @@ -450,7 +468,16 @@ void ByteStream::peek(int128_t& o) const if (length() < 16) throw underflow_error("ByteStream>int128_t: not enough data in stream to fill datatype"); - o = *((int128_t*) fCurOutPtr); + __asm__ volatile("movdqu %0,%%xmm0;" + : + :"m"( *fCurOutPtr ) // input + :"xmm0" // clobbered + ); + __asm__ volatile("movups %%xmm0,%0;" + : "=m" (o)// output + : // input + : "memory", "xmm0" // clobbered + ); } void ByteStream::peek(string& s) const diff --git a/utils/messageqcpp/bytestream.h b/utils/messageqcpp/bytestream.h index 3eccb7b0f..ade220d76 100644 --- a/utils/messageqcpp/bytestream.h +++ b/utils/messageqcpp/bytestream.h @@ -151,6 +151,7 @@ public: * push an int128_t onto the end of the stream. The byte order is whatever the native byte order is. */ EXPORT ByteStream& operator<<(const int128_t& o); + /** * push an uint128_t onto the end of the stream. The byte order is whatever the native byte order is. */ diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 748bf3dfa..9064aa8b8 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -848,6 +848,22 @@ inline void Row::setBinaryField_offset(uint8_t* value, uint32_t width, memcpy(&data[offset], value, width); } +template<> +inline void Row::setBinaryField_offset(int128_t* value, uint32_t width, uint32_t offset) +{ + int128_t *dst128Ptr = reinterpret_cast(&data[offset]); + __asm__ volatile("movdqu %0,%%xmm0;" + : + :"m"( *value ) // input + :"xmm0" // clobbered + ); + __asm__ volatile("movups %%xmm0,%0;" + : "=m" (*dst128Ptr)// output + : // input + : "memory", "xmm0" // clobbered + ); +} + inline void Row::setStringField(const uint8_t* strdata, uint32_t length, uint32_t colIndex) { uint64_t offset; From 1f4a7817045677a666ace83fb73539fa1bb825a2 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Thu, 22 Oct 2020 12:16:48 -0400 Subject: [PATCH 64/78] MCOL-641 Fixes for arithmetic operations. 1. Perform type promotion to wide decimal if the result of an arithmetic operation has a precision > 18. 2. Only set the decimal width of an arithmetic operation to wide if both the LHS and RHS of the operation are decimal types. --- datatypes/mcs_decimal.h | 15 ++++++ dbcon/joblist/tupleaggregatestep.cpp | 51 ++++++++++++-------- dbcon/mysql/ha_mcs_execplan.cpp | 71 +++++++++++----------------- 3 files changed, 72 insertions(+), 65 deletions(-) diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index 329487743..178dce1ba 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -470,6 +470,21 @@ class Decimal scale += (scaleAvailable >= MAXSCALEINC4AVG) ? MAXSCALEINC4AVG : scaleAvailable; precision += (precisionAvailable >= MAXSCALEINC4AVG) ? MAXSCALEINC4AVG : precisionAvailable; } + + /** + @brief Returns true if all arguments have a DECIMAL/UDECIMAL type + */ + static inline bool isDecimalOperands(const ColDataTypeAlias resultDataType, + const ColDataTypeAlias leftColDataType, + const ColDataTypeAlias rightColDataType) + { + return ((resultDataType == execplan::CalpontSystemCatalog::DECIMAL || + resultDataType == execplan::CalpontSystemCatalog::UDECIMAL) && + (leftColDataType == execplan::CalpontSystemCatalog::DECIMAL || + leftColDataType == execplan::CalpontSystemCatalog::UDECIMAL) && + (rightColDataType == execplan::CalpontSystemCatalog::DECIMAL || + rightColDataType == execplan::CalpontSystemCatalog::UDECIMAL)); + } }; /** diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index 50767ab36..90d9f9148 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -343,12 +343,8 @@ namespace joblist void wideDecimalOrLongDouble(const uint64_t colProj, const CalpontSystemCatalog::ColDataType type, const vector& precisionProj, - const vector& oidsProj, - const uint32_t aggKey, const vector& scaleProj, const vector& width, - vector& oidsAgg, - vector& keysAgg, vector& typeAgg, vector& scaleAgg, vector& precisionAgg, @@ -358,8 +354,6 @@ void wideDecimalOrLongDouble(const uint64_t colProj, || type == CalpontSystemCatalog::UDECIMAL) && datatypes::Decimal::isWideDecimalType(precisionProj[colProj])) { - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(aggKey); typeAgg.push_back(type); scaleAgg.push_back(scaleProj[colProj]); precisionAgg.push_back(precisionProj[colProj]); @@ -367,8 +361,6 @@ void wideDecimalOrLongDouble(const uint64_t colProj, } else { - oidsAgg.push_back(oidsProj[colProj]); - keysAgg.push_back(aggKey); typeAgg.push_back(CalpontSystemCatalog::LONGDOUBLE); scaleAgg.push_back(0); precisionAgg.push_back(-1); @@ -758,29 +750,38 @@ void TupleAggregateStep::configDeliveredRowGroup(const JobInfo& jobInfo) if (jobInfo.havingStep) { retColCount = jobInfo.returnedColVec.size(); + idbassert(jobInfo.returnedColVec.size() == jobInfo.nonConstCols.size()); - for (auto& rc : jobInfo.nonConstCols) + + for (size_t i = 0; i < jobInfo.nonConstCols.size() && + scaleIter != scale.end(); i++) { - auto& colType = rc->resultType(); + const auto& colType = jobInfo.nonConstCols[i]->resultType(); + if (datatypes::Decimal::isWideDecimalType(colType)) { *scaleIter = colType.scale; *precisionIter = colType.precision; } + scaleIter++; precisionIter++; } } else { retColCount = jobInfo.nonConstDelCols.size(); - for (auto& rc : jobInfo.nonConstDelCols) + + for (size_t i = 0; i < jobInfo.nonConstDelCols.size() && + scaleIter != scale.end(); i++) { - auto& colType = rc->resultType(); + const auto& colType = jobInfo.nonConstDelCols[i]->resultType(); + if (datatypes::Decimal::isWideDecimalType(colType)) { *scaleIter = colType.scale; *precisionIter = colType.precision; } + scaleIter++; precisionIter++; } } @@ -1356,10 +1357,13 @@ void TupleAggregateStep::prep1PhaseAggregate( cerr << "prep1PhaseAggregate: " << emsg << endl; throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); } + wideDecimalOrLongDouble(colProj, typeProj[colProj], - precisionProj, oidsProj, key, scaleProj, width, - oidsAgg, keysAgg, typeAgg, scaleAgg, - precisionAgg, widthAgg); + precisionProj, scaleProj, width, + typeAgg, scaleAgg, precisionAgg, widthAgg); + + oidsAgg.push_back(oidsProj[colProj]); + keysAgg.push_back(key); csNumAgg.push_back(csNumProj[colProj]); } break; @@ -3181,10 +3185,13 @@ void TupleAggregateStep::prep2PhasesAggregate( cerr << "prep2PhasesAggregate: " << emsg << endl; throw IDBExcept(emsg, ERR_AGGREGATE_TYPE_NOT_SUPPORT); } + wideDecimalOrLongDouble(colProj, typeProj[colProj], - precisionProj, oidsProj, aggKey, scaleProj, width, - oidsAggPm, keysAggPm, typeAggPm, scaleAggPm, - precisionAggPm, widthAggPm); + precisionProj, scaleProj, width, + typeAggPm, scaleAggPm, precisionAggPm, widthAggPm); + + oidsAggPm.push_back(oidsProj[colProj]); + keysAggPm.push_back(aggKey); csNumAggPm.push_back(8); colAggPm++; } @@ -3469,9 +3476,11 @@ void TupleAggregateStep::prep2PhasesAggregate( if (aggOp == ROWAGG_SUM) { wideDecimalOrLongDouble(colPm, typeProj[colPm], - precisionProj, oidsProj, retKey, scaleProj, widthAggPm, - oidsAggUm, keysAggUm, typeAggUm, scaleAggUm, - precisionAggUm, widthAggUm); + precisionProj, scaleProj, widthAggPm, + typeAggUm, scaleAggUm, precisionAggUm, widthAggUm); + + oidsAggUm.push_back(oidsProj[colPm]); + keysAggUm.push_back(retKey); csNumAggUm.push_back(8); } else diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index aa5dd5184..666bb467a 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -3602,68 +3602,51 @@ ArithmeticColumn* buildArithmeticColumn( // @bug5715. Use InfiniDB adjusted coltype for result type. // decimal arithmetic operation gives double result when the session variable is set. - CalpontSystemCatalog::ColType mysql_type = colType_MysqlToIDB(item); + CalpontSystemCatalog::ColType mysqlType = colType_MysqlToIDB(item); - if (mysql_type.colDataType == CalpontSystemCatalog::DECIMAL || - mysql_type.colDataType == CalpontSystemCatalog::UDECIMAL) + const CalpontSystemCatalog::ColType& leftColType = pt->left()->data()->resultType(); + const CalpontSystemCatalog::ColType& rightColType = pt->right()->data()->resultType(); + + // Only tinker with the type if all columns involved are decimal + if (datatypes::Decimal::isDecimalOperands(mysqlType.colDataType, + leftColType.colDataType, rightColType.colDataType)) { - int32_t leftColWidth = pt->left()->data()->resultType().colWidth; - int32_t rightColWidth = pt->right()->data()->resultType().colWidth; + int32_t leftColWidth = leftColType.colWidth; + int32_t rightColWidth = rightColType.colWidth; - // Revert back to legacy values of scale and precision - // if the 2 columns involved in the expression are not wide - if (leftColWidth <= utils::MAXLEGACYWIDTH && - rightColWidth <= utils::MAXLEGACYWIDTH) + if (leftColWidth == datatypes::MAXDECIMALWIDTH || + rightColWidth == datatypes::MAXDECIMALWIDTH) { - Item_decimal* idp = (Item_decimal*)item; + mysqlType.colWidth = datatypes::MAXDECIMALWIDTH; - mysql_type.colWidth = 8; + string funcName = item->func_name(); - unsigned int precision = idp->max_length; - unsigned int scale = idp->decimals; + int32_t scale1 = leftColType.scale; + int32_t scale2 = rightColType.scale; - datatypes::Decimal::setDecimalScalePrecisionLegacy(mysql_type, - precision, scale); - } - else - { - - if (leftColWidth == datatypes::MAXDECIMALWIDTH || - rightColWidth == datatypes::MAXDECIMALWIDTH) - mysql_type.colWidth = datatypes::MAXDECIMALWIDTH; - - if (mysql_type.colWidth == datatypes::MAXDECIMALWIDTH) + if (funcName == "/" && + (mysqlType.scale - (scale1 - scale2)) > datatypes::INT128MAXPRECISION) { - string funcName = item->func_name(); + Item_decimal* idp = (Item_decimal*)item; - int32_t scale1 = pt->left()->data()->resultType().scale; - int32_t scale2 = pt->right()->data()->resultType().scale; + unsigned int precision = idp->decimal_precision(); + unsigned int scale = idp->decimal_scale(); - if (funcName == "/" && - (mysql_type.scale - (scale1 - scale2)) > - datatypes::INT128MAXPRECISION) - { - Item_decimal* idp = (Item_decimal*)item; + datatypes::Decimal::setDecimalScalePrecisionHeuristic(mysqlType, precision, scale); - unsigned int precision = idp->decimal_precision(); - unsigned int scale = idp->decimal_scale(); + if (mysqlType.scale < scale1) + mysqlType.scale = scale1; - datatypes::Decimal::setDecimalScalePrecisionHeuristic(mysql_type, precision, scale); - - if (mysql_type.scale < scale1) - mysql_type.scale = scale1; - - if (mysql_type.precision < mysql_type.scale) - mysql_type.precision = mysql_type.scale; - } + if (mysqlType.precision < mysqlType.scale) + mysqlType.precision = mysqlType.scale; } } } if (get_double_for_decimal_math(current_thd) == true) - aop->adjustResultType(mysql_type); + aop->adjustResultType(mysqlType); else - aop->resultType(mysql_type); + aop->resultType(mysqlType); // adjust decimal result type according to internalDecimalScale if (gwi.internalDecimalScale >= 0 && aop->resultType().colDataType == CalpontSystemCatalog::DECIMAL) From 68244ab957532514f6a689fa35ea779e4ff68da0 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Thu, 29 Oct 2020 16:04:34 -0400 Subject: [PATCH 65/78] MCOL-641 Fix regression in aggregate distinct on narrow decimal. The else if block in Row::equals() was incorrectly getting triggered for narrow decimals earlier. We now specifically check if the column is a wide decimal. Furthermore, we need to dereference the int128_t pointers for equality comparison. --- utils/rowgroup/rowgroup.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 7a8ec8a28..731cf61e5 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -1203,9 +1203,9 @@ bool Row::equals(const Row& r2, const std::vector& keyCols) const if (getLongDoubleField(col) != r2.getLongDoubleField(col)) return false; } - else if (UNLIKELY(execplan::isDecimal(columnType))) + else if (UNLIKELY(datatypes::Decimal::isWideDecimalType(columnType, colWidths[col]))) { - if (getBinaryField(col) != r2.getBinaryField(col)) + if (*getBinaryField(col) != *r2.getBinaryField(col)) return false; } else if (getUintField(col) != r2.getUintField(col)) @@ -1217,6 +1217,7 @@ bool Row::equals(const Row& r2, const std::vector& keyCols) const return true; } + bool Row::equals(const Row& r2, uint32_t lastCol) const { // This check fires with empty r2 only. @@ -1261,9 +1262,9 @@ bool Row::equals(const Row& r2, uint32_t lastCol) const if (getLongDoubleField(col) != r2.getLongDoubleField(col)) return false; } - else if (UNLIKELY(execplan::isDecimal(columnType))) + else if (UNLIKELY(datatypes::Decimal::isWideDecimalType(columnType, colWidths[col]))) { - if (getBinaryField(col) != r2.getBinaryField(col)) + if (*getBinaryField(col) != *r2.getBinaryField(col)) return false; } else if (getUintField(col) != r2.getUintField(col)) @@ -1651,8 +1652,8 @@ void applyMapping(const int* mapping, const Row& in, Row* out) // WIP this doesn't look right b/c we can pushdown colType // Migrate to offset based methods here // code precision 2 width convertor - else if (UNLIKELY(execplan::isDecimal(in.getColTypes()[i]) - && in.getColumnWidth(i) == 16)) + else if (UNLIKELY(datatypes::Decimal::isWideDecimalType(in.getColTypes()[i], + in.getColumnWidth(i)))) out->setBinaryField_offset(in.getBinaryField(i), 16, out->getOffset(mapping[i])); else if (in.isUnsigned(i)) From 129d5b5a0ffa2f6f3d8d7686db4e4cbac454b56b Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Tue, 20 Oct 2020 15:35:36 +0400 Subject: [PATCH 66/78] MCOL-4174 Review/refactor frontend/connector code --- datatypes/mcs_datatype.cpp | 1766 ++++++++++++ datatypes/mcs_datatype.h | 2510 +++++++++++++++++ datatypes/mcs_decimal.cpp | 126 +- datatypes/mcs_decimal.h | 481 +++- dbcon/ddlpackageproc/createindexprocessor.cpp | 8 +- dbcon/ddlpackageproc/ddlindexpopulator.h | 2 +- dbcon/ddlpackageproc/ddlpackageprocessor.cpp | 232 +- dbcon/ddlpackageproc/ddlpackageprocessor.h | 6 - dbcon/dmlpackageproc/dmlpackageprocessor.cpp | 1 - dbcon/execplan/arithmeticoperator.h | 4 +- dbcon/execplan/calpontsystemcatalog.cpp | 55 +- dbcon/execplan/calpontsystemcatalog.h | 190 +- dbcon/execplan/functioncolumn.h | 2 +- dbcon/execplan/treenode.h | 289 +- dbcon/joblist/batchprimitiveprocessor-jl.cpp | 2 +- dbcon/joblist/columncommand-jl.cpp | 2 +- dbcon/joblist/crossenginestep.cpp | 5 +- dbcon/joblist/groupconcat.cpp | 6 +- dbcon/joblist/jlf_common.cpp | 2 +- dbcon/joblist/jlf_execplantojoblist.cpp | 15 +- dbcon/joblist/jlf_tuplejoblist.cpp | 2 +- dbcon/joblist/lbidlist.cpp | 11 +- dbcon/joblist/pseudocc-jl.cpp | 8 +- dbcon/joblist/rowestimator.cpp | 16 +- dbcon/joblist/tuple-bps.cpp | 8 +- dbcon/joblist/tupleaggregatestep.cpp | 6 +- dbcon/joblist/tuplehashjoin.cpp | 2 +- dbcon/mysql/CMakeLists.txt | 1 + dbcon/mysql/ha_mcs_datatype.h | 1301 +++++++++ dbcon/mysql/ha_mcs_ddl.cpp | 158 +- dbcon/mysql/ha_mcs_dml.cpp | 1082 +------ dbcon/mysql/ha_mcs_execplan.cpp | 104 +- dbcon/mysql/ha_mcs_impl.cpp | 441 +-- dbcon/mysql/ha_mcs_partition.cpp | 1139 +------- dbcon/mysql/is_columnstore_extents.cpp | 9 +- primitives/primproc/columncommand.cpp | 2 +- primitives/primproc/filtercommand.cpp | 8 +- primitives/primproc/pseudocc.cpp | 4 +- tests/dataconvert-tests.cpp | 105 +- tests/mcs_decimal-tests.cpp | 118 +- tests/rowgroup-tests.cpp | 3 +- tools/editem/editem.cpp | 13 +- utils/cloudio/CMakeLists.txt | 2 +- utils/common/emptyvaluemanip.cpp | 3 +- utils/common/widedecimalutils.h | 42 - utils/dataconvert/dataconvert.cpp | 1193 ++++---- utils/dataconvert/dataconvert.h | 16 +- utils/funcexp/func_round.cpp | 14 +- utils/funcexp/func_truncate.cpp | 14 +- utils/joiner/tuplejoiner.cpp | 19 +- utils/regr/moda.cpp | 2 +- utils/rowgroup/rowaggregation.cpp | 12 +- utils/rowgroup/rowgroup.cpp | 17 +- utils/rowgroup/rowgroup.h | 12 +- utils/udfsdk/mcsv1_udaf.cpp | 2 +- utils/windowfunction/windowfunctiontype.cpp | 2 +- versioning/BRM/CMakeLists.txt | 3 +- versioning/BRM/dbrm.cpp | 5 +- versioning/BRM/extentmap.cpp | 1 - writeengine/bulk/we_brmreporter.cpp | 9 +- writeengine/bulk/we_bulkloadbuffer.cpp | 1 - writeengine/bulk/we_bulkloadbuffer.h | 1 - writeengine/bulk/we_colextinf.cpp | 1 - writeengine/bulk/we_colextinf.h | 1 - writeengine/server/CMakeLists.txt | 3 +- writeengine/server/we_ddlcommandproc.cpp | 28 +- writeengine/server/we_ddlcommon.h | 233 -- writeengine/server/we_dmlcommandproc.cpp | 22 +- writeengine/wrapper/we_colop.cpp | 1 - writeengine/wrapper/writeengine.cpp | 5 +- 70 files changed, 6982 insertions(+), 4927 deletions(-) create mode 100644 datatypes/mcs_datatype.cpp create mode 100644 datatypes/mcs_datatype.h create mode 100644 dbcon/mysql/ha_mcs_datatype.h diff --git a/datatypes/mcs_datatype.cpp b/datatypes/mcs_datatype.cpp new file mode 100644 index 000000000..82c87ade9 --- /dev/null +++ b/datatypes/mcs_datatype.cpp @@ -0,0 +1,1766 @@ +/* + Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include +#include +#include +#ifdef _MSC_VER +#include +#else +#include +#endif +#include +#include +#include +#include +#ifdef _MSC_VER +#include +#else +#include +#endif +#include +#include +using namespace std; + +#include + + +#include "dataconvert.h" +using namespace dataconvert; + +#include "simplecolumn.h" +#include "simplecolumn_int.h" +#include "simplecolumn_uint.h" +#include "simplecolumn_decimal.h" + +#include "rowgroup.h" +#include "ddlpkg.h" +#include "dbrm.h" +#include "we_typeext.h" +#include "joblisttypes.h" + +namespace datatypes +{ + +int128_t SystemCatalog::TypeAttributesStd::decimal128FromString(const std::string& value) const +{ + int128_t result = 0; + bool pushWarning = false; + bool noRoundup = false; + dataconvert::number_int_value(value, + SystemCatalog::DECIMAL, + *this, + pushWarning, + noRoundup, + result); + return result; +} + + +const string & TypeHandlerSInt8::name() const +{ + static const string xname= "TINYINT"; + return xname; +} + + +const string & TypeHandlerUInt8::name() const +{ + static const string xname= "UTINYINT"; + return xname; +} + + +const string & TypeHandlerSInt16::name() const +{ + static const string xname= "SMALLINT"; + return xname; +} + + +const string & TypeHandlerUInt16::name() const +{ + static const string xname= "USMALLINT"; + return xname; +} + + +const string & TypeHandlerSInt24::name() const +{ + static const string xname= "MEDINT"; + return xname; +} + + +const string & TypeHandlerUInt24::name() const +{ + static const string xname= "UMEDINT"; + return xname; +} + + +const string & TypeHandlerSInt32::name() const +{ + static const string xname= "INT"; + return xname; +} + + +const string & TypeHandlerUInt32::name() const +{ + static const string xname= "UINT"; + return xname; +} + + +const string & TypeHandlerSInt64::name() const +{ + static const string xname= "BIGINT"; + return xname; +} + + +const string & TypeHandlerUInt64::name() const +{ + static const string xname= "UBIGINT"; + return xname; +} + + +const string & TypeHandlerSFloat::name() const +{ + static const string xname= "FLOAT"; + return xname; +} + + +const string & TypeHandlerUFloat::name() const +{ + static const string xname= "UFLOAT"; + return xname; +} + + +const string & TypeHandlerSDouble::name() const +{ + static const string xname= "DOUBLE"; + return xname; +} + + +const string & TypeHandlerUDouble::name() const +{ + static const string xname= "UDOUBLE"; + return xname; +} + + +const string & TypeHandlerSLongDouble::name() const +{ + static const string xname= "LONGDOUBLE"; + return xname; +} + + +const string & TypeHandlerSDecimal64::name() const +{ + static const string xname= "DECIMAL"; + return xname; +} + + +const string & TypeHandlerUDecimal64::name() const +{ + static const string xname= "UDECIMAL"; + return xname; +} + + +const string & TypeHandlerSDecimal128::name() const +{ + static const string xname= "DECIMAL"; + return xname; +} + + +const string & TypeHandlerUDecimal128::name() const +{ + static const string xname= "UDECIMAL"; + return xname; +} + + +const string & TypeHandlerDate::name() const +{ + static const string xname= "DATE"; + return xname; +} + + +const string & TypeHandlerDatetime::name() const +{ + static const string xname= "DATETIME"; + return xname; +} + + +const string & TypeHandlerTime::name() const +{ + static const string xname= "TIME"; + return xname; +} + + +const string & TypeHandlerTimestamp::name() const +{ + static const string xname= "TIMESTAMP"; + return xname; +} + + +const string & TypeHandlerChar::name() const +{ + static const string xname= "CHAR"; + return xname; +} + + +const string & TypeHandlerVarchar::name() const +{ + static const string xname= "VARCHAR"; + return xname; +} + + +const string & TypeHandlerVarbinary::name() const +{ + static const string xname= "VARBINARY"; + return xname; +} + + +const string & TypeHandlerBlob::name() const +{ + static const string xname= "BLOB"; + return xname; +} + + +const string & TypeHandlerClob::name() const +{ + static const string xname= "CLOB"; + return xname; +} + + +const string & TypeHandlerText::name() const +{ + static const string xname= "TEXT"; + return xname; +} + + +const string & TypeHandlerBit::name() const +{ + static const string xname= "BIT"; + return xname; +} + + +TypeHandlerBit mcs_type_handler_bit; + +TypeHandlerSInt8 mcs_type_handler_sint8; +TypeHandlerSInt16 mcs_type_handler_sint16; +TypeHandlerSInt24 mcs_type_handler_sint24; +TypeHandlerSInt32 mcs_type_handler_sint32; +TypeHandlerSInt64 mcs_type_handler_sint64; + +TypeHandlerUInt8 mcs_type_handler_uint8; +TypeHandlerUInt16 mcs_type_handler_uint16; +TypeHandlerUInt24 mcs_type_handler_uint24; +TypeHandlerUInt32 mcs_type_handler_uint32; +TypeHandlerUInt64 mcs_type_handler_uint64; + +TypeHandlerSFloat mcs_type_handler_sfloat; +TypeHandlerSDouble mcs_type_handler_sdouble; +TypeHandlerSLongDouble mcs_type_handler_slongdouble; + +TypeHandlerUFloat mcs_type_handler_ufloat; +TypeHandlerUDouble mcs_type_handler_udouble; + +TypeHandlerSDecimal64 mcs_type_handler_sdecimal64; +TypeHandlerUDecimal64 mcs_type_handler_udecimal64; + +TypeHandlerSDecimal128 mcs_type_handler_sdecimal128; +TypeHandlerUDecimal128 mcs_type_handler_udecimal128; + +TypeHandlerDate mcs_type_handler_date; +TypeHandlerTime mcs_type_handler_time; +TypeHandlerDatetime mcs_type_handler_datetime; +TypeHandlerTimestamp mcs_type_handler_timestamp; + + +TypeHandlerChar mcs_type_handler_char; +TypeHandlerVarchar mcs_type_handler_varchar; +TypeHandlerText mcs_type_handler_text; +TypeHandlerClob mcs_type_handler_clob; +TypeHandlerVarbinary mcs_type_handler_varbinary; +TypeHandlerBlob mcs_type_handler_blob; + + +const TypeHandler * +TypeHandler::find(SystemCatalog::ColDataType typeCode, + const SystemCatalog::TypeAttributesStd &ct) +{ + switch (typeCode) { + case SystemCatalog::BIT: return &mcs_type_handler_bit; + case SystemCatalog::TINYINT: return &mcs_type_handler_sint8; + case SystemCatalog::SMALLINT: return &mcs_type_handler_sint16; + case SystemCatalog::MEDINT: return &mcs_type_handler_sint24; + case SystemCatalog::INT: return &mcs_type_handler_sint32; + case SystemCatalog::BIGINT: return &mcs_type_handler_sint64; + case SystemCatalog::UTINYINT: return &mcs_type_handler_uint8; + case SystemCatalog::USMALLINT: return &mcs_type_handler_uint16; + case SystemCatalog::UMEDINT: return &mcs_type_handler_uint24; + case SystemCatalog::UINT: return &mcs_type_handler_uint32; + case SystemCatalog::UBIGINT: return &mcs_type_handler_uint64; + case SystemCatalog::FLOAT: return &mcs_type_handler_sfloat; + case SystemCatalog::DOUBLE: return &mcs_type_handler_sdouble; + case SystemCatalog::LONGDOUBLE: return &mcs_type_handler_slongdouble; + case SystemCatalog::UFLOAT: return &mcs_type_handler_ufloat; + case SystemCatalog::UDOUBLE: return &mcs_type_handler_udouble; + + case SystemCatalog::DECIMAL: + if (ct.colWidth < datatypes::MAXDECIMALWIDTH) + return &mcs_type_handler_sdecimal64; + else + return &mcs_type_handler_sdecimal128; + + case SystemCatalog::UDECIMAL: + if (ct.colWidth < datatypes::MAXDECIMALWIDTH) + return &mcs_type_handler_udecimal64; + else + return &mcs_type_handler_udecimal128; + + case SystemCatalog::TIME: return &mcs_type_handler_time; + case SystemCatalog::DATE: return &mcs_type_handler_date; + case SystemCatalog::DATETIME: return &mcs_type_handler_datetime; + case SystemCatalog::TIMESTAMP: return &mcs_type_handler_timestamp; + case SystemCatalog::CHAR: return &mcs_type_handler_char; + case SystemCatalog::VARCHAR: return &mcs_type_handler_varchar; + case SystemCatalog::TEXT: return &mcs_type_handler_text; + case SystemCatalog::CLOB: return &mcs_type_handler_clob; + case SystemCatalog::VARBINARY: return &mcs_type_handler_varbinary; + case SystemCatalog::BLOB: return &mcs_type_handler_blob; + + case SystemCatalog::NUM_OF_COL_DATA_TYPE: + case SystemCatalog::STRINT: + case SystemCatalog::UNDEFINED: + case SystemCatalog::BINARY: + break; + } + return NULL; +} + + +const TypeHandler * +SystemCatalog::TypeHolderStd::typeHandler() const +{ + return TypeHandler::find(colDataType, *this); +} + + +boost::any +SystemCatalog::TypeHolderStd::getNullValueForType() const +{ + const TypeHandler *h= typeHandler(); + if (!h) + { + throw std::runtime_error("getNullValueForType: unkown column data type"); + return boost::any(); + } + return h->getNullValueForType(*this); +} + + +const TypeHandler * +TypeHandler::find_by_ddltype(const ddlpackage::ColumnType &ct) +{ + switch (ct.fType) { + case ddlpackage::DDL_CHAR: return &mcs_type_handler_char; + case ddlpackage::DDL_VARCHAR: return &mcs_type_handler_varchar; + case ddlpackage::DDL_VARBINARY: return &mcs_type_handler_varbinary; + case ddlpackage::DDL_BIT: return &mcs_type_handler_bit; + + case ddlpackage::DDL_REAL: + case ddlpackage::DDL_DECIMAL: + case ddlpackage::DDL_NUMERIC: + case ddlpackage::DDL_NUMBER: + + if (ct.fLength < datatypes::MAXDECIMALWIDTH) + return &mcs_type_handler_sdecimal64; + return &mcs_type_handler_sdecimal128; + + case ddlpackage::DDL_FLOAT: return &mcs_type_handler_sfloat; + case ddlpackage::DDL_DOUBLE: return &mcs_type_handler_sdouble; + + case ddlpackage::DDL_INT: + case ddlpackage::DDL_INTEGER: return &mcs_type_handler_sint32; + + case ddlpackage::DDL_BIGINT: return &mcs_type_handler_sint64; + case ddlpackage::DDL_MEDINT: return &mcs_type_handler_sint24; + case ddlpackage::DDL_SMALLINT: return &mcs_type_handler_sint16; + case ddlpackage::DDL_TINYINT: return &mcs_type_handler_sint8; + + case ddlpackage::DDL_DATE: return &mcs_type_handler_date; + case ddlpackage::DDL_DATETIME: return &mcs_type_handler_datetime; + case ddlpackage::DDL_TIME: return &mcs_type_handler_time; + case ddlpackage::DDL_TIMESTAMP: return &mcs_type_handler_timestamp; + + case ddlpackage::DDL_CLOB: return &mcs_type_handler_clob; + case ddlpackage::DDL_BLOB: return &mcs_type_handler_blob; + case ddlpackage::DDL_TEXT: return &mcs_type_handler_text; + + case ddlpackage::DDL_UNSIGNED_TINYINT: return &mcs_type_handler_uint8; + case ddlpackage::DDL_UNSIGNED_SMALLINT: return &mcs_type_handler_uint16; + case ddlpackage::DDL_UNSIGNED_MEDINT: return &mcs_type_handler_uint24; + case ddlpackage::DDL_UNSIGNED_INT: return &mcs_type_handler_uint32; + case ddlpackage::DDL_UNSIGNED_BIGINT: return &mcs_type_handler_uint64; + + case ddlpackage::DDL_UNSIGNED_DECIMAL: + case ddlpackage::DDL_UNSIGNED_NUMERIC: + + if (ct.fLength < datatypes::MAXDECIMALWIDTH) + return &mcs_type_handler_udecimal64; + return &mcs_type_handler_udecimal128; + + case ddlpackage::DDL_UNSIGNED_FLOAT: return &mcs_type_handler_ufloat; + case ddlpackage::DDL_UNSIGNED_DOUBLE: return &mcs_type_handler_udouble; + + case ddlpackage::DDL_BINARY: //return &mcs_type_handler_binary; + case ddlpackage::DDL_INVALID_DATATYPE: + break; + } + return NULL; +} + + +/****************************************************************************/ + + +int TypeHandlerDate::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + int64_t intColVal = row.getUintField<4>(pos); + return f->store_date(intColVal); +} + + +int TypeHandlerDatetime::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + int64_t intColVal = row.getUintField<8>(pos); + return f->store_datetime(intColVal); +} + + +int TypeHandlerTime::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + int64_t intColVal = row.getUintField<8>(pos); + return f->store_time(intColVal); +} + + +int TypeHandlerTimestamp::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + int64_t intColVal = row.getUintField<8>(pos); + return f->store_timestamp(intColVal); +} + + +int TypeHandlerStr::storeValueToFieldCharVarchar(rowgroup::Row &row, int pos, + StoreField *f) const +{ + int64_t intColVal; + switch (f->colWidth()) + { + case 1: + intColVal = row.getUintField<1>(pos); + return f->store_string((const char*)(&intColVal), strlen((char*)(&intColVal))); + + case 2: + intColVal = row.getUintField<2>(pos); + return f->store_string((char*)(&intColVal), strlen((char*)(&intColVal))); + + case 4: + intColVal = row.getUintField<4>(pos); + return f->store_string((char*)(&intColVal), strlen((char*)(&intColVal))); + + case 8: + { + //make sure we don't send strlen off into the weeds... + intColVal = row.getUintField<8>(pos); + char tmp[256]; + memcpy(tmp, &intColVal, 8); + tmp[8] = 0; + return f->store_string(tmp, strlen(tmp)); + } + default: + return f->store_string((const char*)row.getStringPointer(pos), row.getStringLength(pos)); + } +} + + +int TypeHandlerVarbinary::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + uint32_t l; + const uint8_t* p = row.getVarBinaryField(l, pos); + return f->store_varbinary((const char *) p, l); +} + + +int TypeHandlerSInt64::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + int64_t val = row.getIntField<8>(pos); + return f->store_xlonglong(val); +} + + +int TypeHandlerUInt64::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + uint64_t val = row.getUintField<8>(pos); + return f->store_xlonglong(static_cast(val)); +} + + +int TypeHandlerInt::storeValueToFieldSInt32(rowgroup::Row &row, int pos, + StoreField *f) const +{ + int64_t val = row.getIntField<4>(pos); + return f->store_xlonglong(val); +} + + +int TypeHandlerInt::storeValueToFieldUInt32(rowgroup::Row &row, int pos, + StoreField *f) const +{ + uint64_t val = row.getUintField<4>(pos); + return f->store_xlonglong(static_cast(val)); +} + + +int TypeHandlerSInt16::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + int64_t val = row.getIntField<2>(pos); + return f->store_xlonglong(val); +} + + +int TypeHandlerUInt16::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + uint64_t val = row.getUintField<2>(pos); + return f->store_xlonglong(static_cast(val)); +} + + +int TypeHandlerSInt8::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + int64_t val = row.getIntField<1>(pos); + return f->store_xlonglong(val); +} + + +int TypeHandlerUInt8::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + uint64_t val = row.getUintField<1>(pos); + return f->store_xlonglong(static_cast(val)); +} + + +/* + In this case, we're trying to load a double output column with float data. This is the + case when you do sum(floatcol), e.g. +*/ +int TypeHandlerReal::storeValueToFieldXFloat(rowgroup::Row &row, int pos, + StoreField *f) const +{ + float dl = row.getFloatField(pos); + return f->store_float(dl); +} + + +int TypeHandlerReal::storeValueToFieldXDouble(rowgroup::Row &row, int pos, + StoreField *f) const +{ + double dl = row.getDoubleField(pos); + return f->store_double(dl); +} + + +int TypeHandlerSLongDouble::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + long double dl = row.getLongDoubleField(pos); + return f->store_long_double(dl); +} + + +int TypeHandlerXDecimal::storeValueToField64(rowgroup::Row &row, int pos, + StoreField *f) const +{ + int64_t val = row.getIntField(pos); + return f->store_decimal64(val); +} + + +int TypeHandlerXDecimal::storeValueToField128(rowgroup::Row &row, int pos, + StoreField *f) const +{ + int128_t* dec= row.getBinaryField(pos); + return f->store_decimal128(*dec); +} + + +int TypeHandlerStr::storeValueToFieldBlobText(rowgroup::Row &row, int pos, + StoreField *f) const +{ + return f->store_lob((const char*) row.getVarBinaryField(pos), + row.getVarBinaryLength(pos)); +} + + +/* +int TypeHandlerBinary::storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const +{ + Field_varstring* f2 = static_cast(f); + // WIP MCOL-641 Binary representation could contain \0. + char* binaryString = row.getBinaryField(pos); + return f2->store(binaryString, colType.colWidth, f2->charset()); +} +*/ + +/* + Default behaviour: treat as int64 + int64_t intColVal = row.getUintField<8>(pos); + storeNumericField(f, intColVal, colType); +*/ + + +/****************************************************************************/ + +string TypeHandlerDate::format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + return DataConvert::dateToString(v.toSInt64()); +} + + +string TypeHandlerDatetime::format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + return DataConvert::datetimeToString(v.toSInt64()); +} + +string TypeHandlerTimestamp::format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + return DataConvert::timestampToString(v.toSInt64(), v.tzname()); +} + + +string TypeHandlerTime::format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + return DataConvert::timeToString(v.toSInt64()); +} + + +string TypeHandlerChar::format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + // swap again to retain the string byte order + ostringstream oss; + uint64_t tmp = uint64ToStr(v.toSInt64()); + oss << (char*)(&tmp); + return oss.str(); +} + + +string TypeHandlerVarchar::format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + // swap again to retain the string byte order + ostringstream oss; + uint64_t tmp = uint64ToStr(v.toSInt64()); + oss << (char*)(&tmp); + return oss.str(); +} + + +string TypeHandlerInt::formatSInt64(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + ostringstream oss; + oss << v.toSInt64(); + return oss.str(); +} + + +string TypeHandlerInt::formatUInt64(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + ostringstream oss; + oss << static_cast(v.toSInt64()); + return oss.str(); +} + + +string TypeHandlerVarbinary::format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + return "N/A"; +} + + +string +TypeHandlerXDecimal::format64(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + idbassert(isValidXDecimal64(attr)); + ostringstream oss; + if (attr.scale > 0) + { + double d = ((double)(v.toSInt64()) / (double)pow((double)10, attr.scale)); + oss << setprecision(attr.scale) << fixed << d; + } + else + oss << v.toSInt64(); + return oss.str(); +} + + +string +TypeHandlerXDecimal::format128(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) + const +{ + idbassert(isValidXDecimal128(attr)); + ostringstream oss; + char buf[datatypes::Decimal::MAXLENGTH16BYTES]; + int128_t tmp= v.toSInt128(); + DataConvert::decimalToString(&tmp, (unsigned) attr.scale, buf, (uint8_t) sizeof(buf), code()); + oss << buf; + return oss.str(); +} + + +/****************************************************************************/ + +string TypeHandler::formatPartitionInfoSInt64( + const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &pi) const +{ + ostringstream output; + if (pi.isEmptyOrNullSInt64()) + output << setw(30) << "Empty/Null" + << setw(30) << "Empty/Null"; + else + output << setw(30) << format(SimpleValueSInt64(pi.min), attr) + << setw(30) << format(SimpleValueSInt64(pi.max), attr); + return output.str(); +} + + +string TypeHandler::formatPartitionInfoUInt64( + const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &pi) const +{ + ostringstream output; + if (pi.isEmptyOrNullUInt64()) + output << setw(30) << "Empty/Null" + << setw(30) << "Empty/Null"; + else + output << setw(30) << format(SimpleValueSInt64(pi.min), attr) + << setw(30) << format(SimpleValueSInt64(pi.max), attr); + return output.str(); +} + + + +string +TypeHandlerXDecimal::formatPartitionInfo128( + const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &pi) const +{ + ostringstream output; + if (pi.isEmptyOrNullSInt128()) + output << setw(datatypes::Decimal::MAXLENGTH16BYTES) << "Empty/Null" + << setw(datatypes::Decimal::MAXLENGTH16BYTES) << "Empty/Null"; + else + output << setw(datatypes::Decimal::MAXLENGTH16BYTES) << format(SimpleValueSInt128(pi.int128Min), attr) + << setw(datatypes::Decimal::MAXLENGTH16BYTES) << format(SimpleValueSInt128(pi.int128Max), attr); + return output.str(); +} + + +string +TypeHandlerStr::formatPartitionInfoSmallCharVarchar( + const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &pi) const +{ + ostringstream output; + int64_t maxLimit = numeric_limits::max(); + int64_t minLimit = numeric_limits::min(); + maxLimit = uint64ToStr(maxLimit); + minLimit = uint64ToStr(minLimit); + if (pi.min == maxLimit && pi.max == minLimit) + output << setw(30) << "Empty/Null" + << setw(30) << "Empty/Null"; + else + output << setw(30) << format(SimpleValueSInt64(pi.min), attr) + << setw(30) << format(SimpleValueSInt64(pi.max), attr); + return output.str(); +} + + +string +TypeHandlerChar::formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &pi) const +{ + // char column order swap for compare in subsequent loop + if (attr.colWidth <= 8) + return formatPartitionInfoSmallCharVarchar(attr, pi); + return formatPartitionInfoSInt64(attr, pi); +} + + +string +TypeHandlerVarchar::formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &pi) const +{ + // varchar column order swap for compare in subsequent loop + if (attr.colWidth <= 7) + return formatPartitionInfoSmallCharVarchar(attr, pi); + return formatPartitionInfoSInt64(attr, pi); +} + + +/****************************************************************************/ + +execplan::SimpleColumn * +TypeHandlerSInt8::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + if (ct.scale == 0) + return new execplan::SimpleColumn_INT<1>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + ct.colDataType = SystemCatalog::DECIMAL; + return new execplan::SimpleColumn_Decimal<1>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerSInt16::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + if (ct.scale == 0) + return new execplan::SimpleColumn_INT<2>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + ct.colDataType = SystemCatalog::DECIMAL; + return new execplan::SimpleColumn_Decimal<2>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerSInt24::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + if (ct.scale == 0) + return new execplan::SimpleColumn_INT<4>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + ct.colDataType = SystemCatalog::DECIMAL; + return new execplan::SimpleColumn_Decimal<4>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerSInt32::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + if (ct.scale == 0) + return new execplan::SimpleColumn_INT<4>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + ct.colDataType = SystemCatalog::DECIMAL; + return new execplan::SimpleColumn_Decimal<4>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerSInt64::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + if (ct.scale == 0) + return new execplan::SimpleColumn_INT<8>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); + ct.colDataType = SystemCatalog::DECIMAL; + return new execplan::SimpleColumn_Decimal<8>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerUInt8::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + // QQ: why scale is not checked (unlike SInt1)? + return new execplan::SimpleColumn_UINT<1>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerUInt16::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + return new execplan::SimpleColumn_UINT<2>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerUInt24::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + return new execplan::SimpleColumn_UINT<4>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerUInt32::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + return new execplan::SimpleColumn_UINT<4>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerUInt64::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + return new execplan::SimpleColumn_UINT<8>(name.db(), name.table(), name.column(), prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerReal::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + // QQ + return new execplan::SimpleColumn(name.db(), name.table(), name.column(), + prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerXDecimal::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + // QQ + return new execplan::SimpleColumn(name.db(), name.table(), name.column(), + prm.columnStore(), prm.sessionid()); +} + + +execplan::SimpleColumn * +TypeHandlerStr::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + // QQ + return new execplan::SimpleColumn(name.db(), name.table(), name.column(), + prm.columnStore(), prm.sessionid()); +} + + + +execplan::SimpleColumn * +TypeHandlerTemporal::newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) const +{ + // QQ + return new execplan::SimpleColumn(name.db(), name.table(), name.column(), + prm.columnStore(), prm.sessionid()); +} + + +/****************************************************************************/ + +class SimpleConverter: public boost::any +{ + bool &initPushWarning() + { + m_pushWarning= false; + return m_pushWarning; + } + bool m_pushWarning; +public: + SimpleConverter(const SessionParam &sp, + SystemCatalog::ColDataType typeCode, + const SystemCatalog::TypeAttributesStd &attr, + const char *str) + :boost::any(DataConvert::convertColumnData(typeCode, attr, + str, initPushWarning(), + sp.tzname(), + false, true, false)) + { } + round_style_t roundStyle() const + { + return m_pushWarning ? round_style_t::POS : round_style_t::NONE; + } + round_style_t roundStyle(const char *str) const + { + return m_pushWarning ? roundStyleDetect(str) : round_style_t::NONE; + } + static round_style_t roundStyleDetect(const char *str) + { + // get rid of leading white spaces and parentheses + string data(str); + size_t fpos = data.find_first_of(" \t()"); + while (string::npos != fpos) + { + data.erase(fpos, 1); + fpos = data.find_first_of(" \t()"); + } + return (data[0] == '-') ? round_style_t::NEG : round_style_t::POS; + } + int64_t to_sint64() const { return boost::any_cast(*this); } + uint64_t to_uint64() const { return boost::any_cast(*this); } + uint32_t to_uint32() const { return boost::any_cast(*this); } + int128_t to_sint128() const { return boost::any_cast(*this); } +}; + + +class SimpleConverterSNumeric: public SimpleConverter +{ +public: + SimpleConverterSNumeric(const SessionParam &sp, + SystemCatalog::ColDataType typeCode, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, + round_style_t &rf) + :SimpleConverter(sp, typeCode, attr, str) + { + rf = roundStyle(str); + } +}; + + +template +SimpleValue +toSimpleValueSInt(const SessionParam &sp, + SystemCatalog::ColDataType typeCode, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) +{ + idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); + SimpleConverterSNumeric anyVal(sp, typeCode, attr, str, rf); + return SimpleValueSInt64(static_cast(boost::any_cast(anyVal))); +} + + +SimpleValue +TypeHandlerSInt8::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + return toSimpleValueSInt(sp, SystemCatalog::TINYINT, attr, str, rf); +} + + +SimpleValue +TypeHandlerSInt16::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + return toSimpleValueSInt(sp, SystemCatalog::SMALLINT, attr, str, rf); +} + + +SimpleValue +TypeHandlerSInt24::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + return toSimpleValueSInt(sp, SystemCatalog::MEDINT, attr, str, rf); +} + + +SimpleValue +TypeHandlerSInt32::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + return toSimpleValueSInt(sp, SystemCatalog::INT, attr, str, rf); +} + + +SimpleValue +TypeHandlerSInt64::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + return toSimpleValueSInt(sp, SystemCatalog::BIGINT, attr, str, rf); +} + + +template +SimpleValue toSimpleValueUInt(const SessionParam &sp, + SystemCatalog::ColDataType typeCode, + const SystemCatalog::TypeAttributesStd &attr, + const char *str) +{ + idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); + SimpleConverter anyVal(sp, typeCode, attr, str); + return SimpleValueSInt64(static_cast(boost::any_cast(anyVal))); +} + + +SimpleValue +TypeHandlerUInt8::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + return toSimpleValueUInt(sp, SystemCatalog::UTINYINT, attr, str); +} + + +SimpleValue +TypeHandlerUInt16::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + return toSimpleValueUInt(sp, SystemCatalog::USMALLINT, attr, str); +} + + +SimpleValue +TypeHandlerUInt24::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + return toSimpleValueUInt(sp, SystemCatalog::UMEDINT, attr, str); +} + + +SimpleValue +TypeHandlerUInt32::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + return toSimpleValueUInt(sp, SystemCatalog::UINT, attr, str); +} + + +SimpleValue +TypeHandlerUInt64::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + return toSimpleValueUInt(sp, SystemCatalog::UBIGINT, attr, str); +} + + +SimpleValue +TypeHandlerDate::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); + SimpleConverter anyVal(sp, SystemCatalog::DATE, attr, str); + return SimpleValueSInt64(static_cast(anyVal.to_uint32())); +} + + +SimpleValue +TypeHandlerDatetime::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); + SimpleConverter anyVal(sp, SystemCatalog::DATETIME, attr, str); + return SimpleValueSInt64(static_cast(anyVal.to_uint64())); +} + + +SimpleValue +TypeHandlerTimestamp::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); + SimpleConverter anyVal(sp, SystemCatalog::TIMESTAMP, attr, str); + return SimpleValueTimestamp(anyVal.to_uint64(), sp.tzname()); +} + + +SimpleValue +TypeHandlerTime::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); + SimpleConverter anyVal(sp, SystemCatalog::TIME, attr, str); + return SimpleValueSInt64(anyVal.to_sint64()); +} + + +SimpleValue +TypeHandlerXDecimal::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + if (attr.colWidth <= SystemCatalog::EIGHT_BYTE) + { + SimpleConverterSNumeric anyVal(sp, code(), attr, str, rf); + int64_t v; + if (attr.colWidth == SystemCatalog::ONE_BYTE) + v = boost::any_cast(anyVal); + else if (attr.colWidth == SystemCatalog::TWO_BYTE) + v = boost::any_cast(anyVal); + else if (attr.colWidth == SystemCatalog::FOUR_BYTE) + v = boost::any_cast(anyVal); + else if (attr.colWidth == SystemCatalog::EIGHT_BYTE) + v = anyVal.to_sint64(); + else + { + idbassert(0); + v = 0; + } + return SimpleValueSInt64(v); + } + else + { + idbassert(attr.colWidth == datatypes::MAXDECIMALWIDTH); + SimpleConverterSNumeric anyVal(sp, code(), attr, str, rf); + return SimpleValueSInt128(anyVal.to_sint128()); + } +} + + +SimpleValue +TypeHandlerStr::toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const +{ + SimpleConverter anyVal(sp, code(), attr, str); + rf= anyVal.roundStyle(); + string i = boost::any_cast(anyVal); + // bug 1932, pad nulls up to the size of v + i.resize(sizeof(int64_t), 0); + return SimpleValueSInt64(static_cast(uint64ToStr(*((uint64_t*) i.data())))); +} + + +/****************************************************************************/ + +MinMaxPartitionInfo::MinMaxPartitionInfo(const BRM::EMEntry &entry) + :m_status(entry.status == BRM::EXTENTOUTOFSERVICE ? ET_DISABLED : 0) +{ } + + +MinMaxPartitionInfo +TypeHandler::getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const +{ + int32_t seqNum; + MinMaxPartitionInfo partInfo(entry); + *state = em.getExtentMaxMin(entry.range.start, partInfo.max, partInfo.min, seqNum); + return partInfo; +} + + +MinMaxPartitionInfo +TypeHandlerXDecimal::getExtentPartitionInfo64(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const +{ + int32_t seqNum; + MinMaxPartitionInfo partInfo(entry); + *state = em.getExtentMaxMin(entry.range.start, partInfo.max, partInfo.min, seqNum); + return partInfo; +} + + +MinMaxPartitionInfo +TypeHandlerXDecimal::getExtentPartitionInfo128(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const +{ + int32_t seqNum; + MinMaxPartitionInfo partInfo(entry); + *state = em.getExtentMaxMin(entry.range.start, partInfo.int128Max, partInfo.int128Min, seqNum); + return partInfo; +} + + +MinMaxPartitionInfo +TypeHandlerChar::getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const +{ + int32_t seqNum; + MinMaxPartitionInfo partInfo(entry); + *state = em.getExtentMaxMin(entry.range.start, partInfo.max, partInfo.min, seqNum); + // char column order swap + if (attr.colWidth <= 8) + { + partInfo.max = uint64ToStr(partInfo.max); + partInfo.min = uint64ToStr(partInfo.min); + } + return partInfo; +} + + +MinMaxPartitionInfo +TypeHandlerVarchar::getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const +{ + int32_t seqNum; + MinMaxPartitionInfo partInfo(entry); + *state = em.getExtentMaxMin(entry.range.start, partInfo.max, partInfo.min, seqNum); + // char column order swap + if (attr.colWidth <= 7) + { + partInfo.max = uint64ToStr(partInfo.max); + partInfo.min = uint64ToStr(partInfo.min); + } + return partInfo; +} + + +/****************************************************************************/ + + +string TypeHandler::PrintPartitionValueSInt64( + const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const +{ + if (!partInfo.isSuitableSInt64(startVal, rfMin, endVal, rfMax)) + return ""; + + ostringstream oss; + if (partInfo.min > partInfo.max) + oss << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; + else + oss << setw(30) << format(SimpleValueSInt64(partInfo.min), attr) + << setw(30) << format(SimpleValueSInt64(partInfo.max), attr); + return oss.str(); +} + + +string TypeHandler::PrintPartitionValueUInt64( + const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const +{ + if (!partInfo.isSuitableUInt64(startVal, rfMin, endVal, rfMax)) + return ""; + + ostringstream oss; + if (static_cast(partInfo.min) > static_cast(partInfo.max)) + oss << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; + else + oss << setw(30) << format(SimpleValueSInt64(partInfo.min), attr) + << setw(30) << format(SimpleValueSInt64(partInfo.max), attr); + return oss.str(); +} + + +string TypeHandlerXDecimal::PrintPartitionValue128( + const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const +{ + if (!partInfo.isSuitableSInt128(startVal, rfMin, endVal, rfMax)) + return ""; + + ostringstream oss; + if (partInfo.int128Min > partInfo.int128Max) + oss << setw(datatypes::Decimal::MAXLENGTH16BYTES) << "Empty/Null" + << setw(datatypes::Decimal::MAXLENGTH16BYTES) << "Empty/Null"; + else + oss << setw(datatypes::Decimal::MAXLENGTH16BYTES) << format(SimpleValueSInt128(partInfo.int128Min), attr) + << setw(datatypes::Decimal::MAXLENGTH16BYTES) << format(SimpleValueSInt128(partInfo.int128Max), attr); + return oss.str(); +} + +/****************************************************************************/ + + +boost::any +TypeHandlerSInt8::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + char tinyintvalue = joblist::TINYINTNULL; + boost::any value = tinyintvalue; + return value; +} + + +boost::any +TypeHandlerUInt8::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint8_t utinyintvalue = joblist::UTINYINTNULL; + boost::any value = utinyintvalue; + return value; +} + + +boost::any +TypeHandlerSInt16::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + short smallintvalue = joblist::SMALLINTNULL; + boost::any value = smallintvalue; + return value; +} + + +boost::any +TypeHandlerUInt16::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint16_t usmallintvalue = joblist::USMALLINTNULL; + boost::any value = usmallintvalue; + return value; +} + + +boost::any +TypeHandlerSInt24::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + int intvalue = joblist::INTNULL; + boost::any value = intvalue; + return value; +} + + +boost::any +TypeHandlerSInt32::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + int intvalue = joblist::INTNULL; + boost::any value = intvalue; + return value; +} + + +boost::any +TypeHandlerUInt24::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint32_t uintvalue = joblist::UINTNULL; + boost::any value = uintvalue; + return value; +} + + +boost::any +TypeHandlerUInt32::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint32_t uintvalue = joblist::UINTNULL; + boost::any value = uintvalue; + return value; +} + + +boost::any +TypeHandlerSInt64::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + long long bigint = joblist::BIGINTNULL; + boost::any value = bigint; + return value; +} + + +boost::any +TypeHandlerUInt64::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint64_t ubigint = joblist::UBIGINTNULL; + boost::any value = ubigint; + return value; +} + + +boost::any +TypeHandlerXDecimal::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + if (LIKELY(attr.colWidth == 16)) + { + int128_t val; + datatypes::Decimal::setWideDecimalNullValue(val); + boost::any value = val; + return value; + } + if (attr.colWidth == SystemCatalog::EIGHT_BYTE) + { + long long eightbyte = joblist::BIGINTNULL; + boost::any value = eightbyte; + return value; + } + if (attr.colWidth == SystemCatalog::FOUR_BYTE) + { + int intvalue = joblist::INTNULL; + boost::any value = intvalue; + return value; + } + if (attr.colWidth == SystemCatalog::TWO_BYTE) + { + short smallintvalue = joblist::SMALLINTNULL; + boost::any value = smallintvalue; + return value; + } + if (attr.colWidth == SystemCatalog::ONE_BYTE) + { + char tinyintvalue = joblist::TINYINTNULL; + boost::any value = tinyintvalue; + return value; + } + WriteEngine::Token nullToken; + boost::any value = nullToken; + return value; +} + + +boost::any +TypeHandlerSFloat::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint32_t jlfloatnull = joblist::FLOATNULL; + float* fp = reinterpret_cast(&jlfloatnull); + boost::any value = *fp; + return value; +} + + +boost::any +TypeHandlerUFloat::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint32_t jlfloatnull = joblist::FLOATNULL; + float* fp = reinterpret_cast(&jlfloatnull); + boost::any value = *fp; + return value; +} + + +boost::any +TypeHandlerSDouble::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint64_t jldoublenull = joblist::DOUBLENULL; + double* dp = reinterpret_cast(&jldoublenull); + boost::any value = *dp; + return value; +} + + +boost::any +TypeHandlerUDouble::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint64_t jldoublenull = joblist::DOUBLENULL; + double* dp = reinterpret_cast(&jldoublenull); + boost::any value = *dp; + return value; +} + + +boost::any +TypeHandlerDate::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint32_t d = joblist::DATENULL; + boost::any value = d; + return value; +} + + +boost::any +TypeHandlerDatetime::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint64_t d = joblist::DATETIMENULL; + boost::any value = d; + return value; +} + + +boost::any +TypeHandlerTime::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + int64_t d = joblist::TIMENULL; + boost::any value = d; + return value; +} + + +boost::any +TypeHandlerTimestamp::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + uint64_t d = joblist::TIMESTAMPNULL; + boost::any value = d; + return value; +} + + +boost::any +TypeHandlerChar::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + switch (attr.colWidth) + { + case 1: + { + //charnull = joblist::CHAR1NULL; + std::string charnull = "\376"; + boost::any value = charnull; + return value; + } + case 2: + { + //charnull = joblist::CHAR2NULL; + std::string charnull = "\377\376"; + boost::any value = charnull; + return value; + } + case 3: + case 4: + { + //charnull = joblist::CHAR4NULL; + std::string charnull = "\377\377\377\376"; + boost::any value = charnull; + return value; + } + case 5: + case 6: + case 7: + case 8: + { + //charnull = joblist::CHAR8NULL; + std::string charnull = "\377\377\377\377\377\377\377\376"; + boost::any value = charnull; + return value; + } + } + WriteEngine::Token nullToken; + boost::any value = nullToken; + return value; +} + + +boost::any +TypeHandlerStr::getNullValueForTypeVarcharText(const SystemCatalog::TypeAttributesStd &attr) const +{ + switch (attr.colWidth) + { + case 1: + { + //charnull = joblist::CHAR2NULL; + std::string charnull = "\377\376"; + boost::any value = charnull; + return value; + } + case 2: + case 3: + { + //charnull = joblist::CHAR4NULL; + std::string charnull = "\377\377\377\376"; + boost::any value = charnull; + return value; + } + case 4: + case 5: + case 6: + case 7: + { + //charnull = joblist::CHAR8NULL; + std::string charnull = "\377\377\377\377\377\377\377\376"; + boost::any value = charnull; + return value; + } + } + WriteEngine::Token nullToken; + boost::any value = nullToken; + return value; +} + + +boost::any +TypeHandlerBlob::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + WriteEngine::Token nullToken; + boost::any value = nullToken; + return value; +} + + +boost::any +TypeHandlerVarbinary::getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const +{ + WriteEngine::Token nullToken; + boost::any value = nullToken; + return value; +} + +/****************************************************************************/ + +} // end of namespace datatypes + + +// vim:ts=2 sw=2: diff --git a/datatypes/mcs_datatype.h b/datatypes/mcs_datatype.h new file mode 100644 index 000000000..1e161b7ae --- /dev/null +++ b/datatypes/mcs_datatype.h @@ -0,0 +1,2510 @@ +/* + Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ +#ifndef MCS_DATATYPE_H_INCLUDED +#define MCS_DATATYPE_H_INCLUDED + +#include +#include +#include +#include "exceptclasses.h" +#include "mcs_decimal.h" + + +#ifdef _MSC_VER +typedef int mcs_sint32_t; +#else +typedef int32_t mcs_sint32_t; +#endif + + +using int128_t = __int128; +using uint128_t = unsigned __int128; + + +// Because including my_sys.h in a Columnstore header causes too many conflicts +struct charset_info_st; +typedef const struct charset_info_st CHARSET_INFO; + + +#ifdef _MSC_VER +#define __attribute__(x) +#endif + +namespace +{ +const int64_t MIN_TINYINT __attribute__ ((unused)) = std::numeric_limits::min() + 2; // -126; +const int64_t MAX_TINYINT __attribute__ ((unused)) = std::numeric_limits::max(); // 127; +const int64_t MIN_SMALLINT __attribute__ ((unused)) = std::numeric_limits::min() + 2; // -32766; +const int64_t MAX_SMALLINT __attribute__ ((unused)) = std::numeric_limits::max(); // 32767; +const int64_t MIN_MEDINT __attribute__ ((unused)) = -(1ULL << 23); // -8388608; +const int64_t MAX_MEDINT __attribute__ ((unused)) = (1ULL << 23) - 1; // 8388607; +const int64_t MIN_INT __attribute__ ((unused)) = std::numeric_limits::min() + 2; // -2147483646; +const int64_t MAX_INT __attribute__ ((unused)) = std::numeric_limits::max(); // 2147483647; +const int64_t MIN_BIGINT __attribute__ ((unused)) = std::numeric_limits::min() + 2; // -9223372036854775806LL; +const int64_t MAX_BIGINT __attribute__ ((unused)) = std::numeric_limits::max(); // 9223372036854775807 + +const uint64_t MIN_UINT __attribute__ ((unused)) = 0; +const uint64_t MIN_UTINYINT __attribute__ ((unused)) = 0; +const uint64_t MIN_USMALLINT __attribute__ ((unused)) = 0; +const uint64_t MIN_UMEDINT __attribute__ ((unused)) = 0; +const uint64_t MIN_UBIGINT __attribute__ ((unused)) = 0; +const uint64_t MAX_UINT __attribute__ ((unused)) = std::numeric_limits::max() - 2; // 4294967293 +const uint64_t MAX_UTINYINT __attribute__ ((unused)) = std::numeric_limits::max() - 2; // 253; +const uint64_t MAX_USMALLINT __attribute__ ((unused)) = std::numeric_limits::max() - 2; // 65533; +const uint64_t MAX_UMEDINT __attribute__ ((unused)) = (1ULL << 24) - 1; // 16777215 +const uint64_t MAX_UBIGINT __attribute__ ((unused)) = std::numeric_limits::max() - 2; // 18446744073709551613 + +const float MAX_FLOAT __attribute__ ((unused)) = std::numeric_limits::max(); // 3.402823466385289e+38 +const float MIN_FLOAT __attribute__ ((unused)) = -std::numeric_limits::max(); +const double MAX_DOUBLE __attribute__ ((unused)) = std::numeric_limits::max(); // 1.7976931348623157e+308 +const double MIN_DOUBLE __attribute__ ((unused)) = -std::numeric_limits::max(); +const long double MAX_LONGDOUBLE __attribute__ ((unused)) = std::numeric_limits::max(); // 1.7976931348623157e+308 +const long double MIN_LONGDOUBLE __attribute__ ((unused)) = -std::numeric_limits::max(); + +const uint64_t AUTOINCR_SATURATED __attribute__ ((unused)) = std::numeric_limits::max(); +} + + +using namespace std; // e.g. string + + +namespace ddlpackage +{ + struct ColumnType; +}; + + +namespace BRM +{ + struct EMEntry; + class DBRM; +}; + + +namespace rowgroup +{ + struct Row; +}; + + +namespace execplan +{ + struct SimpleColumn; +}; + + + +namespace datatypes +{ + +class SystemCatalog +{ +public: + + /** the set of Calpont column widths + * + */ + enum ColWidth { ONE_BIT, ONE_BYTE, TWO_BYTE, THREE_BYTE, FOUR_BYTE, FIVE_BYTE, SIX_BYTE, SEVEN_BYTE, EIGHT_BYTE }; + + + /** the set of Calpont column data types + * + */ + enum ColDataType + { + BIT, /*!< BIT type */ + TINYINT, /*!< TINYINT type */ + CHAR, /*!< CHAR type */ + SMALLINT, /*!< SMALLINT type */ + DECIMAL, /*!< DECIMAL type */ + MEDINT, /*!< MEDINT type */ + INT, /*!< INT type */ + FLOAT, /*!< FLOAT type */ + DATE, /*!< DATE type */ + BIGINT, /*!< BIGINT type */ + DOUBLE, /*!< DOUBLE type */ + DATETIME, /*!< DATETIME type */ + VARCHAR, /*!< VARCHAR type */ + VARBINARY, /*!< VARBINARY type */ + CLOB, /*!< CLOB type */ + BLOB, /*!< BLOB type */ + UTINYINT, /*!< Unsigned TINYINT type */ + USMALLINT, /*!< Unsigned SMALLINT type */ + UDECIMAL, /*!< Unsigned DECIMAL type */ + UMEDINT, /*!< Unsigned MEDINT type */ + UINT, /*!< Unsigned INT type */ + UFLOAT, /*!< Unsigned FLOAT type */ + UBIGINT, /*!< Unsigned BIGINT type */ + UDOUBLE, /*!< Unsigned DOUBLE type */ + TEXT, /*!< TEXT type */ + TIME, /*!< TIME type */ + TIMESTAMP, /*!< TIMESTAMP type */ + NUM_OF_COL_DATA_TYPE, /* NEW TYPES ABOVE HERE */ + LONGDOUBLE, /* @bug3241, dev and variance calculation only */ + STRINT, /* @bug3532, string as int for fast comparison */ + UNDEFINED, /*!< Undefined - used in UDAF API */ + BINARY, /*!< BINARY type */ + }; + + class TypeAttributesStd + { + public: + int32_t colWidth; + int32_t scale; //number after decimal points + int32_t precision; + TypeAttributesStd() + :colWidth(0), + scale(0), + precision(-1) + {} + /** + @brief Convenience method to get int128 from a std::string. + */ + int128_t decimal128FromString(const std::string& value) const; + + /** + @brief The method sets the legacy scale and precision of a wide decimal + column which is the result of an arithmetic operation. + */ + inline void setDecimalScalePrecisionLegacy(unsigned int p, unsigned int s) + { + scale = s; + + if (s == 0) + precision = p - 1; + else + precision = p - s; + } + + /** + @brief The method sets the scale and precision of a wide decimal + column which is the result of an arithmetic operation. + */ + inline void setDecimalScalePrecision(unsigned int p, unsigned int s) + { + colWidth = (p > INT64MAXPRECISION) ? + MAXDECIMALWIDTH : MAXLEGACYWIDTH; + + precision = (p > INT128MAXPRECISION) ? + INT128MAXPRECISION : p; + + scale = s; + } + + /** + @brief The method sets the scale and precision of a wide decimal + column which is the result of an arithmetic operation, based on a heuristic. + */ + inline void setDecimalScalePrecisionHeuristic(unsigned int p, unsigned int s) + { + unsigned int diff = 0; + + if (p > INT128MAXPRECISION) + { + precision = INT128MAXPRECISION; + diff = p - INT128MAXPRECISION; + } + else + { + precision = p; + } + + scale = s; + + if (diff != 0) + { + scale = s - (int)(diff * (38.0/65.0)); + + if (scale < 0) + scale = 0; + } + } + + }; + + class TypeHolderStd: public TypeAttributesStd + { + public: + ColDataType colDataType; + TypeHolderStd() + :colDataType(MEDINT) + { } + const class TypeHandler *typeHandler() const; + boost::any getNullValueForType() const; + + /** + @brief The method detects whether decimal type is wide + using csc colType. + */ + constexpr inline bool isWideDecimalType() const + { + return (colDataType == DECIMAL || + colDataType == UDECIMAL) && + colWidth == MAXDECIMALWIDTH; + } + }; + +}; + + +/** + @brief The method detects whether decimal type is wide + using datatype and width. +*/ +static constexpr inline bool isWideDecimalType(const datatypes::SystemCatalog::ColDataType &dt, + const int32_t width) +{ + return width == MAXDECIMALWIDTH && + (dt == SystemCatalog::DECIMAL || + dt == SystemCatalog::UDECIMAL); +} + + +/** convenience function to determine if column type is a char + * type + */ +inline bool isCharType(const datatypes::SystemCatalog::ColDataType type) +{ + return (datatypes::SystemCatalog::VARCHAR == type || + datatypes::SystemCatalog::CHAR == type || + datatypes::SystemCatalog::BLOB == type || + datatypes::SystemCatalog::TEXT == type); +} + +/** convenience function to determine if column type is a + * numeric type + */ +inline bool isNumeric(const datatypes::SystemCatalog::ColDataType type) +{ + switch (type) + { + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + case datatypes::SystemCatalog::UDECIMAL: + return true; + + default: + return false; + } +} + +inline bool isDecimal(const datatypes::SystemCatalog::ColDataType type) +{ + return (type == datatypes::SystemCatalog::DECIMAL || + type == datatypes::SystemCatalog::UDECIMAL); +} + +/** convenience function to determine if column type is an + * unsigned type + */ +inline bool isUnsigned(const datatypes::SystemCatalog::ColDataType type) +{ + switch (type) + { + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + return true; + + default: + return false; + } +} + +inline bool isSignedInteger(const datatypes::SystemCatalog::ColDataType type) +{ + switch (type) + { + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + return true; + + default: + return false; + } +} + + +/** + @brief Returns true if all arguments have a DECIMAL/UDECIMAL type +*/ +static inline bool isDecimalOperands(const SystemCatalog::ColDataType resultDataType, + const SystemCatalog::ColDataType leftColDataType, + const SystemCatalog::ColDataType rightColDataType) +{ + return ((resultDataType == SystemCatalog::DECIMAL || + resultDataType == SystemCatalog::UDECIMAL) && + (leftColDataType == SystemCatalog::DECIMAL || + leftColDataType == SystemCatalog::UDECIMAL) && + (rightColDataType == SystemCatalog::DECIMAL || + rightColDataType == SystemCatalog::UDECIMAL)); +} + +} // end of namespace datatypes + + + + +namespace datatypes +{ + +static constexpr int128_t minInt128 = int128_t(0x8000000000000000LL) << 64; +static constexpr int128_t maxInt128 = (int128_t(0x7FFFFFFFFFFFFFFFLL) << 64) + 0xFFFFFFFFFFFFFFFFLL; + +class ConstString +{ + const char *m_str; + size_t m_length; +public: + ConstString(const char *str, size_t length) + :m_str(str), m_length(length) + { } + const char *str() const { return m_str; } + const size_t length() const { return m_length; } + const char *end() const { return m_str + m_length; } + void bin2hex(char *o) + { + static const char hexdig[] = { '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}; + const char *e= end(); + for (const char *s= m_str; s < e; s++) + { + *o++ = hexdig[*s >> 4]; + *o++ = hexdig[*s & 0xf]; + } + } +}; + + +enum class round_style_t: uint8_t +{ + NONE= 0, + POS= 0x01, + NEG= 0x80 +}; + + +class SessionParam +{ + const char *m_tzname; +public: + SessionParam(const char *tzname) + :m_tzname(tzname) + { } + const char *tzname() const + { + return m_tzname; + } +}; + + +class SimpleValue +{ + int64_t m_sint64; + int128_t m_sint128; + const char *m_tzname; +public: + SimpleValue(const int64_t sint64, + const int128_t &sint128, + const char *tzname) + :m_sint64(sint64), m_sint128(sint128), m_tzname(tzname) + { } + SimpleValue() + :m_sint64(0), m_sint128(0), m_tzname(0) + { } + int64_t toSInt64() const { return m_sint64; } + uint64_t toUInt64() const { return static_cast(m_sint64); } + int128_t toSInt128() const { return m_sint128; } + const char *tzname() const { return m_tzname; } +}; + + +class SimpleValueSInt64: public SimpleValue +{ +public: + SimpleValueSInt64(int64_t value) + :SimpleValue(value, 0, NULL) + { } +}; + + +class SimpleValueUInt64: public SimpleValue +{ +public: + SimpleValueUInt64(uint64_t value) + :SimpleValue(static_cast(value), 0, NULL) + { } +}; + + +class SimpleValueSInt128: public SimpleValue +{ +public: + SimpleValueSInt128(int128_t value) + :SimpleValue(0, value, NULL) + { } +}; + + +class SimpleValueTimestamp: public SimpleValue +{ +public: + SimpleValueTimestamp(uint64_t value, const char *tzname) + :SimpleValue(static_cast(value), 0, tzname) + { } +}; + + +class MinMaxInfo +{ +public: + int64_t min; + int64_t max; + union + { + int128_t int128Min; + int64_t min_; + }; + union + { + int128_t int128Max; + int64_t max_; + }; + MinMaxInfo() + :min((uint64_t)0x8000000000000001ULL), + max((uint64_t) - 0x8000000000000001LL) + { + int128Min = datatypes::minInt128; + int128Max = datatypes::maxInt128; + }; + bool isEmptyOrNullSInt64() const + { + return min == std::numeric_limits::max() && + max == std::numeric_limits::min(); + } + bool isEmptyOrNullUInt64() const + { + return static_cast(min) == std::numeric_limits::max() && + static_cast(max) == std::numeric_limits::min(); + } + bool isEmptyOrNullSInt128() const + { + return int128Min == datatypes::maxInt128 && + int128Max == datatypes::minInt128; + } + void widenSInt64(const MinMaxInfo &partInfo) + { + min = partInfo.min < min ? partInfo.min : min; + max = partInfo.max > max ? partInfo.max : max; + } + void widenUInt64(const MinMaxInfo &partInfo) + { + min = static_cast(partInfo.min < static_cast(min) ? partInfo.min : min); + max = static_cast(partInfo.max > static_cast(max) ? partInfo.max : max); + } + void widenSInt128(const MinMaxInfo &partInfo) + { + int128Min = partInfo.int128Min < int128Min ? partInfo.int128Min : int128Min; + int128Max = partInfo.int128Max > int128Max ? partInfo.int128Max : int128Max; + } + static MinMaxInfo widenSInt64(const MinMaxInfo &a, + const MinMaxInfo &b) + { + MinMaxInfo tmp= a; + tmp.widenSInt64(b); + return tmp; + } + static MinMaxInfo widenUInt64(const MinMaxInfo &a, + const MinMaxInfo &b) + { + MinMaxInfo tmp(a); + tmp.widenUInt64(b); + return tmp; + } + static MinMaxInfo widenSInt128(const MinMaxInfo &a, + const MinMaxInfo &b) + { + MinMaxInfo tmp= a; + tmp.widenSInt128(b); + return tmp; + } +}; + + +class MinMaxPartitionInfo: public MinMaxInfo +{ + enum status_flag_t : uint64_t + { + ET_DISABLED = 0x0002, + CPINVALID = 0x0004 + }; + uint64_t m_status; +public: + MinMaxPartitionInfo() + :m_status(0) + { }; + MinMaxPartitionInfo(const BRM::EMEntry &entry); + void set_invalid() { m_status|= CPINVALID; } + const bool is_invalid() const { return m_status & CPINVALID; } + const bool is_disabled() const { return m_status & ET_DISABLED; } + + bool isSuitableSInt64(const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const + { + if (min >= startVal.toSInt64() && + max <= endVal.toSInt64() && + !(min == std::numeric_limits::max() && + max == std::numeric_limits::min())) + { + if (rfMin == round_style_t::POS && min == startVal.toSInt64()) + return false; + + if (rfMax == round_style_t::NEG && max == endVal.toSInt64()) + return false; + + return true; + } + return false; + } + bool isSuitableUInt64(const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const + { + if (static_cast(min) >= startVal.toUInt64() && + static_cast(max) <= endVal.toUInt64() && + !(static_cast(min) == std::numeric_limits::max() && + max == 0)) + { + if (rfMin == round_style_t::POS && min == startVal.toSInt64()) + return false; + + if (rfMax == round_style_t::NEG && max == endVal.toSInt64()) + return false; + return true; + } + return false; + } + bool isSuitableSInt128(const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const + { + if (int128Min >= startVal.toSInt128() && + int128Max <= endVal.toSInt128() && + !(int128Min == datatypes::maxInt128 && + int128Max == datatypes::minInt128)) + { + if (rfMin == round_style_t::POS && int128Min == startVal.toSInt128()) + return false; + + if (rfMax == round_style_t::NEG && int128Max == endVal.toSInt128()) + return false; + + return true; + } + return false; + } +}; + + +class ColBatchWriter +{ + FILE *m_filePtr; + char m_delimiter; + char m_enclosed_by; + bool m_utf8; +public: + ColBatchWriter(FILE *f, + char delimiter, + char enclosed_by, + bool utf8) + :m_filePtr(f), + m_delimiter(delimiter), + m_enclosed_by(enclosed_by), + m_utf8(utf8) + { } + FILE *filePtr() const { return m_filePtr; } + char delimiter() const { return m_delimiter; } + char enclosed_by() const { return m_enclosed_by; } + bool utf8() const { return m_utf8; } +}; + + +class SimpleColumnParam +{ + uint32_t m_sessionid; + bool m_columnStore; +public: + SimpleColumnParam(uint32_t sessionid, bool columnStore) + :m_sessionid(sessionid), + m_columnStore(columnStore) + { } + uint32_t sessionid() const { return m_sessionid; } + bool columnStore() const { return m_columnStore; } + void columnStore(bool value) { m_columnStore= value; } +}; + + +class DatabaseQualifiedColumnName +{ + std::string m_db; + std::string m_table; + std::string m_column; +public: + DatabaseQualifiedColumnName(const std::string &db, + const std::string &table, + const std::string &column) + :m_db(db), + m_table(table), + m_column(column) + { } + const std::string &db() const { return m_db; } + const std::string &table() const { return m_table; } + const std::string &column() const { return m_column; } +}; + + +class StoreField +{ +public: + virtual int32_t colWidth() const = 0; + virtual int32_t precision() const = 0; + virtual int store_date(int64_t val) = 0; + virtual int store_datetime(int64_t val) = 0; + virtual int store_time(int64_t val) = 0; + virtual int store_timestamp(int64_t val) = 0; + virtual int store_string(const char *str, size_t length) = 0; + virtual int store_varbinary(const char *str, size_t length) = 0; + virtual int store_xlonglong(int64_t val) = 0; + virtual int store_float(float val) = 0; + virtual int store_double(double val) = 0; + virtual int store_long_double(long double val) = 0; + virtual int store_decimal64(int64_t val) = 0; + virtual int store_decimal128(const int128_t &val) = 0; + virtual int store_lob(const char *str, size_t length) = 0; +}; + + +class WriteBatchField +{ +public: + virtual size_t ColWriteBatchDate(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchDatetime(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchTime(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchTimestamp(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchChar(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchVarchar(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchSInt64(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchUInt64(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchSInt32(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchUInt32(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchSInt16(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchUInt16(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchSInt8(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchUInt8(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchXFloat(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchXDouble(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchSLongDouble(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchXDecimal(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchVarbinary(const unsigned char *buf0, bool nullVal, ColBatchWriter &ci) = 0; + virtual size_t ColWriteBatchBlob(const unsigned char *buf0, bool nullVal, ColBatchWriter &ci) = 0; +}; + + +class TypeHandler +{ +public: + using code_t = datatypes::SystemCatalog::ColDataType; +protected: + std::string formatPartitionInfoSInt64(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const; + std::string formatPartitionInfoUInt64(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const; + + std::string PrintPartitionValueSInt64(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const; + + std::string PrintPartitionValueUInt64(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const; + +public: + static const TypeHandler *find(SystemCatalog::ColDataType typeCode, + const SystemCatalog::TypeAttributesStd &attr); + static const TypeHandler *find_by_ddltype(const ddlpackage::ColumnType &ct); + virtual ~TypeHandler() { } + virtual const string & name() const= 0; + virtual const string print(const SystemCatalog::TypeAttributesStd &attr) const + { + return name(); + } + virtual code_t code() const= 0; + virtual bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const + { + return false; + } + virtual uint8_t PartitionValueCharLength(const SystemCatalog::TypeAttributesStd &attr) + const + { + return 30; + } + virtual size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const= 0; + virtual int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const= 0; + + virtual std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const = 0; + + virtual std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const= 0; + virtual execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const= 0; + virtual SimpleValue getMinValueSimple() const + { + return SimpleValue(std::numeric_limits::min(), + std::numeric_limits::min(), + 0); + } + virtual SimpleValue getMaxValueSimple() const + { + return SimpleValue(std::numeric_limits::max(), + std::numeric_limits::max(), + 0); + } + virtual SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const= 0; + virtual MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &a, + const MinMaxInfo &b) const + { + return MinMaxInfo::widenSInt64(a, b); + } + virtual MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const; + virtual string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const + { + return PrintPartitionValueSInt64(attr, partInfo, + startVal, rfMin, + endVal, rfMax); + } + virtual bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &part, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const + { + return part.isSuitableSInt64(startVal, rfMin, endVal, rfMax); + } + virtual boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const = 0; +}; + + +// QQ: perhaps not needed yet +class TypeHandlerBit: public TypeHandler +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::BIT; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + idbassert(0); // QQ + return 0; + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + idbassert(0); // QQ + return 1; + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return "0"; // QQ + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) const override + { + idbassert(0); + return "Error"; + } + + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override + { + idbassert(0); + return NULL; + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override + { + idbassert(0); + return SimpleValue(); + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override + { + //TODO: How to communicate with write engine? + return boost::any(); + } +}; + + +class TypeHandlerInt: public TypeHandler +{ +protected: + int storeValueToFieldSInt32(rowgroup::Row &row, int pos, + StoreField *f) const; + int storeValueToFieldUInt32(rowgroup::Row &row, int pos, + StoreField *f) const; + std::string formatSInt64(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const; + std::string formatUInt64(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const; + +}; + + +class TypeHandlerSInt8: public TypeHandlerInt +{ +public: + const string &name() const override; + code_t code() const override + { + return SystemCatalog::TINYINT; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchSInt8(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return formatSInt64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoSInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue getMinValueSimple() const override + { + return SimpleValueSInt64(std::numeric_limits::min()); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueSInt64(std::numeric_limits::max()); + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerSInt16: public TypeHandlerInt +{ +public: + const string &name() const override; + code_t code() const override + { + return SystemCatalog::SMALLINT; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchSInt16(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return formatSInt64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoSInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue getMinValueSimple() const override + { + return SimpleValueSInt64(std::numeric_limits::min()); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueSInt64(std::numeric_limits::max()); + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerSInt24: public TypeHandlerInt +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::MEDINT; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchSInt32(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldSInt32(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return formatSInt64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoSInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue getMinValueSimple() const override + { + return SimpleValueSInt64(MIN_MEDINT); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueSInt64(MAX_MEDINT); + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerSInt32: public TypeHandlerInt +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::INT; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchSInt32(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldSInt32(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return formatSInt64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoSInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue getMinValueSimple() const override + { + return SimpleValueSInt64(std::numeric_limits::min()); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueSInt64(std::numeric_limits::max()); + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerSInt64: public TypeHandlerInt +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::BIGINT; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchSInt64(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return formatSInt64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoSInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue getMinValueSimple() const override + { + return SimpleValueSInt64(std::numeric_limits::min()); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueSInt64(std::numeric_limits::max()); + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerUInt8: public TypeHandlerInt +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::UTINYINT; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchUInt8(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return formatUInt64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoUInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + + SimpleValue getMinValueSimple() const override + { + return SimpleValueUInt64(0); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueUInt64(std::numeric_limits::max()); + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &a, + const MinMaxInfo &b) const override + { + return MinMaxInfo::widenUInt64(a, b); + } + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return PrintPartitionValueUInt64(attr, partInfo, + startVal, rfMin, + endVal, rfMax); + } + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &part, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return part.isSuitableUInt64(startVal, rfMin, endVal, rfMax); + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerUInt16: public TypeHandlerInt +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::USMALLINT; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchUInt16(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return formatUInt64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoUInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue getMinValueSimple() const override + { + return SimpleValueUInt64(0); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueUInt64(std::numeric_limits::max()); + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &a, + const MinMaxInfo &b) const override + { + return MinMaxInfo::widenUInt64(a, b); + } + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return PrintPartitionValueUInt64(attr, partInfo, + startVal, rfMin, + endVal, rfMax); + } + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &part, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return part.isSuitableUInt64(startVal, rfMin, endVal, rfMax); + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerUInt24: public TypeHandlerInt +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::UMEDINT; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchUInt32(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldUInt32(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return formatUInt64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoUInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue getMinValueSimple() const override + { + return SimpleValueUInt64(0); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueUInt64(MAX_UMEDINT); + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &a, + const MinMaxInfo &b) const override + { + return MinMaxInfo::widenUInt64(a, b); + } + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return PrintPartitionValueUInt64(attr, partInfo, + startVal, rfMin, + endVal, rfMax); + } + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &part, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return part.isSuitableUInt64(startVal, rfMin, endVal, rfMax); + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerUInt32: public TypeHandlerInt +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::UINT; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchUInt32(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldUInt32(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return formatUInt64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoUInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue getMinValueSimple() const override + { + return SimpleValueUInt64(0); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueUInt64(std::numeric_limits::max()); + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &a, + const MinMaxInfo &b) const override + { + return MinMaxInfo::widenUInt64(a, b); + } + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return PrintPartitionValueUInt64(attr, partInfo, + startVal, rfMin, + endVal, rfMax); + } + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &part, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return part.isSuitableUInt64(startVal, rfMin, endVal, rfMax); + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerUInt64: public TypeHandlerInt +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::BIGINT; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchUInt64(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return formatUInt64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoUInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue getMinValueSimple() const override + { + return SimpleValueUInt64(0); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueUInt64(std::numeric_limits::max()); + } + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &a, + const MinMaxInfo &b) const override + { + return MinMaxInfo::widenUInt64(a, b); + } + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return PrintPartitionValueUInt64(attr, partInfo, + startVal, rfMin, + endVal, rfMax); + } + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &part, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return part.isSuitableUInt64(startVal, rfMin, endVal, rfMax); + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerXDecimal: public TypeHandler +{ +protected: + + static bool isValidXDecimal64(const SystemCatalog::TypeAttributesStd &attr) + { + return attr.colWidth <= 8; + } + static bool isValidXDecimal128(const SystemCatalog::TypeAttributesStd &attr) + { + return attr.colWidth == 16; + } + int storeValueToField64(rowgroup::Row &row, int pos, + StoreField *f) const; + int storeValueToField128(rowgroup::Row &row, int pos, + StoreField *f) const; + + std::string format64(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const; + std::string format128(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const; + std::string formatPartitionInfo128(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const; + + MinMaxPartitionInfo getExtentPartitionInfo64(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const; + MinMaxPartitionInfo getExtentPartitionInfo128(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const; + string PrintPartitionValue128(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const; + +public: + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchXDecimal(buf, nullVal, writer); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerSDecimal64: public TypeHandlerXDecimal +{ +public: + const string &name() const override; + code_t code() const override + { + return SystemCatalog::DECIMAL; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + uint8_t PartitionValueCharLength(const SystemCatalog::TypeAttributesStd &attr) + const override + { + return 30; + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToField64(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return format64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) const override + { + return formatPartitionInfoSInt64(attr, i); + } + SimpleValue getMinValueSimple() const override + { + return SimpleValueSInt64(std::numeric_limits::min()); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueSInt64(std::numeric_limits::max()); + } + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &a, + const MinMaxInfo &b) const override + { + return MinMaxInfo::widenSInt64(a, b); + } + MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const override + { + return getExtentPartitionInfo64(attr, em, entry, state); + } + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return PrintPartitionValueSInt64(attr, partInfo, + startVal, rfMin, + endVal, rfMax); + } + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &part, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return part.isSuitableSInt64(startVal, rfMin, endVal, rfMax); + } +}; + + +class TypeHandlerUDecimal64: public TypeHandlerXDecimal +{ +public: + const string &name() const override; + code_t code() const override + { + return SystemCatalog::UDECIMAL; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + uint8_t PartitionValueCharLength(const SystemCatalog::TypeAttributesStd &attr) + const override + { + return 30; + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToField64(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return format64(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) const override + { + return formatPartitionInfoSInt64(attr, i); + } + SimpleValue getMinValueSimple() const override + { + return SimpleValueUInt64(0); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueUInt64(std::numeric_limits::max()); + } + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &a, + const MinMaxInfo &b) const override + { + return MinMaxInfo::widenSInt64(a, b); + } + MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const override + { + return getExtentPartitionInfo64(attr, em, entry, state); + } + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return PrintPartitionValueSInt64(attr, partInfo, + startVal, rfMin, + endVal, rfMax); + } + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &part, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return part.isSuitableSInt64(startVal, rfMin, endVal, rfMax); + } +}; + + +class TypeHandlerSDecimal128: public TypeHandlerXDecimal +{ +public: + const string &name() const override; + code_t code() const override + { + return SystemCatalog::DECIMAL; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + uint8_t PartitionValueCharLength(const SystemCatalog::TypeAttributesStd &attr) + const override + { + return Decimal::MAXLENGTH16BYTES; + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToField128(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return format128(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) const override + { + return formatPartitionInfo128(attr, i); + } + SimpleValue getMinValueSimple() const override + { + return SimpleValue(std::numeric_limits::min(), + datatypes::minInt128, + 0); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValue(std::numeric_limits::max(), + datatypes::maxInt128, + 0); + } + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &a, + const MinMaxInfo &b) const override + { + return MinMaxInfo::widenSInt128(a, b); + } + MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const override + { + return getExtentPartitionInfo128(attr, em, entry, state); + } + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return PrintPartitionValue128(attr, partInfo, startVal, rfMin, endVal, rfMax); + } + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &part, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return part.isSuitableSInt128(startVal, rfMin, endVal, rfMax); + } +}; + + +class TypeHandlerUDecimal128: public TypeHandlerXDecimal +{ +public: + const string &name() const override; + code_t code() const override + { + return SystemCatalog::UDECIMAL; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + uint8_t PartitionValueCharLength(const SystemCatalog::TypeAttributesStd &attr) + const override + { + return Decimal::MAXLENGTH16BYTES; + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToField128(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return format128(v, attr); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) const override + { + return formatPartitionInfo128(attr, i); + } + SimpleValue getMinValueSimple() const override + { + return SimpleValueSInt128(0); + } + SimpleValue getMaxValueSimple() const override + { + return SimpleValueSInt128(-1); + } + MinMaxInfo widenMinMaxInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &a, + const MinMaxInfo &b) const override + { + return MinMaxInfo::widenSInt128(a, b); + } + MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const override + { + return getExtentPartitionInfo128(attr, em, entry, state); + } + string PrintPartitionValue(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &partInfo, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return PrintPartitionValue128(attr, partInfo, startVal, rfMin, endVal, rfMax); + } + bool isSuitablePartition(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxPartitionInfo &part, + const SimpleValue &startVal, + round_style_t rfMin, + const SimpleValue &endVal, + round_style_t rfMax) const override + { + return part.isSuitableSInt128(startVal, rfMin, endVal, rfMax); + } +}; + + +class TypeHandlerReal: public TypeHandler +{ +public: + int storeValueToFieldXFloat(rowgroup::Row &row, int pos, + StoreField *f) const; + int storeValueToFieldXDouble(rowgroup::Row &row, int pos, + StoreField *f) const; + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override + { + return SimpleValue(); // QQ: real types were not handled in IDB_format() + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return "0"; // QQ + } +}; + + +class TypeHandlerSFloat: public TypeHandlerReal +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::FLOAT; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchXFloat(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldXFloat(row, pos, f); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoSInt64(attr, i); + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerSDouble: public TypeHandlerReal +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::DOUBLE; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchXDouble(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldXDouble(row, pos, f); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoSInt64(attr, i); + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerUFloat: public TypeHandlerReal +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::UFLOAT; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchXFloat(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldXFloat(row, pos, f); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoSInt64(attr, i); + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerUDouble: public TypeHandlerReal +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::UDOUBLE; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchXDouble(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldXDouble(row, pos, f); + } + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoSInt64(attr, i); + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerSLongDouble: public TypeHandlerReal +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::LONGDOUBLE; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchSLongDouble(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + idbassert(0); + return "Error"; + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override + { + // QQ: DDLPackageProcessor::getNullValueForType() did not handle LONGDOUBLE + return boost::any(); + } +}; + + +class TypeHandlerTemporal: public TypeHandler +{ +public: + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + return formatPartitionInfoSInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; +}; + + +class TypeHandlerDate: public TypeHandlerTemporal +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::DATE; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchDate(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override; + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerDatetime: public TypeHandlerTemporal +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::DATETIME; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchDatetime(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override; + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerTime: public TypeHandlerTemporal +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::TIME; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchTime(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override; + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerTimestamp: public TypeHandlerTemporal +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::TIMESTAMP; + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return true; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchTimestamp(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override; + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerStr: public TypeHandler +{ +protected: + std::string formatPartitionInfoSmallCharVarchar( + const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const; + boost::any getNullValueForTypeVarcharText(const SystemCatalog::TypeAttributesStd &attr) + const; +public: + int storeValueToFieldCharVarchar(rowgroup::Row &row, int pos, + StoreField *f) const; + int storeValueToFieldBlobText(rowgroup::Row &row, int pos, + StoreField *f) const; + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override + { + // QQ: Check with Roman if this correct: + return formatPartitionInfoSInt64(attr, i); + } + execplan::SimpleColumn *newSimpleColumn(const DatabaseQualifiedColumnName &name, + SystemCatalog::TypeHolderStd &ct, + const SimpleColumnParam &prm) + const override; + SimpleValue toSimpleValue(const SessionParam &sp, + const SystemCatalog::TypeAttributesStd &attr, + const char *str, round_style_t & rf) const override; +}; + + +class TypeHandlerChar: public TypeHandlerStr +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::CHAR; + } + const string print(const SystemCatalog::TypeAttributesStd &attr) const override + { + ostringstream oss; + oss << name () << "(" << attr.colWidth << ")"; + return oss.str(); + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return attr.colWidth <= 8; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchChar(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldCharVarchar(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override; + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override; + MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerVarchar: public TypeHandlerStr +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::VARCHAR; + } + const string print(const SystemCatalog::TypeAttributesStd &attr) const override + { + ostringstream oss; + oss << name () << "(" << attr.colWidth << ")"; + return oss.str(); + } + bool CP_type(const SystemCatalog::TypeAttributesStd &attr) const override + { + return attr.colWidth <= 7; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchVarchar(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldCharVarchar(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override; + std::string formatPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + const MinMaxInfo &i) + const override; + MinMaxPartitionInfo getExtentPartitionInfo(const SystemCatalog::TypeAttributesStd &attr, + BRM::DBRM &em, + const BRM::EMEntry &entry, + int *state) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override + { + return getNullValueForTypeVarcharText(attr); + } +}; + + +class TypeHandlerVarbinary: public TypeHandlerStr +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::VARBINARY; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchVarbinary(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override; + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerBlob: public TypeHandlerStr +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::BLOB; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchBlob(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldBlobText(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return "0"; // QQ + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override; +}; + + +class TypeHandlerText: public TypeHandlerStr +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::TEXT; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + return field->ColWriteBatchBlob(buf, nullVal, writer); + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + return storeValueToFieldBlobText(row, pos, f); + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return "0"; // QQ + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override + { + return getNullValueForTypeVarcharText(attr); + } +}; + + +class TypeHandlerClob: public TypeHandlerStr +{ + const string &name() const override; + code_t code() const override + { + return SystemCatalog::CLOB; + } + size_t ColWriteBatch(WriteBatchField *field, + const unsigned char *buf, + bool nullVal, + ColBatchWriter & writer) const override + { + idbassert(0); // QQ + return 0; + } + int storeValueToField(rowgroup::Row &row, int pos, + StoreField *f) const override + { + idbassert(0); // QQ + return 1; + } + std::string format(const SimpleValue &v, + const SystemCatalog::TypeAttributesStd &attr) const override + { + return "0"; // QQ + } + boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) + const override + { + return boost::any(); // QQ + } +}; + + +}// end of namespace datatypes + +#endif //MCS_DATATYPE_H_INCLUDED + +// vim:ts=2 sw=2: diff --git a/datatypes/mcs_decimal.cpp b/datatypes/mcs_decimal.cpp index 4b7c4e7c6..f66cc68ef 100644 --- a/datatypes/mcs_decimal.cpp +++ b/datatypes/mcs_decimal.cpp @@ -18,9 +18,10 @@ #include #include +#include "utils/common/branchpred.h" #include "mcs_decimal.h" -#include "treenode.h" #include "exceptclasses.h" +#include "dataconvert.h" namespace datatypes { @@ -48,16 +49,16 @@ namespace datatypes template - void addSubtractExecute(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, - execplan::IDB_Decimal& result, + void addSubtractExecute(const VDecimal& l, + const VDecimal& r, + VDecimal& result, BinaryOperation op, OpOverflowCheck opOverflowCheck, MultiplicationOverflowCheck mulOverflowCheck) { - int128_t lValue = Decimal::isWideDecimalType(l.precision) + int128_t lValue = Decimal::isWideDecimalTypeByPrecision(l.precision) ? l.s128Value : l.value; - int128_t rValue = Decimal::isWideDecimalType(r.precision) + int128_t rValue = Decimal::isWideDecimalTypeByPrecision(r.precision) ? r.s128Value : r.value; if (result.scale == l.scale && result.scale == r.scale) @@ -108,15 +109,15 @@ namespace datatypes template - void divisionExecute(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, - execplan::IDB_Decimal& result, + void divisionExecute(const VDecimal& l, + const VDecimal& r, + VDecimal& result, OpOverflowCheck opOverflowCheck, MultiplicationOverflowCheck mulOverflowCheck) { - int128_t lValue = Decimal::isWideDecimalType(l.precision) + int128_t lValue = Decimal::isWideDecimalTypeByPrecision(l.precision) ? l.s128Value : l.value; - int128_t rValue = Decimal::isWideDecimalType(r.precision) + int128_t rValue = Decimal::isWideDecimalTypeByPrecision(r.precision) ? r.s128Value : r.value; opOverflowCheck(lValue, rValue); @@ -147,15 +148,15 @@ namespace datatypes template - void multiplicationExecute(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, - execplan::IDB_Decimal& result, + void multiplicationExecute(const VDecimal& l, + const VDecimal& r, + VDecimal& result, OpOverflowCheck opOverflowCheck, MultiplicationOverflowCheck mulOverflowCheck) { - int128_t lValue = Decimal::isWideDecimalType(l.precision) + int128_t lValue = Decimal::isWideDecimalTypeByPrecision(l.precision) ? l.s128Value : l.value; - int128_t rValue = Decimal::isWideDecimalType(r.precision) + int128_t rValue = Decimal::isWideDecimalTypeByPrecision(r.precision) ? r.s128Value : r.value; if (lValue == 0 || rValue == 0) @@ -194,36 +195,21 @@ namespace datatypes } } - std::string Decimal::toString(execplan::IDB_Decimal& value) + std::string Decimal::toString(VDecimal& value) { - char buf[utils::MAXLENGTH16BYTES]; + char buf[Decimal::MAXLENGTH16BYTES]; dataconvert::DataConvert::decimalToString(&value.s128Value, - value.scale, buf, sizeof(buf), - execplan::CalpontSystemCatalog::DECIMAL); + value.scale, buf, (uint8_t) sizeof(buf), + datatypes::SystemCatalog::DECIMAL); return std::string(buf); } - std::string Decimal::toString(const execplan::IDB_Decimal& value) + std::string Decimal::toString(const VDecimal& value) { - return toString(const_cast(value)); + return toString(const_cast(value)); } - // Compare perf with f(string&, ColTypeAlias&, int128_t&) - int128_t Decimal::int128FromString(const std::string& value, - ColTypeAlias& colType) - { - int128_t result = 0; - bool pushWarning = false; - bool noRoundup = false; - dataconvert::number_int_value(value, - colType, - pushWarning, - noRoundup, - result); - return result; - } - - int Decimal::compare(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r) + int Decimal::compare(const VDecimal& l, const VDecimal& r) { int128_t divisorL, divisorR; getScaleDivisor(divisorL, l.scale); @@ -270,8 +256,8 @@ namespace datatypes // no overflow check template<> - void Decimal::addition(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + void Decimal::addition(const VDecimal& l, + const VDecimal& r, VDecimal& result) { std::plus add; NoOverflowCheck noOverflowCheck; @@ -280,8 +266,8 @@ namespace datatypes // with overflow check template<> - void Decimal::addition(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + void Decimal::addition(const VDecimal& l, + const VDecimal& r, VDecimal& result) { std::plus add; AdditionOverflowCheck overflowCheck; @@ -291,8 +277,8 @@ namespace datatypes // no overflow check template<> - void Decimal::addition(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + void Decimal::addition(const VDecimal& l, + const VDecimal& r, VDecimal& result) { if (result.scale == l.scale && result.scale == r.scale) { @@ -321,8 +307,8 @@ namespace datatypes // with overflow check template<> - void Decimal::addition(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + void Decimal::addition(const VDecimal& l, + const VDecimal& r, VDecimal& result) { AdditionOverflowCheck additionOverflowCheck; MultiplicationOverflowCheck mulOverflowCheck; @@ -356,8 +342,8 @@ namespace datatypes // no overflow check template<> - void Decimal::subtraction(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + void Decimal::subtraction(const VDecimal& l, + const VDecimal& r, VDecimal& result) { std::minus subtract; NoOverflowCheck noOverflowCheck; @@ -366,8 +352,8 @@ namespace datatypes // with overflow check template<> - void Decimal::subtraction(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + void Decimal::subtraction(const VDecimal& l, + const VDecimal& r, VDecimal& result) { std::minus subtract; SubtractionOverflowCheck overflowCheck; @@ -377,8 +363,8 @@ namespace datatypes // no overflow check template<> - void Decimal::subtraction(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + void Decimal::subtraction(const VDecimal& l, + const VDecimal& r, VDecimal& result) { if (result.scale == l.scale && result.scale == r.scale) { @@ -407,8 +393,8 @@ namespace datatypes // with overflow check template<> - void Decimal::subtraction(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + void Decimal::subtraction(const VDecimal& l, + const VDecimal& r, VDecimal& result) { SubtractionOverflowCheck subtractionOverflowCheck; MultiplicationOverflowCheck mulOverflowCheck; @@ -442,8 +428,8 @@ namespace datatypes // no overflow check template<> - void Decimal::division(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + void Decimal::division(const VDecimal& l, + const VDecimal& r, VDecimal& result) { NoOverflowCheck noOverflowCheck; divisionExecute(l, r, result, noOverflowCheck, noOverflowCheck); @@ -451,8 +437,8 @@ namespace datatypes // With overflow check template<> - void Decimal::division(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + void Decimal::division(const VDecimal& l, + const VDecimal& r, VDecimal& result) { DivisionOverflowCheck overflowCheck; MultiplicationOverflowCheck mulOverflowCheck; @@ -462,8 +448,8 @@ namespace datatypes // no overflow check // We rely on the zero check from ArithmeticOperator::execute template<> - void Decimal::division(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + void Decimal::division(const VDecimal& l, + const VDecimal& r, VDecimal& result) { if (result.scale >= l.scale - r.scale) result.value = (int64_t)(( (l.value > 0 && r.value > 0) || (l.value < 0 && r.value < 0) ? @@ -477,8 +463,8 @@ namespace datatypes // With overflow check template<> - void Decimal::division(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + void Decimal::division(const VDecimal& l, + const VDecimal& r, VDecimal& result) { DivisionOverflowCheck divisionOverflowCheck; @@ -497,8 +483,8 @@ namespace datatypes // no overflow check template<> - void Decimal::multiplication(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + void Decimal::multiplication(const VDecimal& l, + const VDecimal& r, VDecimal& result) { MultiplicationNoOverflowCheck noOverflowCheck; multiplicationExecute(l, r, result, noOverflowCheck, noOverflowCheck); @@ -506,8 +492,8 @@ namespace datatypes // With overflow check template<> - void Decimal::multiplication(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + void Decimal::multiplication(const VDecimal& l, + const VDecimal& r, VDecimal& result) { MultiplicationOverflowCheck mulOverflowCheck; multiplicationExecute(l, r, result, mulOverflowCheck, mulOverflowCheck); @@ -515,8 +501,8 @@ namespace datatypes // no overflow check template<> - void Decimal::multiplication(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + void Decimal::multiplication(const VDecimal& l, + const VDecimal& r, VDecimal& result) { if (result.scale >= l.scale + r.scale) result.value = l.value * r.value * mcs_pow_10[result.scale - (l.scale + r.scale)]; @@ -528,8 +514,8 @@ namespace datatypes // With overflow check template<> - void Decimal::multiplication(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, execplan::IDB_Decimal& result) + void Decimal::multiplication(const VDecimal& l, + const VDecimal& r, VDecimal& result) { MultiplicationOverflowCheck mulOverflowCheck; diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index 178dce1ba..dcef8b947 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -20,17 +20,16 @@ #include #include - -#include "calpontsystemcatalog.h" +#include +#include "exceptclasses.h" +#include "widedecimalutils.h" using int128_t = __int128; using uint128_t = unsigned __int128; -using ColTypeAlias = execplan::CalpontSystemCatalog::ColType; -using ColDataTypeAlias = execplan::CalpontSystemCatalog::ColDataType; -namespace execplan +namespace datatypes { - struct IDB_Decimal; + struct VDecimal; } // A class by Fabio Fernandes pulled off of stackoverflow @@ -81,8 +80,6 @@ constexpr uint8_t INT128MAXPRECISION = 38U; constexpr uint8_t MAXLEGACYWIDTH = 8U; constexpr uint8_t MAXSCALEINC4AVG = 4U; constexpr int8_t IGNOREPRECISION = -1; -constexpr uint8_t MAXLENGTH16BYTES = 42; -constexpr uint8_t MAXLENGTH8BYTES = 23; @@ -175,154 +172,109 @@ class Decimal Decimal() { }; ~Decimal() { }; + static constexpr uint8_t MAXLENGTH16BYTES = 42; + static constexpr uint8_t MAXLENGTH8BYTES = 23; + + static inline bool isWideDecimalNullValue(const int128_t& val) + { + const uint64_t* ptr = reinterpret_cast(&val); + return (ptr[0] == utils::BINARYNULLVALUELOW && ptr[1] == utils::BINARYNULLVALUEHIGH); + } + + static inline bool isWideDecimalEmptyValue(const int128_t& val) + { + const uint64_t* ptr = reinterpret_cast(&val); + return (ptr[0] == utils::BINARYEMPTYVALUELOW && ptr[1] == utils::BINARYEMPTYVALUEHIGH); + } + + static inline void setWideDecimalNullValue(int128_t& val) + { + uint64_t* ptr = reinterpret_cast(&val); + ptr[0] = utils::BINARYNULLVALUELOW; + ptr[1] = utils::BINARYNULLVALUEHIGH; + } + + static inline void setWideDecimalEmptyValue(int128_t& val) + { + uint64_t* ptr = reinterpret_cast(&val); + ptr[0] = utils::BINARYEMPTYVALUELOW; + ptr[1] = utils::BINARYEMPTYVALUEHIGH; + } + + static inline void setWideDecimalNullValue(int128_t* val) + { + uint64_t* ptr = reinterpret_cast(val); + ptr[0] = utils::BINARYNULLVALUELOW; + ptr[1] = utils::BINARYNULLVALUEHIGH; + } + + static inline void setWideDecimalEmptyValue(int128_t* val) + { + uint64_t* ptr = reinterpret_cast(val); + ptr[0] = utils::BINARYEMPTYVALUELOW; + ptr[1] = utils::BINARYEMPTYVALUEHIGH; + } + + static constexpr int128_t minInt128 = int128_t(0x8000000000000000LL) << 64; static constexpr int128_t maxInt128 = (int128_t(0x7FFFFFFFFFFFFFFFLL) << 64) + 0xFFFFFFFFFFFFFFFFLL; /** - @brief Compares two IDB_Decimal taking scale into account. + @brief Compares two VDecimal taking scale into account. */ - static int compare(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r); + static int compare(const VDecimal& l, const VDecimal& r); /** @brief Addition template that supports overflow check and two internal representations of decimal. */ template - static void addition(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, - execplan::IDB_Decimal& result); + static void addition(const VDecimal& l, + const VDecimal& r, + VDecimal& result); /** @brief Subtraction template that supports overflow check and two internal representations of decimal. */ template - static void subtraction(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, - execplan::IDB_Decimal& result); + static void subtraction(const VDecimal& l, + const VDecimal& r, + VDecimal& result); /** @brief Division template that supports overflow check and two internal representations of decimal. */ template - static void division(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, - execplan::IDB_Decimal& result); + static void division(const VDecimal& l, + const VDecimal& r, + VDecimal& result); /** @brief Multiplication template that supports overflow check and two internal representations of decimal. */ template - static void multiplication(const execplan::IDB_Decimal& l, - const execplan::IDB_Decimal& r, - execplan::IDB_Decimal& result); + static void multiplication(const VDecimal& l, + const VDecimal& r, + VDecimal& result); /** @brief Convenience method to put decimal into a std::string. */ - static std::string toString(execplan::IDB_Decimal& value); - static std::string toString(const execplan::IDB_Decimal& value); - - /** - @brief Convenience method to get int128 from a std::string. - */ - static int128_t int128FromString(const std::string& value, ColTypeAlias& colType); - - /** - @brief The method detects whether decimal type is wide - using csc colType. - */ - static constexpr inline bool isWideDecimalType(const ColTypeAlias& ct) - { - return ((ct.colDataType == execplan::CalpontSystemCatalog::DECIMAL || - ct.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && - ct.colWidth == MAXDECIMALWIDTH); - } + static std::string toString(VDecimal& value); + static std::string toString(const VDecimal& value); /** @brief The method detects whether decimal type is wide using precision. */ - static constexpr inline bool isWideDecimalType(const int32_t precision) + static constexpr inline bool isWideDecimalTypeByPrecision(const int32_t precision) { return precision > INT64MAXPRECISION && precision <= INT128MAXPRECISION; } - /** - @brief The method detects whether decimal type is wide - using datatype and width. - */ - static constexpr inline bool isWideDecimalType(const ColDataTypeAlias dt, - const int32_t width) - { - return (width == MAXDECIMALWIDTH && - (dt == execplan::CalpontSystemCatalog::DECIMAL || - dt == execplan::CalpontSystemCatalog::UDECIMAL)); - } - - /** - @brief The method sets the legacy scale and precision of a wide decimal - column which is the result of an arithmetic operation. - */ - static inline void setDecimalScalePrecisionLegacy(ColTypeAlias& ct, - unsigned int precision, unsigned int scale) - { - ct.scale = scale; - - if (ct.scale == 0) - ct.precision = precision - 1; - else - ct.precision = precision - scale; - } - - /** - @brief The method sets the scale and precision of a wide decimal - column which is the result of an arithmetic operation. - */ - static inline void setDecimalScalePrecision(ColTypeAlias& ct, - unsigned int precision, unsigned int scale) - { - ct.colWidth = (precision > INT64MAXPRECISION) - ? MAXDECIMALWIDTH : MAXLEGACYWIDTH; - - ct.precision = (precision > INT128MAXPRECISION) - ? INT128MAXPRECISION : precision; - - ct.scale = scale; - } - - /** - @brief The method sets the scale and precision of a wide decimal - column which is the result of an arithmetic operation, based on a heuristic. - */ - static inline void setDecimalScalePrecisionHeuristic(ColTypeAlias& ct, - unsigned int precision, unsigned int scale) - { - unsigned int diff = 0; - - if (precision > INT128MAXPRECISION) - { - ct.precision = INT128MAXPRECISION; - diff = precision - INT128MAXPRECISION; - } - else - { - ct.precision = precision; - } - - ct.scale = scale; - - if (diff != 0) - { - ct.scale = scale - (int)(diff * (38.0/65.0)); - - if (ct.scale < 0) - ct.scale = 0; - } - } - /** @brief The method converts a __float128 value to an int64_t. */ @@ -470,21 +422,6 @@ class Decimal scale += (scaleAvailable >= MAXSCALEINC4AVG) ? MAXSCALEINC4AVG : scaleAvailable; precision += (precisionAvailable >= MAXSCALEINC4AVG) ? MAXSCALEINC4AVG : precisionAvailable; } - - /** - @brief Returns true if all arguments have a DECIMAL/UDECIMAL type - */ - static inline bool isDecimalOperands(const ColDataTypeAlias resultDataType, - const ColDataTypeAlias leftColDataType, - const ColDataTypeAlias rightColDataType) - { - return ((resultDataType == execplan::CalpontSystemCatalog::DECIMAL || - resultDataType == execplan::CalpontSystemCatalog::UDECIMAL) && - (leftColDataType == execplan::CalpontSystemCatalog::DECIMAL || - leftColDataType == execplan::CalpontSystemCatalog::UDECIMAL) && - (rightColDataType == execplan::CalpontSystemCatalog::DECIMAL || - rightColDataType == execplan::CalpontSystemCatalog::UDECIMAL)); - } }; /** @@ -627,5 +564,291 @@ struct NoOverflowCheck { } }; + +/** + * @brief VDecimal type + * + */ +struct VDecimal +{ + VDecimal(): s128Value(0), value(0), scale(0), precision(0) + { + } + + VDecimal(int64_t val, int8_t s, uint8_t p, const int128_t &val128 = 0) : + s128Value(val128), + value(val), + scale(s), + precision(p) + { + } + + int decimalComp(const VDecimal& d) const + { + lldiv_t d1 = lldiv(value, static_cast(mcs_pow_10[scale])); + lldiv_t d2 = lldiv(d.value, static_cast(mcs_pow_10[d.scale])); + + int ret = 0; + + if (d1.quot > d2.quot) + { + ret = 1; + } + else if (d1.quot < d2.quot) + { + ret = -1; + } + else + { + // rem carries the value's sign, but needs to be normalized. + int64_t s = scale - d.scale; + + if (s < 0) + { + if ((d1.rem * static_cast(mcs_pow_10[-s])) > d2.rem) + ret = 1; + else if ((d1.rem * static_cast(mcs_pow_10[-s])) < d2.rem) + ret = -1; + } + else + { + if (d1.rem > (d2.rem * static_cast(mcs_pow_10[s]))) + ret = 1; + else if (d1.rem < (d2.rem * static_cast(mcs_pow_10[s]))) + ret = -1; + } + } + + return ret; + } + + bool operator==(const VDecimal& rhs) const + { + if (precision > datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return s128Value == rhs.s128Value; + else + return (datatypes::Decimal::compare(*this, rhs) == 0); + } + else if (precision > datatypes::INT64MAXPRECISION && + rhs.precision <= datatypes::INT64MAXPRECISION) + { + const_cast(rhs).s128Value = rhs.value; + + if (scale == rhs.scale) + return s128Value == rhs.s128Value; + else + return (datatypes::Decimal::compare(*this, rhs) == 0); + } + else if (precision <= datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return (int128_t) value == rhs.s128Value; + else + return (datatypes::Decimal::compare(VDecimal(0, scale, precision, (int128_t) value), rhs) == 0); + } + else + { + if (scale == rhs.scale) + return value == rhs.value; + else + return (decimalComp(rhs) == 0); + } + } + + bool operator>(const VDecimal& rhs) const + { + if (precision > datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return s128Value > rhs.s128Value; + else + return (datatypes::Decimal::compare(*this, rhs) > 0); + } + else if (precision > datatypes::INT64MAXPRECISION && + rhs.precision <= datatypes::INT64MAXPRECISION) + { + VDecimal rhstmp(0, rhs.scale, rhs.precision, (int128_t) rhs.value); + + if (scale == rhstmp.scale) + return s128Value > rhstmp.s128Value; + else + return (datatypes::Decimal::compare(*this, rhstmp) > 0); + } + else if (precision <= datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return (int128_t) value > rhs.s128Value; + else + return (datatypes::Decimal::compare(VDecimal(0, scale, precision, (int128_t) value), rhs) > 0); + } + else + { + if (scale == rhs.scale) + return value > rhs.value; + else + return (decimalComp(rhs) > 0); + } + } + + bool operator<(const VDecimal& rhs) const + { + if (precision > datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return s128Value < rhs.s128Value; + else + return (datatypes::Decimal::compare(*this, rhs) < 0); + } + else if (precision > datatypes::INT64MAXPRECISION && + rhs.precision <= datatypes::INT64MAXPRECISION) + { + VDecimal rhstmp(0, rhs.scale, rhs.precision, (int128_t) rhs.value); + + if (scale == rhstmp.scale) + return s128Value < rhstmp.s128Value; + else + return (datatypes::Decimal::compare(*this, rhstmp) < 0); + } + else if (precision <= datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return (int128_t) value < rhs.s128Value; + else + return (datatypes::Decimal::compare(VDecimal(0, scale, precision, (int128_t) value), rhs) < 0); + } + else + { + if (scale == rhs.scale) + return value < rhs.value; + else + return (decimalComp(rhs) < 0); + } + } + + bool operator>=(const VDecimal& rhs) const + { + if (precision > datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return s128Value >= rhs.s128Value; + else + return (datatypes::Decimal::compare(*this, rhs) >= 0); + } + else if (precision > datatypes::INT64MAXPRECISION && + rhs.precision <= datatypes::INT64MAXPRECISION) + { + VDecimal rhstmp(0, rhs.scale, rhs.precision, (int128_t) rhs.value); + + if (scale == rhstmp.scale) + return s128Value >= rhstmp.s128Value; + else + return (datatypes::Decimal::compare(*this, rhstmp) >= 0); + } + else if (precision <= datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return (int128_t) value >= rhs.s128Value; + else + return (datatypes::Decimal::compare(VDecimal(0, scale, precision, (int128_t) value), rhs) >= 0); + } + else + { + if (scale == rhs.scale) + return value >= rhs.value; + else + return (decimalComp(rhs) >= 0); + } + } + + bool operator<=(const VDecimal& rhs) const + { + if (precision > datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return s128Value <= rhs.s128Value; + else + return (datatypes::Decimal::compare(*this, rhs) <= 0); + } + else if (precision > datatypes::INT64MAXPRECISION && + rhs.precision <= datatypes::INT64MAXPRECISION) + { + VDecimal rhstmp(0, rhs.scale, rhs.precision, (int128_t) rhs.value); + + if (scale == rhstmp.scale) + return s128Value <= rhstmp.s128Value; + else + return (datatypes::Decimal::compare(*this, rhstmp) <= 0); + } + else if (precision <= datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return (int128_t) value <= rhs.s128Value; + else + return (datatypes::Decimal::compare(VDecimal(0, scale, precision, (int128_t) value), rhs) <= 0); + } + else + { + if (scale == rhs.scale) + return value <= rhs.value; + else + return (decimalComp(rhs) <= 0); + } + } + + bool operator!=(const VDecimal& rhs) const + { + if (precision > datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return s128Value != rhs.s128Value; + else + return (datatypes::Decimal::compare(*this, rhs) != 0); + } + else if (precision > datatypes::INT64MAXPRECISION && + rhs.precision <= datatypes::INT64MAXPRECISION) + { + VDecimal rhstmp(0, rhs.scale, rhs.precision, (int128_t) rhs.value); + + if (scale == rhstmp.scale) + return s128Value != rhstmp.s128Value; + else + return (datatypes::Decimal::compare(*this, rhstmp) != 0); + } + else if (precision <= datatypes::INT64MAXPRECISION && + rhs.precision > datatypes::INT64MAXPRECISION) + { + if (scale == rhs.scale) + return (int128_t) value != rhs.s128Value; + else + return (datatypes::Decimal::compare(VDecimal(0, scale, precision, (int128_t) value), rhs) != 0); + } + else + { + if (scale == rhs.scale) + return value != rhs.value; + else + return (decimalComp(rhs) != 0); + } + } + + int128_t s128Value; + int64_t value; + int8_t scale; // 0~38 + uint8_t precision; // 1~38 +}; + } //end of namespace #endif diff --git a/dbcon/ddlpackageproc/createindexprocessor.cpp b/dbcon/ddlpackageproc/createindexprocessor.cpp index 30b8eeaf7..92787f02f 100644 --- a/dbcon/ddlpackageproc/createindexprocessor.cpp +++ b/dbcon/ddlpackageproc/createindexprocessor.cpp @@ -220,12 +220,12 @@ CreateIndexProcessor::DDLResult CreateIndexProcessor::processPackage(ddlpackage: } else if (CONSTRAINTPRIM_COL == column.tableColName.column) { - colTuple.data = getNullValueForType(column.colType); + colTuple.data =column.colType.getNullValueForType(); isNull = true; } else if (CONSTRAINTTEXT_COL == column.tableColName.column) { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); isNull = true; } else if (INDEXNAME_COL == column.tableColName.column) @@ -235,7 +235,7 @@ CreateIndexProcessor::DDLResult CreateIndexProcessor::processPackage(ddlpackage: } else { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); isNull = true; } @@ -336,7 +336,7 @@ CreateIndexProcessor::DDLResult CreateIndexProcessor::processPackage(ddlpackage: } else { - colTupleCol.data = getNullValueForType(column.colType); + colTupleCol.data = column.colType.getNullValueForType(); isNull = true; } diff --git a/dbcon/ddlpackageproc/ddlindexpopulator.h b/dbcon/ddlpackageproc/ddlindexpopulator.h index de623aa76..71910c32d 100644 --- a/dbcon/ddlpackageproc/ddlindexpopulator.h +++ b/dbcon/ddlpackageproc/ddlindexpopulator.h @@ -291,7 +291,7 @@ private: : DDLPackageProcessor(), fType(ctype) {} boost::any operator()(execplan::CalpontSystemCatalog::ColType& ctype) { - return getNullValueForType(fType); + return ctype.getNullValueForType(); } const execplan::CalpontSystemCatalog::ColType& fType; }; diff --git a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp index 97712bf38..0f21db9cb 100644 --- a/dbcon/ddlpackageproc/ddlpackageprocessor.cpp +++ b/dbcon/ddlpackageproc/ddlpackageprocessor.cpp @@ -27,6 +27,7 @@ using namespace std; #include "ddlpackageprocessor.h" +#include "mcs_datatype.h" #include "dataconvert.h" using namespace dataconvert; #include "calpontselectexecutionplan.h" @@ -390,237 +391,6 @@ char DDLPackageProcessor::getConstraintCode(ddlpackage::DDL_CONSTRAINTS type) return constraint_type; } -boost::any -DDLPackageProcessor::getNullValueForType(const execplan::CalpontSystemCatalog::ColType& colType) -{ - boost::any value; - - switch (colType.colDataType) - { - case execplan::CalpontSystemCatalog::BIT: - break; - - case execplan::CalpontSystemCatalog::TINYINT: - { - char tinyintvalue = joblist::TINYINTNULL; - value = tinyintvalue; - - } - break; - - case execplan::CalpontSystemCatalog::SMALLINT: - { - short smallintvalue = joblist::SMALLINTNULL; - value = smallintvalue; - } - break; - - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - { - int intvalue = joblist::INTNULL; - value = intvalue; - } - break; - - case execplan::CalpontSystemCatalog::BIGINT: - { - long long bigint = joblist::BIGINTNULL; - value = bigint; - } - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - if (colType.colWidth <= 4) - { - short smallintvalue = joblist::SMALLINTNULL; - value = smallintvalue; - } - else if (colType.colWidth <= 9) - { - int intvalue = joblist::INTNULL; - value = intvalue; - } - else if (colType.colWidth <= 18) - { - long long eightbyte = joblist::BIGINTNULL; - value = eightbyte; - } - else - { - WriteEngine::Token nullToken; - value = nullToken; - } - } - break; - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - uint32_t jlfloatnull = joblist::FLOATNULL; - float* fp = reinterpret_cast(&jlfloatnull); - value = *fp; - } - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - uint64_t jldoublenull = joblist::DOUBLENULL; - double* dp = reinterpret_cast(&jldoublenull); - value = *dp; - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int d = joblist::DATENULL; - value = d; - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - long long d = joblist::DATETIMENULL; - value = d; - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - long long d = joblist::TIMENULL; - value = d; - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - long long d = joblist::TIMESTAMPNULL; - value = d; - } - break; - - case execplan::CalpontSystemCatalog::CHAR: - { - std::string charnull; - - if (colType.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - { - //charnull = joblist::CHAR1NULL; - charnull = "\376"; - value = charnull; - } - else if (colType.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) - { - //charnull = joblist::CHAR2NULL; - charnull = "\377\376"; - value = charnull; - } - else if (colType.colWidth <= execplan::CalpontSystemCatalog::FOUR_BYTE) - { - //charnull = joblist::CHAR4NULL; - charnull = "\377\377\377\376"; - value = charnull; - } - else - { - WriteEngine::Token nullToken; - value = nullToken; - } - - } - break; - - case execplan::CalpontSystemCatalog::VARCHAR: - { - std::string charnull; - - if (colType.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - { - //charnull = joblist::CHAR2NULL; - charnull = "\377\376"; - value = charnull; - } - else if (colType.colWidth < execplan::CalpontSystemCatalog::FOUR_BYTE) - { - //charnull = joblist::CHAR4NULL; - charnull = "\377\377\377\376"; - value = charnull; - } - else - { - WriteEngine::Token nullToken; - value = nullToken; - } - - } - break; - - case execplan::CalpontSystemCatalog::VARBINARY: - { - std::string charnull; - - if (colType.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - { - //charnull = joblist::CHAR2NULL; - charnull = "\377\376"; - value = charnull; - } - else if (colType.colWidth < execplan::CalpontSystemCatalog::FOUR_BYTE) - { - //charnull = joblist::CHAR4NULL; - charnull = "\377\377\377\376"; - value = charnull; - } - else - { - WriteEngine::Token nullToken; - value = nullToken; - } - - } - break; - - case execplan::CalpontSystemCatalog::UTINYINT: - { - uint8_t utinyintvalue = joblist::UTINYINTNULL; - value = utinyintvalue; - - } - break; - - case execplan::CalpontSystemCatalog::USMALLINT: - { - uint16_t usmallintvalue = joblist::USMALLINTNULL; - value = usmallintvalue; - } - break; - - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - { - uint32_t uintvalue = joblist::UINTNULL; - value = uintvalue; - } - break; - - case execplan::CalpontSystemCatalog::UBIGINT: - { - uint64_t ubigint = joblist::UBIGINTNULL; - value = ubigint; - } - break; - - default: - throw std::runtime_error("getNullValueForType: unkown column data type"); - break; - - } - - return value; -} bool DDLPackageProcessor::isIndexConstraint(ddlpackage::DDL_CONSTRAINTS type) { diff --git a/dbcon/ddlpackageproc/ddlpackageprocessor.h b/dbcon/ddlpackageproc/ddlpackageprocessor.h index 5714d5cd6..84d50d805 100644 --- a/dbcon/ddlpackageproc/ddlpackageprocessor.h +++ b/dbcon/ddlpackageproc/ddlpackageprocessor.h @@ -418,12 +418,6 @@ protected: */ execplan::CalpontSystemCatalog::ColDataType convertDataType(int dataType); - /** @brief get the null representation for the given column type - * - * @param colType the column type - */ - boost::any getNullValueForType(const execplan::CalpontSystemCatalog::ColType& colType); - /** @brief return a tokenized value for the supplied data value * * @param result the result of the operation diff --git a/dbcon/dmlpackageproc/dmlpackageprocessor.cpp b/dbcon/dmlpackageproc/dmlpackageprocessor.cpp index ad5bd1df0..c268f238a 100644 --- a/dbcon/dmlpackageproc/dmlpackageprocessor.cpp +++ b/dbcon/dmlpackageproc/dmlpackageprocessor.cpp @@ -52,7 +52,6 @@ using namespace joblist; using namespace messageqcpp; #include "tablelockdata.h" #include "exceptclasses.h" -#include "widedecimalutils.h" namespace { diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index 416daeea4..96c7ad75c 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -400,8 +400,8 @@ inline void ArithmeticOperator::execute(IDB_Decimal& result, IDB_Decimal op1, ID case OP_DIV: if (fOperationType.colWidth == datatypes::MAXDECIMALWIDTH) { - if ((datatypes::Decimal::isWideDecimalType(op2.precision) && op2.s128Value == 0) - || (!datatypes::Decimal::isWideDecimalType(op2.precision) && op2.value == 0)) + if ((datatypes::Decimal::isWideDecimalTypeByPrecision(op2.precision) && op2.s128Value == 0) + || (!datatypes::Decimal::isWideDecimalTypeByPrecision(op2.precision) && op2.value == 0)) { isNull = true; break; diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index 02c72c573..133b5f76e 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -31,6 +31,7 @@ using namespace std; #include "messagequeue.h" #include "calpontsystemcatalog.h" +#include "dataconvert.h" #include "ddlpkg.h" #include "expressionparser.h" #include "calpontselectexecutionplan.h" @@ -6091,13 +6092,9 @@ void CalpontSystemCatalog::checkSysCatVer() } CalpontSystemCatalog::ColType::ColType() : - colWidth(0), constraintType(NO_CONSTRAINT), - colDataType(MEDINT), defaultValue(""), colPosition(-1), - scale(0), - precision(-1), compressionType(NO_COMPRESSION), columnOID(0), autoincrement(0), @@ -6108,15 +6105,12 @@ CalpontSystemCatalog::ColType::ColType() : } CalpontSystemCatalog::ColType::ColType(const ColType& rhs) + :TypeHolderStd(rhs) { - colWidth = rhs.colWidth; constraintType = rhs.constraintType; - colDataType = rhs.colDataType; ddn = rhs.ddn; defaultValue = rhs.defaultValue; colPosition = rhs.colPosition; - scale = rhs.scale; - precision = rhs.precision; compressionType = rhs.compressionType; columnOID = rhs.columnOID; autoincrement = rhs.autoincrement; @@ -6125,26 +6119,6 @@ CalpontSystemCatalog::ColType::ColType(const ColType& rhs) cs = rhs.cs; } -CalpontSystemCatalog::ColType& CalpontSystemCatalog::ColType::operator=(const ColType& rhs) -{ - colWidth = rhs.colWidth; - constraintType = rhs.constraintType; - colDataType = rhs.colDataType; - ddn = rhs.ddn; - defaultValue = rhs.defaultValue; - colPosition = rhs.colPosition; - scale = rhs.scale; - precision = rhs.precision; - compressionType = rhs.compressionType; - columnOID = rhs.columnOID; - autoincrement = rhs.autoincrement; - nextvalue = rhs.nextvalue; - charsetNumber = rhs.charsetNumber; - cs = rhs.cs; - - return *this; -} - CHARSET_INFO* CalpontSystemCatalog::ColType::getCharset() { if (!cs) @@ -6170,6 +6144,31 @@ const string CalpontSystemCatalog::ColType::toString() const return output.str(); } + +boost::any +CalpontSystemCatalog::ColType::convertColumnData(const std::string& dataOrig, + bool& bSaturate, + const std::string& timeZone, + bool nulFlag, + bool noRoundup, + bool isUpdate) const +{ + return dataconvert::DataConvert::convertColumnData(colDataType, *this, + dataOrig, bSaturate, timeZone, + nulFlag, noRoundup, isUpdate); +} + + +CalpontSystemCatalog::ColType CalpontSystemCatalog::ColType::convertUnionColType(vector& types) +{ + idbassert(types.size()); + CalpontSystemCatalog::ColType unionedType = types[0]; + for (uint64_t i = 1; i < types.size(); i++) + dataconvert::DataConvert::joinColTypeForUnion(unionedType, types[i]); + return unionedType; +} + + //format a session id that includes the module id //we want the top bit clear to use as a syscat flag, then we want 7 bits of module id, then 24 bits of thread id /*static*/ diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index 2efd7cef1..29113afc8 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -44,54 +44,13 @@ #include "bytestream.h" #include "joblisttypes.h" #include "stdexcept" -#include "widedecimalutils.h" #undef min #undef max -// Because including my_sys.h in a Columnstore header causes too many conflicts -struct charset_info_st; -typedef const struct charset_info_st CHARSET_INFO; +#include "mcs_datatype.h" -#ifdef _MSC_VER -#define __attribute__(x) -#endif - -namespace -{ -const int64_t MIN_TINYINT __attribute__ ((unused)) = std::numeric_limits::min() + 2; //-126; -const int64_t MAX_TINYINT __attribute__ ((unused)) = std::numeric_limits::max(); //127; -const int64_t MIN_SMALLINT __attribute__ ((unused)) = std::numeric_limits::min() + 2; //-32766; -const int64_t MAX_SMALLINT __attribute__ ((unused)) = std::numeric_limits::max(); //32767; -const int64_t MIN_MEDINT __attribute__ ((unused)) = -(1ULL << 23); //-8388608; -const int64_t MAX_MEDINT __attribute__ ((unused)) = (1ULL << 23) - 1; //8388607; -const int64_t MIN_INT __attribute__ ((unused)) = std::numeric_limits::min() + 2; //-2147483646; -const int64_t MAX_INT __attribute__ ((unused)) = std::numeric_limits::max(); //2147483647; -const int64_t MIN_BIGINT __attribute__ ((unused)) = std::numeric_limits::min() + 2; //-9223372036854775806LL; -const int64_t MAX_BIGINT __attribute__ ((unused)) = std::numeric_limits::max(); //9223372036854775807 - -const uint64_t MIN_UINT __attribute__ ((unused)) = 0; -const uint64_t MIN_UTINYINT __attribute__ ((unused)) = 0; -const uint64_t MIN_USMALLINT __attribute__ ((unused)) = 0; -const uint64_t MIN_UMEDINT __attribute__ ((unused)) = 0; -const uint64_t MIN_UBIGINT __attribute__ ((unused)) = 0; -const uint64_t MAX_UINT __attribute__ ((unused)) = std::numeric_limits::max() - 2; //4294967293 -const uint64_t MAX_UTINYINT __attribute__ ((unused)) = std::numeric_limits::max() - 2; //253; -const uint64_t MAX_USMALLINT __attribute__ ((unused)) = std::numeric_limits::max() - 2; //65533; -const uint64_t MAX_UMEDINT __attribute__ ((unused)) = (1ULL << 24) - 1; //16777215 -const uint64_t MAX_UBIGINT __attribute__ ((unused)) = std::numeric_limits::max() - 2; //18446744073709551613 - -const float MAX_FLOAT __attribute__ ((unused)) = std::numeric_limits::max(); //3.402823466385289e+38 -const float MIN_FLOAT __attribute__ ((unused)) = -std::numeric_limits::max(); -const double MAX_DOUBLE __attribute__ ((unused)) = std::numeric_limits::max(); //1.7976931348623157e+308 -const double MIN_DOUBLE __attribute__ ((unused)) = -std::numeric_limits::max(); -const long double MAX_LONGDOUBLE __attribute__ ((unused)) = std::numeric_limits::max(); //1.7976931348623157e+308 -const long double MIN_LONGDOUBLE __attribute__ ((unused)) = -std::numeric_limits::max(); - - -const uint64_t AUTOINCR_SATURATED __attribute__ ((unused)) = std::numeric_limits::max(); -} class ExecPlanTest; namespace messageqcpp @@ -99,6 +58,7 @@ namespace messageqcpp class MessageQueueClient; } + // This is now set in the Columnstore.xml file // Use, e.g., 0x500 for uid 500 so it is easy to spot in ipcs list //const int32_t BRM_UID = 0x0; @@ -117,7 +77,7 @@ const int32_t IDB_VTABLE_ID = CNX_VTABLE_ID; * * This object encapsulates the system catalog stored in the engine */ -class CalpontSystemCatalog +class CalpontSystemCatalog: public datatypes::SystemCatalog { public: @@ -131,50 +91,6 @@ public: */ enum Identity { EC = 0, FE }; - /** the set of Calpont column widths - * - */ - enum ColWidth { ONE_BIT, ONE_BYTE, TWO_BYTE, THREE_BYTE, FOUR_BYTE, FIVE_BYTE, SIX_BYTE, SEVEN_BYTE, EIGHT_BYTE }; - - /** the set of Calpont column data types - * - */ - enum ColDataType - { - BIT, /*!< BIT type */ - TINYINT, /*!< TINYINT type */ - CHAR, /*!< CHAR type */ - SMALLINT, /*!< SMALLINT type */ - DECIMAL, /*!< DECIMAL type */ - MEDINT, /*!< MEDINT type */ - INT, /*!< INT type */ - FLOAT, /*!< FLOAT type */ - DATE, /*!< DATE type */ - BIGINT, /*!< BIGINT type */ - DOUBLE, /*!< DOUBLE type */ - DATETIME, /*!< DATETIME type */ - VARCHAR, /*!< VARCHAR type */ - VARBINARY, /*!< VARBINARY type */ - CLOB, /*!< CLOB type */ - BLOB, /*!< BLOB type */ - UTINYINT, /*!< Unsigned TINYINT type */ - USMALLINT, /*!< Unsigned SMALLINT type */ - UDECIMAL, /*!< Unsigned DECIMAL type */ - UMEDINT, /*!< Unsigned MEDINT type */ - UINT, /*!< Unsigned INT type */ - UFLOAT, /*!< Unsigned FLOAT type */ - UBIGINT, /*!< Unsigned BIGINT type */ - UDOUBLE, /*!< Unsigned DOUBLE type */ - TEXT, /*!< TEXT type */ - TIME, /*!< TIME type */ - TIMESTAMP, /*!< TIMESTAMP type */ - NUM_OF_COL_DATA_TYPE, /* NEW TYPES ABOVE HERE */ - LONGDOUBLE, /* @bug3241, dev and variance calculation only */ - STRINT, /* @bug3532, string as int for fast comparison */ - UNDEFINED, /*!< Undefined - used in UDAF API */ - BINARY, /*!< BINARY type */ - }; - /** the set of column constraint types * */ @@ -280,21 +196,18 @@ public: */ typedef std::vector DictOIDList; + /** the structure returned by colType * * defaultValue is only meaningful when constraintType == DEFAULT_CONSTRAINT */ - struct ColType + struct ColType: public datatypes::SystemCatalog::TypeHolderStd { ColType(); - int32_t colWidth; ConstraintType constraintType; - ColDataType colDataType; DictOID ddn; std::string defaultValue; int32_t colPosition; // temporally put here. may need to have ColInfo struct later - int32_t scale; //number after decimal points - int32_t precision; int32_t compressionType; OID columnOID; bool autoincrement; //set to true if SYSCOLUMN autoincrement is �y� @@ -329,6 +242,13 @@ public: b >> charsetNumber; } + boost::any convertColumnData(const std::string& dataOrig, + bool& bSaturate, + const std::string& timeZone, + bool nulFlag = false, + bool noRoundup = false, + bool isUpdate = false) const; + const std::string toString() const; //Put these here so udf doesn't need to link libexecplan @@ -364,6 +284,8 @@ public: return !(*this == t); } + static ColType convertUnionColType(std::vector&); + }; /** the structure of a table infomation @@ -964,91 +886,9 @@ const CalpontSystemCatalog::TableAliasName make_aliastable(const std::string& s, const CalpontSystemCatalog::TableAliasName make_aliasview(const std::string& s, const std::string& t, const std::string& a, const std::string& v, const bool fisColumnStore = true, int lower_case_table_names=0); -/** convenience function to determine if column type is a char - * type - */ -inline bool isCharType(const execplan::CalpontSystemCatalog::ColDataType type) -{ - return (execplan::CalpontSystemCatalog::VARCHAR == type || - execplan::CalpontSystemCatalog::CHAR == type || - execplan::CalpontSystemCatalog::BLOB == type || - execplan::CalpontSystemCatalog::TEXT == type); -} - -/** convenience function to determine if column type is a - * numeric type - */ -inline bool isNumeric(const execplan::CalpontSystemCatalog::ColDataType type) -{ - switch (type) - { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - case execplan::CalpontSystemCatalog::UFLOAT: - case execplan::CalpontSystemCatalog::UDOUBLE: - case execplan::CalpontSystemCatalog::UDECIMAL: - return true; - - default: - return false; - } -} - -inline bool isDecimal(const execplan::CalpontSystemCatalog::ColDataType type) -{ - return (type == execplan::CalpontSystemCatalog::DECIMAL || - type == execplan::CalpontSystemCatalog::UDECIMAL); -} - -/** convenience function to determine if column type is an - * unsigned type - */ -inline bool isUnsigned(const execplan::CalpontSystemCatalog::ColDataType type) -{ - switch (type) - { - case execplan::CalpontSystemCatalog::UTINYINT: - case execplan::CalpontSystemCatalog::USMALLINT: - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - case execplan::CalpontSystemCatalog::UBIGINT: - return true; - - default: - return false; - } -} - -inline bool isSignedInteger(const execplan::CalpontSystemCatalog::ColDataType type) -{ - switch (type) - { - case execplan::CalpontSystemCatalog::TINYINT: - case execplan::CalpontSystemCatalog::SMALLINT: - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - case execplan::CalpontSystemCatalog::BIGINT: - return true; - - default: - return false; - } -} - inline bool isNull(int128_t val, const execplan::CalpontSystemCatalog::ColType& ct) { - return utils::isWideDecimalNullValue(val); + return datatypes::Decimal::isWideDecimalNullValue(val); } inline bool isNull(int64_t val, const execplan::CalpontSystemCatalog::ColType& ct) diff --git a/dbcon/execplan/functioncolumn.h b/dbcon/execplan/functioncolumn.h index 874e7ef67..cda618b85 100644 --- a/dbcon/execplan/functioncolumn.h +++ b/dbcon/execplan/functioncolumn.h @@ -253,7 +253,7 @@ public: if (LIKELY(fResultType.colWidth == datatypes::MAXDECIMALWIDTH)) { decimal.s128Value = - (datatypes::Decimal::isWideDecimalType(decimal.precision)) ? + (datatypes::Decimal::isWideDecimalTypeByPrecision(decimal.precision)) ? decimal.s128Value : decimal.value; int128_t scaleMultiplier; diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index b23024990..ad35f1768 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -57,290 +57,13 @@ namespace execplan typedef execplan::CalpontSystemCatalog::ColType Type; -/** - * @brief IDB_Decimal type - * - */ -struct IDB_Decimal +class IDB_Decimal: public datatypes::VDecimal { - IDB_Decimal(): s128Value(0), value(0), scale(0), precision(0) - { - } - - IDB_Decimal(int64_t val, int8_t s, uint8_t p, int128_t val128 = 0) : - s128Value(val128), - value(val), - scale(s), - precision(p) - { - } - - int decimalComp(const IDB_Decimal& d) const - { - lldiv_t d1 = lldiv(value, IDB_pow[scale]); - lldiv_t d2 = lldiv(d.value, IDB_pow[d.scale]); - - int ret = 0; - - if (d1.quot > d2.quot) - { - ret = 1; - } - else if (d1.quot < d2.quot) - { - ret = -1; - } - else - { - // rem carries the value's sign, but needs to be normalized. - int64_t s = scale - d.scale; - - if (s < 0) - { - if ((d1.rem * IDB_pow[-s]) > d2.rem) - ret = 1; - else if ((d1.rem * IDB_pow[-s]) < d2.rem) - ret = -1; - } - else - { - if (d1.rem > (d2.rem * IDB_pow[s])) - ret = 1; - else if (d1.rem < (d2.rem * IDB_pow[s])) - ret = -1; - } - } - - return ret; - } - - bool operator==(const IDB_Decimal& rhs) const - { - if (precision > datatypes::INT64MAXPRECISION && - rhs.precision > datatypes::INT64MAXPRECISION) - { - if (scale == rhs.scale) - return s128Value == rhs.s128Value; - else - return (datatypes::Decimal::compare(*this, rhs) == 0); - } - else if (precision > datatypes::INT64MAXPRECISION && - rhs.precision <= datatypes::INT64MAXPRECISION) - { - const_cast(rhs).s128Value = rhs.value; - - if (scale == rhs.scale) - return s128Value == rhs.s128Value; - else - return (datatypes::Decimal::compare(*this, rhs) == 0); - } - else if (precision <= datatypes::INT64MAXPRECISION && - rhs.precision > datatypes::INT64MAXPRECISION) - { - if (scale == rhs.scale) - return (int128_t) value == rhs.s128Value; - else - return (datatypes::Decimal::compare(IDB_Decimal(0, scale, precision, (int128_t) value), rhs) == 0); - } - else - { - if (scale == rhs.scale) - return value == rhs.value; - else - return (decimalComp(rhs) == 0); - } - } - - bool operator>(const IDB_Decimal& rhs) const - { - if (precision > datatypes::INT64MAXPRECISION && - rhs.precision > datatypes::INT64MAXPRECISION) - { - if (scale == rhs.scale) - return s128Value > rhs.s128Value; - else - return (datatypes::Decimal::compare(*this, rhs) > 0); - } - else if (precision > datatypes::INT64MAXPRECISION && - rhs.precision <= datatypes::INT64MAXPRECISION) - { - IDB_Decimal rhstmp(0, rhs.scale, rhs.precision, (int128_t) rhs.value); - - if (scale == rhstmp.scale) - return s128Value > rhstmp.s128Value; - else - return (datatypes::Decimal::compare(*this, rhstmp) > 0); - } - else if (precision <= datatypes::INT64MAXPRECISION && - rhs.precision > datatypes::INT64MAXPRECISION) - { - if (scale == rhs.scale) - return (int128_t) value > rhs.s128Value; - else - return (datatypes::Decimal::compare(IDB_Decimal(0, scale, precision, (int128_t) value), rhs) > 0); - } - else - { - if (scale == rhs.scale) - return value > rhs.value; - else - return (decimalComp(rhs) > 0); - } - } - - bool operator<(const IDB_Decimal& rhs) const - { - if (precision > datatypes::INT64MAXPRECISION && - rhs.precision > datatypes::INT64MAXPRECISION) - { - if (scale == rhs.scale) - return s128Value < rhs.s128Value; - else - return (datatypes::Decimal::compare(*this, rhs) < 0); - } - else if (precision > datatypes::INT64MAXPRECISION && - rhs.precision <= datatypes::INT64MAXPRECISION) - { - IDB_Decimal rhstmp(0, rhs.scale, rhs.precision, (int128_t) rhs.value); - - if (scale == rhstmp.scale) - return s128Value < rhstmp.s128Value; - else - return (datatypes::Decimal::compare(*this, rhstmp) < 0); - } - else if (precision <= datatypes::INT64MAXPRECISION && - rhs.precision > datatypes::INT64MAXPRECISION) - { - if (scale == rhs.scale) - return (int128_t) value < rhs.s128Value; - else - return (datatypes::Decimal::compare(IDB_Decimal(0, scale, precision, (int128_t) value), rhs) < 0); - } - else - { - if (scale == rhs.scale) - return value < rhs.value; - else - return (decimalComp(rhs) < 0); - } - } - - bool operator>=(const IDB_Decimal& rhs) const - { - if (precision > datatypes::INT64MAXPRECISION && - rhs.precision > datatypes::INT64MAXPRECISION) - { - if (scale == rhs.scale) - return s128Value >= rhs.s128Value; - else - return (datatypes::Decimal::compare(*this, rhs) >= 0); - } - else if (precision > datatypes::INT64MAXPRECISION && - rhs.precision <= datatypes::INT64MAXPRECISION) - { - IDB_Decimal rhstmp(0, rhs.scale, rhs.precision, (int128_t) rhs.value); - - if (scale == rhstmp.scale) - return s128Value >= rhstmp.s128Value; - else - return (datatypes::Decimal::compare(*this, rhstmp) >= 0); - } - else if (precision <= datatypes::INT64MAXPRECISION && - rhs.precision > datatypes::INT64MAXPRECISION) - { - if (scale == rhs.scale) - return (int128_t) value >= rhs.s128Value; - else - return (datatypes::Decimal::compare(IDB_Decimal(0, scale, precision, (int128_t) value), rhs) >= 0); - } - else - { - if (scale == rhs.scale) - return value >= rhs.value; - else - return (decimalComp(rhs) >= 0); - } - } - - bool operator<=(const IDB_Decimal& rhs) const - { - if (precision > datatypes::INT64MAXPRECISION && - rhs.precision > datatypes::INT64MAXPRECISION) - { - if (scale == rhs.scale) - return s128Value <= rhs.s128Value; - else - return (datatypes::Decimal::compare(*this, rhs) <= 0); - } - else if (precision > datatypes::INT64MAXPRECISION && - rhs.precision <= datatypes::INT64MAXPRECISION) - { - IDB_Decimal rhstmp(0, rhs.scale, rhs.precision, (int128_t) rhs.value); - - if (scale == rhstmp.scale) - return s128Value <= rhstmp.s128Value; - else - return (datatypes::Decimal::compare(*this, rhstmp) <= 0); - } - else if (precision <= datatypes::INT64MAXPRECISION && - rhs.precision > datatypes::INT64MAXPRECISION) - { - if (scale == rhs.scale) - return (int128_t) value <= rhs.s128Value; - else - return (datatypes::Decimal::compare(IDB_Decimal(0, scale, precision, (int128_t) value), rhs) <= 0); - } - else - { - if (scale == rhs.scale) - return value <= rhs.value; - else - return (decimalComp(rhs) <= 0); - } - } - - bool operator!=(const IDB_Decimal& rhs) const - { - if (precision > datatypes::INT64MAXPRECISION && - rhs.precision > datatypes::INT64MAXPRECISION) - { - if (scale == rhs.scale) - return s128Value != rhs.s128Value; - else - return (datatypes::Decimal::compare(*this, rhs) != 0); - } - else if (precision > datatypes::INT64MAXPRECISION && - rhs.precision <= datatypes::INT64MAXPRECISION) - { - IDB_Decimal rhstmp(0, rhs.scale, rhs.precision, (int128_t) rhs.value); - - if (scale == rhstmp.scale) - return s128Value != rhstmp.s128Value; - else - return (datatypes::Decimal::compare(*this, rhstmp) != 0); - } - else if (precision <= datatypes::INT64MAXPRECISION && - rhs.precision > datatypes::INT64MAXPRECISION) - { - if (scale == rhs.scale) - return (int128_t) value != rhs.s128Value; - else - return (datatypes::Decimal::compare(IDB_Decimal(0, scale, precision, (int128_t) value), rhs) != 0); - } - else - { - if (scale == rhs.scale) - return value != rhs.value; - else - return (decimalComp(rhs) != 0); - } - } - - int128_t s128Value; - int64_t value; - int8_t scale; // 0~38 - uint8_t precision; // 1~38 +public: + using datatypes::VDecimal::VDecimal; }; + + typedef IDB_Decimal CNX_Decimal; /** @@ -900,7 +623,7 @@ inline const std::string& TreeNode::getStrVal(const std::string& timeZone) case CalpontSystemCatalog::UDECIMAL: { if (fResultType.colWidth == datatypes::MAXDECIMALWIDTH) - dataconvert::DataConvert::decimalToString(&fResult.decimalVal.s128Value, fResult.decimalVal.scale, tmp, utils::MAXLENGTH16BYTES, fResultType.colDataType); + dataconvert::DataConvert::decimalToString(&fResult.decimalVal.s128Value, fResult.decimalVal.scale, tmp, datatypes::Decimal::MAXLENGTH16BYTES, fResultType.colDataType); else dataconvert::DataConvert::decimalToString(fResult.decimalVal.value, fResult.decimalVal.scale, tmp, 22, fResultType.colDataType); fResult.strVal = std::string(tmp); diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.cpp b/dbcon/joblist/batchprimitiveprocessor-jl.cpp index 2a874c4e2..929252d56 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.cpp +++ b/dbcon/joblist/batchprimitiveprocessor-jl.cpp @@ -773,7 +773,7 @@ void BatchPrimitiveProcessorJL::getRowGroupData(ByteStream& in, vector* if (UNLIKELY(*hasWideColumn)) { idbassert(colType.colWidth > utils::MAXLEGACYWIDTH); - if (LIKELY(datatypes::Decimal::isWideDecimalType(colType))) + if (LIKELY(colType.isWideDecimalType())) { in >> tmp128; *min = tmp128; diff --git a/dbcon/joblist/columncommand-jl.cpp b/dbcon/joblist/columncommand-jl.cpp index b8ef18f5d..66b3acb2e 100644 --- a/dbcon/joblist/columncommand-jl.cpp +++ b/dbcon/joblist/columncommand-jl.cpp @@ -288,7 +288,7 @@ string ColumnCommandJL::toString() if (isDict()) ret << " (tokens)"; - else if (execplan::isCharType(colType.colDataType)) + else if (datatypes::isCharType(colType.colDataType)) ret << " (is char)"; return ret.str(); diff --git a/dbcon/joblist/crossenginestep.cpp b/dbcon/joblist/crossenginestep.cpp index 455906c0f..bbdefce0e 100644 --- a/dbcon/joblist/crossenginestep.cpp +++ b/dbcon/joblist/crossenginestep.cpp @@ -46,9 +46,6 @@ using namespace execplan; #include "rowgroup.h" using namespace rowgroup; -#include "dataconvert.h" -using namespace dataconvert; - #include "querytele.h" using namespace querytele; @@ -247,7 +244,7 @@ int64_t CrossEngineStep::convertValueNum( // bool nulFlag, // bool noRoundup ) bool pushWarning = false; - boost::any anyVal = DataConvert::convertColumnData(ct, str, pushWarning, fTimeZone, false, true, false); + boost::any anyVal = ct.convertColumnData(str, pushWarning, fTimeZone, false, true, false); // Out of range values are treated as NULL as discussed during design review. if (pushWarning) diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index 78fdf24c2..b5ba3390f 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -381,7 +381,7 @@ void GroupConcatAgUM::applyMapping(const boost::shared_array& mapping, cons { fRow.setLongDoubleField(row.getLongDoubleField(mapping[i]), i); } - else if (datatypes::Decimal::isWideDecimalType(fRow.getColType(i), fRow.getColumnWidth(i))) + else if (datatypes::isWideDecimalType(fRow.getColType(i), fRow.getColumnWidth(i))) { row.copyBinaryField(fRow, i, mapping[i]); } @@ -460,12 +460,12 @@ void GroupConcator::outputRow(std::ostringstream& oss, const rowgroup::Row& row) if (LIKELY(row.getColumnWidth(*i) == datatypes::MAXDECIMALWIDTH)) { - char buf[utils::MAXLENGTH16BYTES]; + char buf[datatypes::Decimal::MAXLENGTH16BYTES]; int128_t* dec = row.getBinaryField(*i); dataconvert::DataConvert::decimalToString(dec, static_cast(scale), buf, - sizeof(buf), types[*i]); + (uint8_t) sizeof(buf), types[*i]); oss << fixed << buf; } else diff --git a/dbcon/joblist/jlf_common.cpp b/dbcon/joblist/jlf_common.cpp index c07ab056e..794ed4c07 100644 --- a/dbcon/joblist/jlf_common.cpp +++ b/dbcon/joblist/jlf_common.cpp @@ -333,7 +333,7 @@ string extractTableAlias(const SSC& sc) //------------------------------------------------------------------------------ CalpontSystemCatalog::OID isDictCol(const CalpontSystemCatalog::ColType& colType) { - if (datatypes::Decimal::isWideDecimalType(colType) || + if (colType.isWideDecimalType() || colType.colDataType == CalpontSystemCatalog::BINARY) return 0; if (colType.colWidth > 8) return colType.ddn.dictOID; diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index a184fbe24..42bbf8904 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -65,9 +65,6 @@ namespace ba = boost::algorithm; #include "simplescalarfilter.h" using namespace execplan; -#include "dataconvert.h" -using namespace dataconvert; - #include "configcpp.h" using namespace config; @@ -135,7 +132,7 @@ void valueNullNum(const CalpontSystemCatalog::ColType& ct, const string& timeZon { T& n = val; bool pushWarning = false; - boost::any anyVal = DataConvert::convertColumnData(ct, "", pushWarning, timeZone, true, false, false); + boost::any anyVal = ct.convertColumnData("", pushWarning, timeZone, true, false, false); switch (ct.colDataType) { @@ -324,7 +321,7 @@ void convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct, v = 0; rf = 0; bool pushWarning = false; - boost::any anyVal = DataConvert::convertColumnData(ct, str, pushWarning, timeZone, false, true, false); + boost::any anyVal = ct.convertColumnData(str, pushWarning, timeZone, false, true, false); switch (ct.colDataType) { @@ -1896,7 +1893,7 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) // WIP MCOL-641 width check must be a f() not a literal // make a template from convertValueNum to avoid extra if // this condition doesn't support UDECIMAL - if (datatypes::Decimal::isWideDecimalType(ct)) + if (ct.isWideDecimalType()) convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value128); else convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value); @@ -1933,7 +1930,7 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo) if (sc->isColumnStore()) { - if (datatypes::Decimal::isWideDecimalType(ct)) + if (ct.isWideDecimalType()) pcs->addFilter(cop, value128, rf); else pcs->addFilter(cop, value, rf); @@ -3011,7 +3008,7 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo) uint8_t rf = 0; bool isNull = ConstantColumn::NULLDATA == cc->type(); - if (datatypes::Decimal::isWideDecimalType(ct)) + if (ct.isWideDecimalType()) convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value128); else convertValueNum(constval, ct, isNull, rf, jobInfo.timeZone, value); @@ -3031,7 +3028,7 @@ const JobStepVector doConstantFilter(const ConstantFilter* cf, JobInfo& jobInfo) if (ConstantColumn::NULLDATA == cc->type() && (opeq == *sop || opne == *sop)) cop = COMPARE_NIL; - if (datatypes::Decimal::isWideDecimalType(ct)) + if (ct.isWideDecimalType()) pcs->addFilter(cop, value128, rf); else pcs->addFilter(cop, value, rf); diff --git a/dbcon/joblist/jlf_tuplejoblist.cpp b/dbcon/joblist/jlf_tuplejoblist.cpp index 5f7ea2dff..ddab457cc 100644 --- a/dbcon/joblist/jlf_tuplejoblist.cpp +++ b/dbcon/joblist/jlf_tuplejoblist.cpp @@ -4072,7 +4072,7 @@ SJSTEP unionQueries(JobStepVector& queries, uint64_t distinctUnionNum, JobInfo& // get unioned column types for (uint64_t j = 0; j < colCount; ++j) { - CalpontSystemCatalog::ColType colType = DataConvert::convertUnionColType(queryColTypes[j]); + CalpontSystemCatalog::ColType colType = CalpontSystemCatalog::ColType::convertUnionColType(queryColTypes[j]); types.push_back(colType.colDataType); csNums.push_back(colType.charsetNumber); scale.push_back(colType.scale); diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index 79d053d9a..4f2affc89 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -28,7 +28,6 @@ #include "brm.h" #include "brmtypes.h" #include "dataconvert.h" -#include "widedecimalutils.h" #include "mcs_decimal.h" #define IS_VERBOSE (fDebug >= 4) @@ -415,7 +414,7 @@ void LBIDList::UpdateMinMax(T min, T max, int64_t lbid, CalpontSystemCatalog::Co if (mmp->isValid == BRM::CP_INVALID) { - if (execplan::isCharType(type)) + if (datatypes::isCharType(type)) { if (order_swap(min) < order_swap(mmp->min) || mmp->min == numeric_limits::max()) @@ -425,7 +424,7 @@ void LBIDList::UpdateMinMax(T min, T max, int64_t lbid, CalpontSystemCatalog::Co mmp->max == numeric_limits::min()) mmp->max = max; } - else if (execplan::isUnsigned(type)) + else if (datatypes::isUnsigned(type)) { if (typeid(T) == typeid(__int128)) { @@ -757,8 +756,8 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, bool scan = true; int64_t value = 0; __int128 bigValue = 0; - bool bIsUnsigned = execplan::isUnsigned(ct.colDataType); - bool bIsChar = execplan::isCharType(ct.colDataType); + bool bIsUnsigned = datatypes::isUnsigned(ct.colDataType); + bool bIsChar = datatypes::isCharType(ct.colDataType); for (int i = 0; i < NOPS; i++) { @@ -859,7 +858,7 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, // Should we also check for empty here? // TODO MCOL-641 - if (datatypes::Decimal::isWideDecimalType(ct)) + if (ct.isWideDecimalType()) { if (isNull(bigValue, ct)) continue; diff --git a/dbcon/joblist/pseudocc-jl.cpp b/dbcon/joblist/pseudocc-jl.cpp index 94e7cb574..7a48fb096 100644 --- a/dbcon/joblist/pseudocc-jl.cpp +++ b/dbcon/joblist/pseudocc-jl.cpp @@ -46,7 +46,7 @@ void PseudoCCJL::runCommand(ByteStream& bs) const { if (function == PSEUDO_EXTENTMAX) { - if (!datatypes::Decimal::isWideDecimalType(colType)) + if (!colType.isWideDecimalType()) { int64_t max = extents[currentExtentIndex].partition.cprange.hiVal; int64_t min = extents[currentExtentIndex].partition.cprange.loVal; @@ -66,14 +66,14 @@ void PseudoCCJL::runCommand(ByteStream& bs) const else { int128_t int128Null; - utils::setWideDecimalNullValue(int128Null); + datatypes::Decimal::setWideDecimalNullValue(int128Null); bs << (uint128_t) int128Null; } } } else if (function == PSEUDO_EXTENTMIN) { - if (!datatypes::Decimal::isWideDecimalType(colType)) + if (!colType.isWideDecimalType()) { int64_t max = extents[currentExtentIndex].partition.cprange.hiVal; int64_t min = extents[currentExtentIndex].partition.cprange.loVal; @@ -93,7 +93,7 @@ void PseudoCCJL::runCommand(ByteStream& bs) const else { int128_t int128Null; - utils::setWideDecimalNullValue(int128Null); + datatypes::Decimal::setWideDecimalNullValue(int128Null); bs << (uint128_t) int128Null; } } diff --git a/dbcon/joblist/rowestimator.cpp b/dbcon/joblist/rowestimator.cpp index c28e00781..95ae57140 100644 --- a/dbcon/joblist/rowestimator.cpp +++ b/dbcon/joblist/rowestimator.cpp @@ -224,7 +224,7 @@ float RowEstimator::estimateOpFactor(const T& min, const T& max, const T& value, case COMPARE_NGE: if (cpStatus == BRM::CP_VALID) { - if (!datatypes::Decimal::isWideDecimalType(ct)) + if (!ct.isWideDecimalType()) factor = (1.0 * value - min) / (max - min + 1); else factor = ((__float128) value - min) / (max - min + 1); @@ -236,7 +236,7 @@ float RowEstimator::estimateOpFactor(const T& min, const T& max, const T& value, case COMPARE_NGT: if (cpStatus == BRM::CP_VALID) { - if (!datatypes::Decimal::isWideDecimalType(ct)) + if (!ct.isWideDecimalType()) factor = (1.0 * value - min + 1) / (max - min + 1); else factor = ((__float128) value - min + 1) / (max - min + 1); @@ -248,7 +248,7 @@ float RowEstimator::estimateOpFactor(const T& min, const T& max, const T& value, case COMPARE_NLE: if (cpStatus == BRM::CP_VALID) { - if (!datatypes::Decimal::isWideDecimalType(ct)) + if (!ct.isWideDecimalType()) factor = (1.0 * max - value) / (1.0 * max - min + 1); else factor = ((__float128) max - value) / (max - min + 1); @@ -261,7 +261,7 @@ float RowEstimator::estimateOpFactor(const T& min, const T& max, const T& value, if (cpStatus == BRM::CP_VALID) { // TODO: Best way to convert to floating point arithmetic? - if (!datatypes::Decimal::isWideDecimalType(ct)) + if (!ct.isWideDecimalType()) factor = (1.0 * max - value + 1) / (max - min + 1); else factor = ((__float128) max - value + 1) / (max - min + 1); @@ -299,7 +299,7 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, const uint8_t BOP, const uint32_t& rowsInExtent) { - bool bIsUnsigned = execplan::isUnsigned(ct.colDataType); + bool bIsUnsigned = datatypes::isUnsigned(ct.colDataType); float factor = 1.0; float tempFactor = 1.0; @@ -308,7 +308,7 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, uint32_t distinctValuesEstimate; // Adjust values based on column type and estimate the - if (!datatypes::Decimal::isWideDecimalType(ct)) + if (!ct.isWideDecimalType()) { adjustedMin = adjustValue(ct, emEntry.partition.cprange.loVal); adjustedMax = adjustValue(ct, emEntry.partition.cprange.hiVal); @@ -460,7 +460,7 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, // Get the factor for the individual operation. if (bIsUnsigned) { - if (!datatypes::Decimal::isWideDecimalType(ct)) + if (!ct.isWideDecimalType()) { tempFactor = estimateOpFactor( adjustedMin, adjustedMax, adjustValue(ct, value), op, lcf, @@ -475,7 +475,7 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, } else { - if (!datatypes::Decimal::isWideDecimalType(ct)) + if (!ct.isWideDecimalType()) { tempFactor = estimateOpFactor( adjustedMin, adjustedMax, adjustValue(ct, value), op, lcf, diff --git a/dbcon/joblist/tuple-bps.cpp b/dbcon/joblist/tuple-bps.cpp index dcdc2f5f1..44162ba6e 100644 --- a/dbcon/joblist/tuple-bps.cpp +++ b/dbcon/joblist/tuple-bps.cpp @@ -1693,11 +1693,11 @@ bool TupleBPS::processPseudoColFilters(uint32_t extentIndex, boost::shared_ptrresultType(); - if (datatypes::Decimal::isWideDecimalType(colType)) + if (colType.isWideDecimalType()) { *scaleIter = colType.scale; *precisionIter = colType.precision; @@ -776,7 +776,7 @@ void TupleAggregateStep::configDeliveredRowGroup(const JobInfo& jobInfo) { const auto& colType = jobInfo.nonConstDelCols[i]->resultType(); - if (datatypes::Decimal::isWideDecimalType(colType)) + if (colType.isWideDecimalType()) { *scaleIter = colType.scale; *precisionIter = colType.precision; diff --git a/dbcon/joblist/tuplehashjoin.cpp b/dbcon/joblist/tuplehashjoin.cpp index 35639814a..6b616c083 100644 --- a/dbcon/joblist/tuplehashjoin.cpp +++ b/dbcon/joblist/tuplehashjoin.cpp @@ -487,7 +487,7 @@ void TupleHashJoinStep::forwardCPData() continue; bool isSmallSideWideDecimal = - datatypes::Decimal::isWideDecimalType(smallRGs[i].getColType(idx), smallRGs[i].getColumnWidth(idx)); + datatypes::isWideDecimalType(smallRGs[i].getColType(idx), smallRGs[i].getColumnWidth(idx)); largeBPS->addCPPredicates(largeRG.getOIDs()[joiners[i]->getLargeKeyColumns()[col]], joiners[i]->getCPData()[col], !joiners[i]->discreteCPValues()[col], diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt index 491a59323..7274ed525 100644 --- a/dbcon/mysql/CMakeLists.txt +++ b/dbcon/mysql/CMakeLists.txt @@ -7,6 +7,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} SET ( libcalmysql_SRCS + ../../datatypes/mcs_datatype.cpp ha_mcs_sysvars.cpp ha_mcs_client_udfs.cpp ha_mcs_opt_rewrites.cpp diff --git a/dbcon/mysql/ha_mcs_datatype.h b/dbcon/mysql/ha_mcs_datatype.h new file mode 100644 index 000000000..56ffd7cba --- /dev/null +++ b/dbcon/mysql/ha_mcs_datatype.h @@ -0,0 +1,1301 @@ +/* + Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ +#ifndef HA_MCS_DATATYPE_H_INCLUDED +#define HA_MCS_DATATYPE_H_INCLUDED + +/* + Interface classes for MariaDB data types (e.g. Field) for TypeHandler. + These classes are needed to avoid TypeHandler dependency on + MariaDB header files. +*/ + + +namespace datatypes { + +class StoreFieldMariaDB: public StoreField +{ + Field *m_field; + const CalpontSystemCatalog::ColType &m_type; +public: + StoreFieldMariaDB(Field *f, CalpontSystemCatalog::ColType &type) + :m_field(f), m_type(type) + { } + const CalpontSystemCatalog::ColType &type() const { return m_type; } + int32_t colWidth() const override { return m_type.colWidth; } + int32_t precision() const override { return m_type.precision; } + + int store_date(int64_t val) override + { + char tmp[256]; + DataConvert::dateToString(val, tmp, sizeof(tmp)-1); + return store_string(tmp, strlen(tmp)); + } + + int store_datetime(int64_t val) override + { + char tmp[256]; + DataConvert::datetimeToString(val, tmp, sizeof(tmp)-1, m_type.precision); + return store_string(tmp, strlen(tmp)); + } + + int store_time(int64_t val) override + { + char tmp[256]; + DataConvert::timeToString(val, tmp, sizeof(tmp)-1, m_type.precision); + return store_string(tmp, strlen(tmp)); + } + + int store_timestamp(int64_t val) override + { + char tmp[256]; + DataConvert::timestampToString(val, tmp, sizeof(tmp), + current_thd->variables.time_zone->get_name()->ptr(), + m_type.precision); + return store_string(tmp, strlen(tmp)); + } + + int store_string(const char *str, size_t length) override + { + return m_field->store(str, length, m_field->charset()); + } + int store_varbinary(const char *str, size_t length) override + { + if (get_varbin_always_hex(current_thd)) + { + size_t ll = length * 2; + boost::scoped_array sca(new char[ll]); + ConstString(str, length).bin2hex(sca.get()); + return m_field->store_binary(sca.get(), ll); + } + return m_field->store_binary(str, length); + } + + int store_xlonglong(int64_t val) override + { + idbassert(dynamic_cast(m_field)); + return m_field->store(val, static_cast(m_field)->unsigned_flag); + } + + int store_float(float dl) override + { + if (dl == std::numeric_limits::infinity()) + { + m_field->set_null(); + return 1; + } + + // bug 3485, reserve enough space for the longest float value + // -3.402823466E+38 to -1.175494351E-38, 0, and + // 1.175494351E-38 to 3.402823466E+38. + m_field->field_length = 40; + return m_field->store(dl); + } + + int store_double(double dl) override + { + if (dl == std::numeric_limits::infinity()) + { + m_field->set_null(); + return 1; + } + + if (m_field->type() == MYSQL_TYPE_NEWDECIMAL) + { + char buf[310]; + // reserve enough space for the longest double value + // -1.7976931348623157E+308 to -2.2250738585072014E-308, 0, and + // 2.2250738585072014E-308 to 1.7976931348623157E+308. + snprintf(buf, 310, "%.18g", dl); + return m_field->store(buf, strlen(buf), m_field->charset()); + } + + // The server converts dl=-0 to dl=0 in (*f)->store(). + // This happens in the call to truncate_double(). + // This is an unexpected behaviour, so we directly store the + // double value using the lower level float8store() function. + // TODO Remove this when (*f)->store() handles this properly. + m_field->field_length = 310; + if (dl == 0) + { + float8store(m_field->ptr,dl); + return 0; + } + return m_field->store(dl); + } + + int store_long_double(long double dl) override + { + if (dl == std::numeric_limits::infinity()) + { + m_field->set_null(); + return 1; + } + + if (m_field->type() == MYSQL_TYPE_NEWDECIMAL) + { + char buf[310]; + snprintf(buf, 310, "%.20Lg", dl); + return m_field->store(buf, strlen(buf), m_field->charset()); + } + + // reserve enough space for the longest double value + // -1.7976931348623157E+308 to -2.2250738585072014E-308, 0, and + // 2.2250738585072014E-308 to 1.7976931348623157E+308. + m_field->field_length = 310; + return m_field->store(static_cast(dl)); + } + + int store_decimal64(int64_t val) override + { + // @bug4388 stick to InfiniDB's scale in case mysql gives wrong scale due + // to create vtable limitation. + //if (f2->dec < m_type.scale) + // f2->dec = m_type.scale; + + // WIP MCOL-641 + // This is too much + char buf[256]; + dataconvert::DataConvert::decimalToString(val, (unsigned)m_type.scale, + buf, sizeof(buf), m_type.colDataType); + return m_field->store(buf, strlen(buf), m_field->charset()); + } + + int store_decimal128(const int128_t &val) override + { + // We won't have more than [+-][0][.] + up to 38 digits + char buf[datatypes::Decimal::MAXLENGTH16BYTES]; + dataconvert::DataConvert::decimalToString((int128_t*) &val, + (unsigned) m_type.scale, + buf, (uint8_t) sizeof(buf), + m_type.colDataType); + return m_field->store(buf, strlen(buf), m_field->charset()); + } + + int store_lob(const char *str, size_t length) override + { + idbassert(dynamic_cast(m_field)); + Field_blob* f2 = static_cast(m_field); + f2->set_ptr(length, (uchar*) str); + return 0; + } + +}; + + +/*******************************************************************************/ + +class WriteBatchFieldMariaDB: public WriteBatchField +{ +public: + Field *m_field; + const CalpontSystemCatalog::ColType &m_type; + WriteBatchFieldMariaDB(Field *field, const CalpontSystemCatalog::ColType type) + :m_field(field), m_type(type) + { } + size_t ColWriteBatchDate(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + // QQ: OLD DATE is not handled + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + fprintf(ci.filePtr(), "%c", ci.delimiter()); + } + else + { + const uchar* tmp1 = buf; + uint32_t tmp = (tmp1[2] << 16) + (tmp1[1] << 8) + tmp1[0]; + int day = tmp & 0x0000001fl; + int month = (tmp >> 5) & 0x0000000fl; + int year = tmp >> 9; + fprintf(ci.filePtr(), "%04d-%02d-%02d%c", year, month, day, ci.delimiter()); + } + return 3; + } + size_t ColWriteBatchDatetime(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + fprintf(ci.filePtr(), "%c", ci.delimiter()); + + if (m_field->real_type() == MYSQL_TYPE_DATETIME2) + return m_field->pack_length(); + else + return 8; + } + + if (m_field->real_type() == MYSQL_TYPE_DATETIME2) + { + // mariadb 10.1 compatibility -- MYSQL_TYPE_DATETIME2 introduced in mysql 5.6 + MYSQL_TIME ltime; + longlong tmp = my_datetime_packed_from_binary(buf, m_field->decimals()); + TIME_from_longlong_datetime_packed(<ime, tmp); + + if (!ltime.second_part) + { + fprintf(ci.filePtr(), "%04d-%02d-%02d %02d:%02d:%02d%c", + ltime.year, ltime.month, ltime.day, + ltime.hour, ltime.minute, ltime.second, ci.delimiter()); + } + else + { + fprintf(ci.filePtr(), "%04d-%02d-%02d %02d:%02d:%02d.%ld%c", + ltime.year, ltime.month, ltime.day, + ltime.hour, ltime.minute, ltime.second, + ltime.second_part, ci.delimiter()); + } + + return m_field->pack_length(); + } + + // Old DATETIME + long long value = *((long long*) buf); + long datePart = (long) (value / 1000000ll); + int day = datePart % 100; + int month = (datePart / 100) % 100; + int year = datePart / 10000; + fprintf(ci.filePtr(), "%04d-%02d-%02d ", year, month, day); + + long timePart = (long) (value - (long long) datePart * 1000000ll); + int second = timePart % 100; + int min = (timePart / 100) % 100; + int hour = timePart / 10000; + fprintf(ci.filePtr(), "%02d:%02d:%02d%c", hour, min, second, ci.delimiter()); + return 8; + } + size_t ColWriteBatchTime(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + // QQ: why old TIME is not handled? + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + fprintf(ci.filePtr(), "%c", ci.delimiter()); + return m_field->pack_length(); + } + + MYSQL_TIME ltime; + longlong tmp = my_time_packed_from_binary(buf, m_field->decimals()); + TIME_from_longlong_time_packed(<ime, tmp); + + if (ltime.neg) + fprintf(ci.filePtr(), "-"); + + if (!ltime.second_part) + { + fprintf(ci.filePtr(), "%02d:%02d:%02d%c", + ltime.hour, ltime.minute, ltime.second, ci.delimiter()); + } + else + { + fprintf(ci.filePtr(), "%02d:%02d:%02d.%ld%c", + ltime.hour, ltime.minute, ltime.second, + ltime.second_part, ci.delimiter()); + } + + return m_field->pack_length(); + } + + size_t ColWriteBatchTimestamp(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + // QQ: old TIMESTAMP is not handled + + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + fprintf(ci.filePtr(), "%c", ci.delimiter()); + return m_field->pack_length(); + } + + struct timeval tm; + my_timestamp_from_binary(&tm, buf, m_field->decimals()); + + MySQLTime time; + gmtSecToMySQLTime(tm.tv_sec, time, current_thd->variables.time_zone->get_name()->ptr()); + + if (!tm.tv_usec) + { + fprintf(ci.filePtr(), "%04d-%02d-%02d %02d:%02d:%02d%c", + time.year, time.month, time.day, + time.hour, time.minute, time.second, ci.delimiter()); + } + else + { + fprintf(ci.filePtr(), "%04d-%02d-%02d %02d:%02d:%02d.%ld%c", + time.year, time.month, time.day, + time.hour, time.minute, time.second, + tm.tv_usec, ci.delimiter()); + } + return m_field->pack_length(); + } + + size_t ColWriteBatchChar(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + fprintf(ci.filePtr(), "%c", ci.delimiter()); + } + else + { + if (current_thd->variables.sql_mode & MODE_PAD_CHAR_TO_FULL_LENGTH) + { + std::string escape; + // Pad to the full length of the field + if (ci.utf8()) + escape.assign((char*)buf, m_type.colWidth * 3); + else + escape.assign((char*)buf, m_type.colWidth); + + boost::replace_all(escape, "\\", "\\\\"); + + fprintf(ci.filePtr(), "%c%.*s%c%c", + ci.enclosed_by(), + (int)escape.length(), escape.c_str(), + ci.enclosed_by(), ci.delimiter()); + } + else + { + std::string escape; + // Get the actual data length + bitmap_set_bit(m_field->table->read_set, m_field->field_index); + String attribute; + m_field->val_str(&attribute); + + escape.assign((char*)buf, attribute.length()); + boost::replace_all(escape, "\\", "\\\\"); + + fprintf(ci.filePtr(), "%c%.*s%c%c", + ci.enclosed_by(), + (int)escape.length(), escape.c_str(), + ci.enclosed_by(), ci.delimiter()); + } + } + + if (ci.utf8()) + return m_type.colWidth * 3; + else + return m_type.colWidth; + } + + size_t ColWriteBatchVarchar(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + const uchar *buf0= buf; + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + fprintf(ci.filePtr(), "%c", ci.delimiter()); + + if (!ci.utf8()) + { + if (m_type.colWidth < 256) + { + buf++; + } + else + { + buf = buf + 2 ; + } + } + else //utf8 + { + if (m_type.colWidth < 86) + { + buf++; + } + else + { + buf = buf + 2 ; + } + } + } + else + { + int dataLength = 0; + + if (!ci.utf8()) + { + if (m_type.colWidth < 256) + { + dataLength = *(uint8_t*) buf; + buf++; + } + else + { + dataLength = *(uint16_t*) buf; + buf = buf + 2 ; + } + std::string escape; + escape.assign((char*)buf, dataLength); + boost::replace_all(escape, "\\", "\\\\"); + fprintf(ci.filePtr(), "%c%.*s%c%c", + ci.enclosed_by(), + (int)escape.length(), escape.c_str(), + ci.enclosed_by(), ci.delimiter()); + } + else //utf8 + { + if (m_type.colWidth < 86) + { + dataLength = *(uint8_t*) buf; + buf++; + } + else + { + dataLength = *(uint16_t*) buf; + buf = buf + 2 ; + } + + std::string escape; + escape.assign((char*)buf, dataLength); + boost::replace_all(escape, "\\", "\\\\"); + + fprintf(ci.filePtr(), "%c%.*s%c%c", + ci.enclosed_by(), + (int)escape.length(), escape.c_str(), + ci.enclosed_by(), ci.delimiter()); + } + } + if (ci.utf8()) + buf += (m_type.colWidth * 3); + else + buf += m_type.colWidth; + return buf - buf0; + } + + size_t ColWriteBatchSInt64(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + fprintf(ci.filePtr(), "%lld%c", *((long long*)buf), ci.delimiter()); + return 8; + } + + size_t ColWriteBatchUInt64(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + fprintf(ci.filePtr(), "%llu%c", *((long long unsigned*)buf), ci.delimiter()); + return 8; + } + + + size_t ColWriteBatchSInt32(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + fprintf(ci.filePtr(), "%d%c", *((int32_t*)buf), ci.delimiter()); + return 4; + } + + size_t ColWriteBatchUInt32(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + fprintf(ci.filePtr(), "%u%c", *((uint32_t*)buf), ci.delimiter()); + return 4; + } + + size_t ColWriteBatchSInt16(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + fprintf(ci.filePtr(), "%d%c", *((int16_t*)buf), ci.delimiter()); + return 2; + } + + size_t ColWriteBatchUInt16(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + fprintf(ci.filePtr(), "%u%c", *((uint16_t*)buf), ci.delimiter()); + return 2; + } + + size_t ColWriteBatchSInt8(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + fprintf(ci.filePtr(), "%d%c", *((int8_t*)buf), ci.delimiter()); + return 1; + } + + size_t ColWriteBatchUInt8(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + fprintf(ci.filePtr(), "%u%c", *((uint8_t*)buf), ci.delimiter()); + return 1; + } + + + size_t ColWriteBatchXFloat(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + { + float val = *((float*)buf); + + if ((fabs(val) > (1.0 / IDB_pow[4])) && (fabs(val) < (float) IDB_pow[6])) + { + fprintf(ci.filePtr(), "%.7f%c", val, ci.delimiter()); + } + else + { + fprintf(ci.filePtr(), "%e%c", val, ci.delimiter()); + } + + //fprintf(ci.filePtr(), "%.7g|", *((float*)buf2)); + //printf("%.7f|", *((float*)buf2)); + } + + return 4; + } + + + size_t ColWriteBatchXDouble(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + { + fprintf(ci.filePtr(), "%.15g%c", *((double*)buf), ci.delimiter()); + //printf("%.15g|", *((double*)buf)); + } + + return 8; + } + + + size_t ColWriteBatchSLongDouble(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + fprintf(ci.filePtr(), "%c", ci.delimiter()); + else + fprintf(ci.filePtr(), "%.15Lg%c", *((long double*)buf), ci.delimiter()); + return sizeof(long double); + } + + + size_t ColWriteBatchXDecimal(const uchar *buf, bool nullVal, ColBatchWriter &ci) override + { + uint bytesBefore = 1; + uint totalBytes = 9; + + switch (m_type.precision) + { + case 18: + case 17: + case 16: + { + totalBytes = 8; + break; + } + + case 15: + case 14: + { + totalBytes = 7; + break; + } + + case 13: + case 12: + { + totalBytes = 6; + break; + } + + case 11: + { + totalBytes = 5; + break; + } + + case 10: + { + totalBytes = 5; + break; + } + + case 9: + case 8: + case 7: + { + totalBytes = 4; + break; + } + + case 6: + case 5: + { + totalBytes = 3; + break; + } + + case 4: + case 3: + { + totalBytes = 2; + break; + } + + case 2: + case 1: + { + totalBytes = 1; + break; + } + + default: + break; + } + + switch (m_type.scale) + { + case 0: + { + bytesBefore = totalBytes; + break; + } + + case 1: //1 byte for digits after decimal point + { + if ((m_type.precision != 16) && (m_type.precision != 14) + && (m_type.precision != 12) && (m_type.precision != 10) + && (m_type.precision != 7) && (m_type.precision != 5) + && (m_type.precision != 3) && (m_type.precision != 1)) + totalBytes++; + + bytesBefore = totalBytes - 1; + break; + } + + case 2: //1 byte for digits after decimal point + { + if ((m_type.precision == 18) || (m_type.precision == 9)) + totalBytes++; + + bytesBefore = totalBytes - 1; + break; + } + + case 3: //2 bytes for digits after decimal point + { + if ((m_type.precision != 16) && (m_type.precision != 14) + && (m_type.precision != 12) && (m_type.precision != 7) + && (m_type.precision != 5) && (m_type.precision != 3)) + totalBytes++; + + bytesBefore = totalBytes - 2; + break; + } + + case 4: + { + if ((m_type.precision == 18) || (m_type.precision == 11) + || (m_type.precision == 9)) + totalBytes++; + + bytesBefore = totalBytes - 2; + break; + + } + + case 5: + { + if ((m_type.precision != 16) && (m_type.precision != 14) + && (m_type.precision != 7) && (m_type.precision != 5)) + totalBytes++; + + bytesBefore = totalBytes - 3; + break; + } + + case 6: + { + if ((m_type.precision == 18) || (m_type.precision == 13) + || (m_type.precision == 11) || (m_type.precision == 9)) + totalBytes++; + + bytesBefore = totalBytes - 3; + break; + } + + case 7: + { + if ((m_type.precision != 16) && (m_type.precision != 7)) + totalBytes++; + + bytesBefore = totalBytes - 4; + break; + } + + case 8: + { + if ((m_type.precision == 18) || (m_type.precision == 15) + || (m_type.precision == 13) || (m_type.precision == 11) + || (m_type.precision == 9)) + totalBytes++; + + bytesBefore = totalBytes - 4;; + break; + } + + case 9: + { + bytesBefore = totalBytes - 4;; + break; + } + + case 10: + { + if ((m_type.precision != 16) && (m_type.precision != 14) + && (m_type.precision != 12) && (m_type.precision != 10)) + totalBytes++; + + bytesBefore = totalBytes - 5;; + break; + } + + case 11: + { + if (m_type.precision == 18) + totalBytes++; + + bytesBefore = totalBytes - 5; + break; + } + + case 12: + { + if ((m_type.precision != 16) && (m_type.precision != 14) + && (m_type.precision != 12)) + totalBytes++; + + bytesBefore = totalBytes - 6; + break; + } + + case 13: + { + if (m_type.precision == 18) + totalBytes++; + + bytesBefore = totalBytes - 6; + break; + } + + case 14: + { + if ((m_type.precision != 16) && (m_type.precision != 14)) + totalBytes++; + + bytesBefore = totalBytes - 7; + break; + } + + case 15: + { + if (m_type.precision == 18) + totalBytes++; + + bytesBefore = totalBytes - 7; + break; + } + + case 16: + { + if (m_type.precision != 16) + totalBytes++; + + bytesBefore = totalBytes - 8; + break; + } + + case 17: + { + if (m_type.precision == 18) + totalBytes++; + + bytesBefore = totalBytes - 8; + break; + } + + case 18: + { + bytesBefore = totalBytes - 8; + break; + } + + default: + break; + } + + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + fprintf(ci.filePtr(), "%c", ci.delimiter()); + //printf("|"); + } + else + { + uint32_t mask [5] = {0, 0xFF, 0xFFFF, 0xFFFFFF, 0xFFFFFFFF}; + char neg = '-'; + + if (m_type.scale == 0) + { + const uchar* tmpBuf = buf; + //test flag bit for sign + bool posNum = tmpBuf[0] & (0x80); + uchar tmpChr = tmpBuf[0]; + tmpChr ^= 0x80; //flip the bit + int32_t tmp1 = tmpChr; + + if (totalBytes > 4) + { + for (uint i = 1; i < (totalBytes - 4); i++) + { + tmp1 = (tmp1 << 8) + tmpBuf[i]; + } + + if (( tmp1 != 0 ) && (tmp1 != -1)) + { + if (!posNum) + { + tmp1 = mask[totalBytes - 4] - tmp1; + + if (tmp1 != 0) + { + fprintf(ci.filePtr(), "%c", neg); + //printf("%c", neg); + } + } + + if (tmp1 != 0) + { + fprintf(ci.filePtr(), "%d", tmp1); + ////printf("%d", tmp1); + } + } + + int32_t tmp2 = tmpBuf[totalBytes - 4]; + + for (uint i = (totalBytes - 3); i < totalBytes; i++) + { + tmp2 = (tmp2 << 8) + tmpBuf[i]; + } + + if ( tmp1 != 0 ) + { + if (!posNum) + { + tmp2 = mask[4] - tmp2; + + if (tmp1 == -1) + { + fprintf(ci.filePtr(), "%c", neg); + fprintf(ci.filePtr(), "%d%c", tmp2, ci.delimiter()); + ////printf("%c", neg); + //////printf( "%d|", tmp2); + } + else + { + fprintf(ci.filePtr(), "%09u%c", tmp2, ci.delimiter()); + ////printf("%09u|", tmp2); + } + } + else + { + fprintf(ci.filePtr(), "%09u%c", tmp2, ci.delimiter()); + //printf("%09u|", tmp2); + } + } + else + { + if (!posNum) + { + tmp2 = mask[4] - tmp2; + fprintf(ci.filePtr(), "%c", neg); + //printf("%c", neg); + } + + fprintf(ci.filePtr(), "%d%c", tmp2, ci.delimiter()); + //printf("%d|", tmp2); + } + } + else + { + for (uint i = 1; i < totalBytes; i++) + { + tmp1 = (tmp1 << 8) + tmpBuf[i]; + } + + if (!posNum) + { + tmp1 = mask[totalBytes] - tmp1; + fprintf(ci.filePtr(), "%c", neg); + //printf("%c", neg); + } + + fprintf(ci.filePtr(), "%d%c", tmp1, ci.delimiter()); + //printf("%d|", tmp1); + } + } + else + { + const uchar* tmpBuf = buf; + //test flag bit for sign + bool posNum = tmpBuf[0] & (0x80); + uchar tmpChr = tmpBuf[0]; + tmpChr ^= 0x80; //flip the bit + int32_t tmp1 = tmpChr; + + //fetch the digits before decimal point + if (bytesBefore == 0) + { + if (!posNum) + { + fprintf(ci.filePtr(), "%c", neg); + //printf("%c", neg); + } + + fprintf(ci.filePtr(), "0."); + //printf("0."); + } + else if (bytesBefore > 4) + { + for (uint i = 1; i < (bytesBefore - 4); i++) + { + tmp1 = (tmp1 << 8) + tmpBuf[i]; + } + + if (!posNum) + { + tmp1 = mask[bytesBefore - 4] - tmp1; + } + + if (( tmp1 != 0 ) && (tmp1 != -1)) + { + if (!posNum) + { + fprintf(ci.filePtr(), "%c", neg); + //printf("%c", neg); + } + + fprintf(ci.filePtr(), "%d", tmp1); + //printf("%d", tmp1); + } + + tmpBuf += (bytesBefore - 4); + int32_t tmp2 = *((int32_t*)tmpBuf); + tmp2 = ntohl(tmp2); + + if ( tmp1 != 0 ) + { + if (!posNum) + { + tmp2 = mask[4] - tmp2; + } + + if (tmp1 == -1) + { + fprintf(ci.filePtr(), "%c", neg); + fprintf(ci.filePtr(), "%d.", tmp2); + //printf("%c", neg); + //printf("%d.", tmp2); + } + else + { + fprintf(ci.filePtr(), "%09u.", tmp2); + //printf("%09u.", tmp2); + } + } + else + { + if (!posNum) + { + tmp2 = mask[4] - tmp2; + fprintf(ci.filePtr(), "%c", neg); + //printf("%c", neg); + } + + fprintf(ci.filePtr(), "%d.", tmp2); + //printf("%d.", tmp2); + } + } + else + { + for (uint i = 1; i < bytesBefore; i++) + { + tmp1 = (tmp1 << 8) + tmpBuf[i]; + } + + if (!posNum) + { + tmp1 = mask[bytesBefore] - tmp1; + fprintf(ci.filePtr(), "%c", neg); + //printf("%c", neg); + } + + fprintf(ci.filePtr(), "%d.", tmp1); + //printf("%d.", tmp1); + } + + //fetch the digits after decimal point + int32_t tmp2 = 0; + + if (bytesBefore > 4) + tmpBuf += 4; + else + tmpBuf += bytesBefore; + + tmp2 = tmpBuf[0]; + + if ((totalBytes - bytesBefore) < 5) + { + for (uint j = 1; j < (totalBytes - bytesBefore); j++) + { + tmp2 = (tmp2 << 8) + tmpBuf[j]; + } + + int8_t digits = m_type.scale - 9; //9 digits is a 4 bytes chunk + + if ( digits <= 0 ) + digits = m_type.scale; + + if (!posNum) + { + tmp2 = mask[totalBytes - bytesBefore] - tmp2; + } + + fprintf(ci.filePtr(), "%0*u%c", digits, tmp2, ci.delimiter()); + //printf("%0*u|", digits, tmp2); + } + else + { + for (uint j = 1; j < 4; j++) + { + tmp2 = (tmp2 << 8) + tmpBuf[j]; + } + + if (!posNum) + { + tmp2 = mask[4] - tmp2; + } + + fprintf(ci.filePtr(), "%09u", tmp2); + //printf("%09u", tmp2); + + tmpBuf += 4; + int32_t tmp3 = tmpBuf[0]; + + for (uint j = 1; j < (totalBytes - bytesBefore - 4); j++) + { + tmp3 = (tmp3 << 8) + tmpBuf[j]; + } + + int8_t digits = m_type.scale - 9; //9 digits is a 4 bytes chunk + + if ( digits < 0 ) + digits = m_type.scale; + + if (!posNum) + { + tmp3 = mask[totalBytes - bytesBefore - 4] - tmp3; + } + + fprintf(ci.filePtr(), "%0*u%c", digits, tmp3, ci.delimiter()); + //printf("%0*u|", digits, tmp3); + } + } + } + + return totalBytes; + } + + + size_t ColWriteBatchVarbinary(const uchar *buf0, bool nullVal, ColBatchWriter &ci) override + { + const uchar *buf= buf0; + if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) + { + fprintf(ci.filePtr(), "%c", ci.delimiter()); + + if (!ci.utf8()) + { + if (m_type.colWidth < 256) + { + buf++; + } + else + { + buf = buf + 2; + } + } + else //utf8 + { + if (m_type.colWidth < 86) + { + buf++; + } + else + { + buf = buf + 2 ; + } + } + } + else + { + int dataLength = 0; + + if (!ci.utf8()) + { + if (m_type.colWidth < 256) + { + dataLength = *(int8_t*) buf; + buf++; + } + else + { + dataLength = *(int16_t*) buf; + buf = buf + 2 ; + } + + const uchar* tmpBuf = buf; + + for (int32_t i = 0; i < dataLength; i++) + { + fprintf(ci.filePtr(), "%02x", *(uint8_t*)tmpBuf); + tmpBuf++; + } + + fprintf(ci.filePtr(), "%c", ci.delimiter()); + } + else //utf8 + { + if (m_type.colWidth < 86) + { + dataLength = *(int8_t*) buf; + buf++; + } + else + { + dataLength = *(uint16_t*) buf; + buf = buf + 2 ; + } + + if ( dataLength > m_type.colWidth) + dataLength = m_type.colWidth; + + const uchar* tmpBuf = buf; + + for (int32_t i = 0; i < dataLength; i++) + { + fprintf(ci.filePtr(), "%02x", *(uint8_t*)tmpBuf); + tmpBuf++; + } + + fprintf(ci.filePtr(), "%c", ci.delimiter()); + } + } + + if (ci.utf8()) + buf += (m_type.colWidth * 3); // QQ: why? It is varbinary! + else + buf += m_type.colWidth; + + return buf - buf0; + } + + + size_t ColWriteBatchBlob(const uchar *buf0, bool nullVal, ColBatchWriter &ci) override + { + const uchar *buf= buf0; + // MCOL-4005 Note that we don't handle nulls as a special + // case here as we do for other datatypes, the below works + // as expected for nulls. + uint32_t dataLength = 0; + uintptr_t* dataptr; + uchar* ucharptr; + uint colWidthInBytes = (ci.utf8() ? + m_type.colWidth * 3: m_type.colWidth); + + if (colWidthInBytes < 256) + { + dataLength = *(uint8_t*) buf; + buf++; + } + else if (colWidthInBytes < 65536) + { + dataLength = *(uint16_t*) buf; + buf += 2; + } + else if (colWidthInBytes < 16777216) + { + dataLength = *(uint16_t*) buf; + dataLength |= ((int) buf[2]) << 16; + buf += 3; + } + else + { + dataLength = *(uint32_t*) buf; + buf += 4; + } + + // buf contains pointer to blob, for example: + // (gdb) p (char*)*(uintptr_t*)buf + // $43 = 0x7f68500c58f8 "hello world" + + dataptr = (uintptr_t*)buf; + ucharptr = (uchar*)*dataptr; + buf += sizeof(uintptr_t); + + if (m_type.colDataType == CalpontSystemCatalog::BLOB) + { + for (uint32_t i = 0; i < dataLength; i++) + { + fprintf(ci.filePtr(), "%02x", *(uint8_t*)ucharptr); + ucharptr++; + } + + fprintf(ci.filePtr(), "%c", ci.delimiter()); + } + else + { + // TEXT Column + std::string escape; + escape.assign((char*)ucharptr, dataLength); + boost::replace_all(escape, "\\", "\\\\"); + fprintf(ci.filePtr(), "%c%.*s%c%c", + ci.enclosed_by(), + (int)escape.length(), escape.c_str(), + ci.enclosed_by(), ci.delimiter()); + } + + return buf - buf0; + } + +}; + +} // end of namespace datatypes + +#endif + +// vim:ts=2 sw=2: diff --git a/dbcon/mysql/ha_mcs_ddl.cpp b/dbcon/mysql/ha_mcs_ddl.cpp index 32daf2c48..e1eca830f 100644 --- a/dbcon/mysql/ha_mcs_ddl.cpp +++ b/dbcon/mysql/ha_mcs_ddl.cpp @@ -61,9 +61,6 @@ using namespace ddlpackage; #include "ddlpackageprocessor.h" using namespace ddlpackageprocessor; -#include "dataconvert.h" -using namespace dataconvert; - #include "bytestream.h" using namespace messageqcpp; @@ -133,135 +130,16 @@ static void decode_file_path(const char *path, char *decoded_dbname, decode_objectname(decoded_tbname, tbname_start, FN_REFLEN); } -uint32_t convertDataType(int dataType) + +CalpontSystemCatalog::ColDataType convertDataType(const ddlpackage::ColumnType &ct) { - uint32_t calpontDataType; - - switch (dataType) - { - case ddlpackage::DDL_CHAR: - calpontDataType = CalpontSystemCatalog::CHAR; - break; - - case ddlpackage::DDL_VARCHAR: - calpontDataType = CalpontSystemCatalog::VARCHAR; - break; - - case ddlpackage::DDL_VARBINARY: - calpontDataType = CalpontSystemCatalog::VARBINARY; - break; - - case ddlpackage::DDL_BIT: - calpontDataType = CalpontSystemCatalog::BIT; - break; - - case ddlpackage::DDL_REAL: - case ddlpackage::DDL_DECIMAL: - case ddlpackage::DDL_NUMERIC: - case ddlpackage::DDL_NUMBER: - calpontDataType = CalpontSystemCatalog::DECIMAL; - break; - - case ddlpackage::DDL_FLOAT: - calpontDataType = CalpontSystemCatalog::FLOAT; - break; - - case ddlpackage::DDL_DOUBLE: - calpontDataType = CalpontSystemCatalog::DOUBLE; - break; - - case ddlpackage::DDL_INT: - case ddlpackage::DDL_INTEGER: - calpontDataType = CalpontSystemCatalog::INT; - break; - - case ddlpackage::DDL_BIGINT: - calpontDataType = CalpontSystemCatalog::BIGINT; - break; - - case ddlpackage::DDL_MEDINT: - calpontDataType = CalpontSystemCatalog::MEDINT; - break; - - case ddlpackage::DDL_SMALLINT: - calpontDataType = CalpontSystemCatalog::SMALLINT; - break; - - case ddlpackage::DDL_TINYINT: - calpontDataType = CalpontSystemCatalog::TINYINT; - break; - - case ddlpackage::DDL_DATE: - calpontDataType = CalpontSystemCatalog::DATE; - break; - - case ddlpackage::DDL_DATETIME: - calpontDataType = CalpontSystemCatalog::DATETIME; - break; - - case ddlpackage::DDL_TIME: - calpontDataType = CalpontSystemCatalog::TIME; - break; - - case ddlpackage::DDL_TIMESTAMP: - calpontDataType = CalpontSystemCatalog::TIMESTAMP; - break; - - case ddlpackage::DDL_CLOB: - calpontDataType = CalpontSystemCatalog::CLOB; - break; - - case ddlpackage::DDL_BLOB: - calpontDataType = CalpontSystemCatalog::BLOB; - break; - - case ddlpackage::DDL_TEXT: - calpontDataType = CalpontSystemCatalog::TEXT; - break; - - case ddlpackage::DDL_UNSIGNED_TINYINT: - calpontDataType = CalpontSystemCatalog::UTINYINT; - break; - - case ddlpackage::DDL_UNSIGNED_SMALLINT: - calpontDataType = CalpontSystemCatalog::USMALLINT; - break; - - case ddlpackage::DDL_UNSIGNED_MEDINT: - calpontDataType = CalpontSystemCatalog::UMEDINT; - break; - - case ddlpackage::DDL_UNSIGNED_INT: - calpontDataType = CalpontSystemCatalog::UINT; - break; - - case ddlpackage::DDL_UNSIGNED_BIGINT: - calpontDataType = CalpontSystemCatalog::UBIGINT; - break; - - case ddlpackage::DDL_UNSIGNED_DECIMAL: - case ddlpackage::DDL_UNSIGNED_NUMERIC: - calpontDataType = CalpontSystemCatalog::UDECIMAL; - break; - - case ddlpackage::DDL_UNSIGNED_FLOAT: - calpontDataType = CalpontSystemCatalog::UFLOAT; - break; - - case ddlpackage::DDL_UNSIGNED_DOUBLE: - calpontDataType = CalpontSystemCatalog::UDOUBLE; - break; - - case ddlpackage::DDL_BINARY: - calpontDataType = CalpontSystemCatalog::BINARY; - break; - - default: - throw runtime_error("Unsupported datatype!"); - - } - - return calpontDataType; + const datatypes::TypeHandler *h= datatypes::TypeHandler::find_by_ddltype(ct); + if (!h) + { + throw runtime_error("Unsupported datatype!"); + return CalpontSystemCatalog::UNDEFINED; + } + return h->code(); } @@ -1007,10 +885,10 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl if (!createTable->fTableDef->fColumns[i]->fDefaultValue->fNull) { //validate the default value, if out of range, just error out - uint32_t dataType; + CalpontSystemCatalog::ColDataType dataType; dataType = convertDataType(createTable->fTableDef->fColumns[i]->fType->fType); CalpontSystemCatalog::ColType colType; - colType.colDataType = (CalpontSystemCatalog::ColDataType) dataType; + colType.colDataType = dataType; colType.colWidth = createTable->fTableDef->fColumns[i]->fType->fLength; colType.precision = createTable->fTableDef->fColumns[i]->fType->fPrecision; colType.scale = createTable->fTableDef->fColumns[i]->fType->fScale; @@ -1019,7 +897,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl try { - convertedVal = DataConvert::convertColumnData(colType, createTable->fTableDef->fColumns[i]->fDefaultValue->fValue, pushWarning, thd->variables.time_zone->get_name()->ptr(), false, false, false); + convertedVal = colType.convertColumnData(createTable->fTableDef->fColumns[i]->fDefaultValue->fValue, pushWarning, thd->variables.time_zone->get_name()->ptr(), false, false, false); } catch (std::exception&) { @@ -1383,10 +1261,10 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl } //validate the default value, if out of range, just error out - uint32_t dataType; + CalpontSystemCatalog::ColDataType dataType; dataType = convertDataType(addColumnPtr->fColumnDef->fType->fType); CalpontSystemCatalog::ColType colType; - colType.colDataType = (CalpontSystemCatalog::ColDataType) dataType; + colType.colDataType = dataType; colType.colWidth = addColumnPtr->fColumnDef->fType->fLength; colType.precision = addColumnPtr->fColumnDef->fType->fPrecision; colType.scale = addColumnPtr->fColumnDef->fType->fScale; @@ -1395,7 +1273,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl try { - convertedVal = DataConvert::convertColumnData(colType, addColumnPtr->fColumnDef->fDefaultValue->fValue, pushWarning, thd->variables.time_zone->get_name()->ptr(), false, false, false); + convertedVal = colType.convertColumnData(addColumnPtr->fColumnDef->fDefaultValue->fValue, pushWarning, thd->variables.time_zone->get_name()->ptr(), false, false, false); } catch (std::exception&) { @@ -1737,10 +1615,10 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl } //validate the default value, if out of range, just error out - uint32_t dataType; + CalpontSystemCatalog::ColDataType dataType; dataType = convertDataType(addColumnsPtr->fColumns[0]->fType->fType); CalpontSystemCatalog::ColType colType; - colType.colDataType = (CalpontSystemCatalog::ColDataType) dataType; + colType.colDataType = dataType; colType.colWidth = addColumnsPtr->fColumns[0]->fType->fLength; colType.precision = addColumnsPtr->fColumns[0]->fType->fPrecision; colType.scale = addColumnsPtr->fColumns[0]->fType->fScale; @@ -1749,7 +1627,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl try { - convertedVal = DataConvert::convertColumnData(colType, addColumnsPtr->fColumns[0]->fDefaultValue->fValue, pushWarning, thd->variables.time_zone->get_name()->ptr(), false, false, false); + convertedVal = colType.convertColumnData(addColumnsPtr->fColumns[0]->fDefaultValue->fValue, pushWarning, thd->variables.time_zone->get_name()->ptr(), false, false, false); } catch (std::exception&) { diff --git a/dbcon/mysql/ha_mcs_dml.cpp b/dbcon/mysql/ha_mcs_dml.cpp index 06d1129fe..88d9997f9 100644 --- a/dbcon/mysql/ha_mcs_dml.cpp +++ b/dbcon/mysql/ha_mcs_dml.cpp @@ -75,6 +75,8 @@ using namespace joblist; #include "dbrm.h" +#include "ha_mcs_datatype.h" + namespace { #define BATCH_INSERT_GROUP_ROWS_FOR_CACHE 100000 @@ -669,6 +671,7 @@ int ha_mcs_impl_write_row_(const uchar* buf, TABLE* table, cal_connection_info& } } + int ha_mcs_impl_write_batch_row_(const uchar* buf, TABLE* table, cal_impl_if::cal_connection_info& ci) { ByteStream rowData; @@ -681,7 +684,6 @@ int ha_mcs_impl_write_batch_row_(const uchar* buf, TABLE* table, cal_impl_if::ca uint16_t colpos = 0; buf = buf + ci.headerLength; // Number of bytes used for null bits. //@Bug 6122 if all columns have not null constraint, there is no information in the header - std::string escape; char nullBits = *bufHdr++; if (!ci.useXbit) @@ -730,1078 +732,16 @@ int ha_mcs_impl_write_batch_row_(const uchar* buf, TABLE* table, cal_impl_if::ca nullVal = false; } - switch (ci.columnTypes[colpos].colDataType) + const CalpontSystemCatalog::ColType &colType= ci.columnTypes[colpos]; + const datatypes::TypeHandler *h= colType.typeHandler(); + if (h) // QQ: error reporting { - case CalpontSystemCatalog::DATE: //date fetch - { - - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - { - fprintf(ci.filePtr, "%c", ci.delimiter); - } - else - { - const uchar* tmp1 = buf; - uint32_t tmp = (tmp1[2] << 16) + (tmp1[1] << 8) + tmp1[0]; - - int day = tmp & 0x0000001fl; - int month = (tmp >> 5) & 0x0000000fl; - int year = tmp >> 9; - fprintf(ci.filePtr, "%04d-%02d-%02d%c", year, month, day, ci.delimiter); - } - - buf += 3; - break; - } - - case CalpontSystemCatalog::DATETIME: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - { - fprintf(ci.filePtr, "%c", ci.delimiter); - - if (table->field[colpos]->real_type() == MYSQL_TYPE_DATETIME2) - buf += table->field[colpos]->pack_length(); - else - buf += 8; - } - else - { - if (table->field[colpos]->real_type() == MYSQL_TYPE_DATETIME2) - { - // mariadb 10.1 compatibility -- MYSQL_TYPE_DATETIME2 introduced in mysql 5.6 - MYSQL_TIME ltime; - const uchar* pos = buf; - longlong tmp = my_datetime_packed_from_binary(pos, table->field[colpos]->decimals()); - TIME_from_longlong_datetime_packed(<ime, tmp); - - if (!ltime.second_part) - { - fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d%c", - ltime.year, ltime.month, ltime.day, - ltime.hour, ltime.minute, ltime.second, ci.delimiter); - } - else - { - fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d.%ld%c", - ltime.year, ltime.month, ltime.day, - ltime.hour, ltime.minute, ltime.second, - ltime.second_part, ci.delimiter); - } - - buf += table->field[colpos]->pack_length(); - } - else - { - long long value = *((long long*) buf); - long datePart = (long) (value / 1000000ll); - int day = datePart % 100; - int month = (datePart / 100) % 100; - int year = datePart / 10000; - fprintf(ci.filePtr, "%04d-%02d-%02d ", year, month, day); - - long timePart = (long) (value - (long long) datePart * 1000000ll); - int second = timePart % 100; - int min = (timePart / 100) % 100; - int hour = timePart / 10000; - fprintf(ci.filePtr, "%02d:%02d:%02d%c", hour, min, second, ci.delimiter); - buf += 8; - } - } - - break; - } - - case CalpontSystemCatalog::TIME: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - { - fprintf(ci.filePtr, "%c", ci.delimiter); - - buf += table->field[colpos]->pack_length(); - } - else - { - MYSQL_TIME ltime; - const uchar* pos = buf; - longlong tmp = my_time_packed_from_binary(pos, table->field[colpos]->decimals()); - TIME_from_longlong_time_packed(<ime, tmp); - - if (ltime.neg) - { - fprintf(ci.filePtr, "-"); - } - - if (!ltime.second_part) - { - fprintf(ci.filePtr, "%02d:%02d:%02d%c", - ltime.hour, ltime.minute, ltime.second, ci.delimiter); - } - else - { - fprintf(ci.filePtr, "%02d:%02d:%02d.%ld%c", - ltime.hour, ltime.minute, ltime.second, - ltime.second_part, ci.delimiter); - } - - buf += table->field[colpos]->pack_length(); - } - - break; - } - - case CalpontSystemCatalog::TIMESTAMP: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - { - fprintf(ci.filePtr, "%c", ci.delimiter); - } - else - { - const uchar* pos = buf; - struct timeval tm; - my_timestamp_from_binary(&tm, pos, table->field[colpos]->decimals()); - - MySQLTime time; - gmtSecToMySQLTime(tm.tv_sec, time, current_thd->variables.time_zone->get_name()->ptr()); - - if (!tm.tv_usec) - { - fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d%c", - time.year, time.month, time.day, - time.hour, time.minute, time.second, ci.delimiter); - } - else - { - fprintf(ci.filePtr, "%04d-%02d-%02d %02d:%02d:%02d.%ld%c", - time.year, time.month, time.day, - time.hour, time.minute, time.second, - tm.tv_usec, ci.delimiter); - } - } - - buf += table->field[colpos]->pack_length(); - - break; - } - - case CalpontSystemCatalog::CHAR: - { - Field* field = table->field[colpos]; - - uint32_t colWidthInBytes = - ci.columnTypes[colpos].colWidth * field->charset()->mbmaxlen; - - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - { - fprintf(ci.filePtr, "%c", ci.delimiter); - } - else - { - if (current_thd->variables.sql_mode & MODE_PAD_CHAR_TO_FULL_LENGTH) - { - // Pad to the full length of the field - escape.assign((char*)buf, colWidthInBytes); - - boost::replace_all(escape, "\\", "\\\\"); - - fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, (int)escape.length(), - escape.c_str(), ci.enclosed_by, ci.delimiter); - } - else - { - // Get the actual data length - Field* field = table->field[colpos]; - bitmap_set_bit(table->read_set, field->field_index); - String attribute; - field->val_str(&attribute); - - escape.assign((char*)buf, attribute.length()); - boost::replace_all(escape, "\\", "\\\\"); - - fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, (int)escape.length(), - escape.c_str(), ci.enclosed_by, ci.delimiter); - } - } - - buf += colWidthInBytes; - - break; - } - - case CalpontSystemCatalog::VARCHAR: - { - Field* field = table->field[colpos]; - - uint32_t colWidthInBytes = - ci.columnTypes[colpos].colWidth * field->charset()->mbmaxlen; - - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - { - fprintf(ci.filePtr, "%c", ci.delimiter); - - if (colWidthInBytes < 256) - { - buf++; - } - else - { - buf = buf + 2 ; - } - } - else - { - // Maximum number of bytes allowed for a VARCHAR - // field is 65532, so the max length fits in 2 bytes. - // dataLength is length in bytes, not length in chars - uint16_t dataLength = 0; - - if (colWidthInBytes < 256) - { - dataLength = *(uint8_t*) buf; - buf++; - } - else - { - dataLength = *(uint16_t*) buf; - buf = buf + 2 ; - } - - escape.assign((char*)buf, dataLength); - boost::replace_all(escape, "\\", "\\\\"); - fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, (int)escape.length(), escape.c_str(), ci.enclosed_by, ci.delimiter); - } - - buf += colWidthInBytes; - - break; - } - - case CalpontSystemCatalog::BIGINT: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - fprintf(ci.filePtr, "%lld%c", *((long long*)buf), ci.delimiter); - - buf += 8; - break; - } - - case CalpontSystemCatalog::UBIGINT: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - fprintf(ci.filePtr, "%llu%c", *((long long unsigned*)buf), ci.delimiter); - - buf += 8; - break; - } - - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::MEDINT: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - fprintf(ci.filePtr, "%d%c", *((int32_t*)buf), ci.delimiter); - - buf += 4; - break; - } - - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UMEDINT: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - { - fprintf(ci.filePtr, "%u%c", *((uint32_t*)buf), ci.delimiter); - //printf("%u|", *((uint32_t*)buf)); - //cout << *((uint32_t*)buf) << endl; - } - - buf += 4; - break; - } - - case CalpontSystemCatalog::SMALLINT: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - fprintf(ci.filePtr, "%d%c", *((int16_t*)buf), ci.delimiter); - - buf += 2; - break; - } - - case CalpontSystemCatalog::USMALLINT: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - fprintf(ci.filePtr, "%u%c", *((uint16_t*)buf), ci.delimiter); - - buf += 2; - break; - } - - case CalpontSystemCatalog::TINYINT: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - fprintf(ci.filePtr, "%d%c", *((int8_t*)buf), ci.delimiter); - - buf += 1; - break; - } - - case CalpontSystemCatalog::UTINYINT: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - fprintf(ci.filePtr, "%u%c", *((uint8_t*)buf), ci.delimiter); - - buf += 1; - break; - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - { - float val = *((float*)buf); - - if ((fabs(val) > (1.0 / IDB_pow[4])) && (fabs(val) < (float) IDB_pow[6])) - { - fprintf(ci.filePtr, "%.7f%c", val, ci.delimiter); - } - else - { - fprintf(ci.filePtr, "%e%c", val, ci.delimiter); - } - - - //fprintf(ci.filePtr, "%.7g|", *((float*)buf)); - //printf("%.7f|", *((float*)buf)); - } - - buf += 4; - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - { - fprintf(ci.filePtr, "%.15g%c", *((double*)buf), ci.delimiter); - //printf("%.15g|", *((double*)buf)); - } - - buf += 8; - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - fprintf(ci.filePtr, "%c", ci.delimiter); - else - { - fprintf(ci.filePtr, "%.15Lg%c", *((long double*)buf), ci.delimiter); - //printf("%.15g|", *((double*)buf)); - } - - buf += 8; - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - uint bytesBefore = 1; - uint totalBytes = 9; - - switch (ci.columnTypes[colpos].precision) - { - case 18: - case 17: - case 16: - { - totalBytes = 8; - break; - } - - case 15: - case 14: - { - totalBytes = 7; - break; - } - - case 13: - case 12: - { - totalBytes = 6; - break; - } - - case 11: - { - totalBytes = 5; - break; - } - - case 10: - { - totalBytes = 5; - break; - } - - case 9: - case 8: - case 7: - { - totalBytes = 4; - break; - } - - case 6: - case 5: - { - totalBytes = 3; - break; - } - - case 4: - case 3: - { - totalBytes = 2; - break; - } - - case 2: - case 1: - { - totalBytes = 1; - break; - } - - default: - break; - } - - switch (ci.columnTypes[colpos].scale) - { - case 0: - { - bytesBefore = totalBytes; - break; - } - - case 1: //1 byte for digits after decimal point - { - if ((ci.columnTypes[colpos].precision != 16) && (ci.columnTypes[colpos].precision != 14) - && (ci.columnTypes[colpos].precision != 12) && (ci.columnTypes[colpos].precision != 10) - && (ci.columnTypes[colpos].precision != 7) && (ci.columnTypes[colpos].precision != 5) - && (ci.columnTypes[colpos].precision != 3) && (ci.columnTypes[colpos].precision != 1)) - totalBytes++; - - bytesBefore = totalBytes - 1; - break; - } - - case 2: //1 byte for digits after decimal point - { - if ((ci.columnTypes[colpos].precision == 18) || (ci.columnTypes[colpos].precision == 9)) - totalBytes++; - - bytesBefore = totalBytes - 1; - break; - } - - case 3: //2 bytes for digits after decimal point - { - if ((ci.columnTypes[colpos].precision != 16) && (ci.columnTypes[colpos].precision != 14) - && (ci.columnTypes[colpos].precision != 12) && (ci.columnTypes[colpos].precision != 7) - && (ci.columnTypes[colpos].precision != 5) && (ci.columnTypes[colpos].precision != 3)) - totalBytes++; - - bytesBefore = totalBytes - 2; - break; - } - - case 4: - { - if ((ci.columnTypes[colpos].precision == 18) || (ci.columnTypes[colpos].precision == 11) - || (ci.columnTypes[colpos].precision == 9)) - totalBytes++; - - bytesBefore = totalBytes - 2; - break; - - } - - case 5: - { - if ((ci.columnTypes[colpos].precision != 16) && (ci.columnTypes[colpos].precision != 14) - && (ci.columnTypes[colpos].precision != 7) && (ci.columnTypes[colpos].precision != 5)) - totalBytes++; - - bytesBefore = totalBytes - 3; - break; - } - - case 6: - { - if ((ci.columnTypes[colpos].precision == 18) || (ci.columnTypes[colpos].precision == 13) - || (ci.columnTypes[colpos].precision == 11) || (ci.columnTypes[colpos].precision == 9)) - totalBytes++; - - bytesBefore = totalBytes - 3; - break; - } - - case 7: - { - if ((ci.columnTypes[colpos].precision != 16) && (ci.columnTypes[colpos].precision != 7)) - totalBytes++; - - bytesBefore = totalBytes - 4; - break; - } - - case 8: - { - if ((ci.columnTypes[colpos].precision == 18) || (ci.columnTypes[colpos].precision == 15) - || (ci.columnTypes[colpos].precision == 13) || (ci.columnTypes[colpos].precision == 11) - || (ci.columnTypes[colpos].precision == 9)) - totalBytes++; - - bytesBefore = totalBytes - 4;; - break; - } - - case 9: - { - bytesBefore = totalBytes - 4;; - break; - } - - case 10: - { - if ((ci.columnTypes[colpos].precision != 16) && (ci.columnTypes[colpos].precision != 14) - && (ci.columnTypes[colpos].precision != 12) && (ci.columnTypes[colpos].precision != 10)) - totalBytes++; - - bytesBefore = totalBytes - 5;; - break; - } - - case 11: - { - if (ci.columnTypes[colpos].precision == 18) - totalBytes++; - - bytesBefore = totalBytes - 5; - break; - } - - case 12: - { - if ((ci.columnTypes[colpos].precision != 16) && (ci.columnTypes[colpos].precision != 14) - && (ci.columnTypes[colpos].precision != 12)) - totalBytes++; - - bytesBefore = totalBytes - 6; - break; - } - - case 13: - { - if (ci.columnTypes[colpos].precision == 18) - totalBytes++; - - bytesBefore = totalBytes - 6; - break; - } - - case 14: - { - if ((ci.columnTypes[colpos].precision != 16) && (ci.columnTypes[colpos].precision != 14)) - totalBytes++; - - bytesBefore = totalBytes - 7; - break; - } - - case 15: - { - if (ci.columnTypes[colpos].precision == 18) - totalBytes++; - - bytesBefore = totalBytes - 7; - break; - } - - case 16: - { - if (ci.columnTypes[colpos].precision != 16) - totalBytes++; - - bytesBefore = totalBytes - 8; - break; - } - - case 17: - { - if (ci.columnTypes[colpos].precision == 18) - totalBytes++; - - bytesBefore = totalBytes - 8; - break; - } - - case 18: - { - bytesBefore = totalBytes - 8; - break; - } - - default: - break; - } - - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - { - fprintf(ci.filePtr, "%c", ci.delimiter); - //printf("|"); - } - else - { - uint32_t mask [5] = {0, 0xFF, 0xFFFF, 0xFFFFFF, 0xFFFFFFFF}; - char neg = '-'; - - if (ci.columnTypes[colpos].scale == 0) - { - const uchar* tmpBuf = buf; - //test flag bit for sign - bool posNum = tmpBuf[0] & (0x80); - uchar tmpChr = tmpBuf[0]; - tmpChr ^= 0x80; //flip the bit - int32_t tmp1 = tmpChr; - - if (totalBytes > 4) - { - for (uint i = 1; i < (totalBytes - 4); i++) - { - tmp1 = (tmp1 << 8) + tmpBuf[i]; - } - - if (( tmp1 != 0 ) && (tmp1 != -1)) - { - if (!posNum) - { - tmp1 = mask[totalBytes - 4] - tmp1; - - if (tmp1 != 0) - { - fprintf(ci.filePtr, "%c", neg); - //printf("%c", neg); - } - } - - if (tmp1 != 0) - { - fprintf(ci.filePtr, "%d", tmp1); - ////printf("%d", tmp1); - } - } - - int32_t tmp2 = tmpBuf[totalBytes - 4]; - - for (uint i = (totalBytes - 3); i < totalBytes; i++) - { - tmp2 = (tmp2 << 8) + tmpBuf[i]; - } - - if ( tmp1 != 0 ) - { - if (!posNum) - { - tmp2 = mask[4] - tmp2; - - if (tmp1 == -1) - { - fprintf(ci.filePtr, "%c", neg); - fprintf(ci.filePtr, "%d%c", tmp2, ci.delimiter); - ////printf("%c", neg); - //////printf( "%d|", tmp2); - } - else - { - fprintf(ci.filePtr, "%09u%c", tmp2, ci.delimiter); - ////printf("%09u|", tmp2); - } - } - else - { - fprintf(ci.filePtr, "%09u%c", tmp2, ci.delimiter); - //printf("%09u|", tmp2); - } - } - else - { - if (!posNum) - { - tmp2 = mask[4] - tmp2; - fprintf(ci.filePtr, "%c", neg); - //printf("%c", neg); - } - - fprintf(ci.filePtr, "%d%c", tmp2, ci.delimiter); - //printf("%d|", tmp2); - } - } - else - { - for (uint i = 1; i < totalBytes; i++) - { - tmp1 = (tmp1 << 8) + tmpBuf[i]; - } - - if (!posNum) - { - tmp1 = mask[totalBytes] - tmp1; - fprintf(ci.filePtr, "%c", neg); - //printf("%c", neg); - } - - fprintf(ci.filePtr, "%d%c", tmp1, ci.delimiter); - //printf("%d|", tmp1); - } - } - else - { - const uchar* tmpBuf = buf; - //test flag bit for sign - bool posNum = tmpBuf[0] & (0x80); - uchar tmpChr = tmpBuf[0]; - tmpChr ^= 0x80; //flip the bit - int32_t tmp1 = tmpChr; - - //fetch the digits before decimal point - if (bytesBefore == 0) - { - if (!posNum) - { - fprintf(ci.filePtr, "%c", neg); - //printf("%c", neg); - } - - fprintf(ci.filePtr, "0."); - //printf("0."); - } - else if (bytesBefore > 4) - { - for (uint i = 1; i < (bytesBefore - 4); i++) - { - tmp1 = (tmp1 << 8) + tmpBuf[i]; - } - - if (!posNum) - { - tmp1 = mask[bytesBefore - 4] - tmp1; - } - - if (( tmp1 != 0 ) && (tmp1 != -1)) - { - if (!posNum) - { - fprintf(ci.filePtr, "%c", neg); - //printf("%c", neg); - } - - fprintf(ci.filePtr, "%d", tmp1); - //printf("%d", tmp1); - } - - tmpBuf += (bytesBefore - 4); - int32_t tmp2 = *((int32_t*)tmpBuf); - tmp2 = ntohl(tmp2); - - if ( tmp1 != 0 ) - { - if (!posNum) - { - tmp2 = mask[4] - tmp2; - } - - if (tmp1 == -1) - { - fprintf(ci.filePtr, "%c", neg); - fprintf(ci.filePtr, "%d.", tmp2); - //printf("%c", neg); - //printf("%d.", tmp2); - } - else - { - fprintf(ci.filePtr, "%09u.", tmp2); - //printf("%09u.", tmp2); - } - } - else - { - if (!posNum) - { - tmp2 = mask[4] - tmp2; - fprintf(ci.filePtr, "%c", neg); - //printf("%c", neg); - } - - fprintf(ci.filePtr, "%d.", tmp2); - //printf("%d.", tmp2); - } - } - else - { - for (uint i = 1; i < bytesBefore; i++) - { - tmp1 = (tmp1 << 8) + tmpBuf[i]; - } - - if (!posNum) - { - tmp1 = mask[bytesBefore] - tmp1; - fprintf(ci.filePtr, "%c", neg); - //printf("%c", neg); - } - - fprintf(ci.filePtr, "%d.", tmp1); - //printf("%d.", tmp1); - } - - //fetch the digits after decimal point - int32_t tmp2 = 0; - - if (bytesBefore > 4) - tmpBuf += 4; - else - tmpBuf += bytesBefore; - - tmp2 = tmpBuf[0]; - - if ((totalBytes - bytesBefore) < 5) - { - for (uint j = 1; j < (totalBytes - bytesBefore); j++) - { - tmp2 = (tmp2 << 8) + tmpBuf[j]; - } - - int8_t digits = ci.columnTypes[colpos].scale - 9; //9 digits is a 4 bytes chunk - - if ( digits <= 0 ) - digits = ci.columnTypes[colpos].scale; - - if (!posNum) - { - tmp2 = mask[totalBytes - bytesBefore] - tmp2; - } - - fprintf(ci.filePtr, "%0*u%c", digits, tmp2, ci.delimiter); - //printf("%0*u|", digits, tmp2); - } - else - { - for (uint j = 1; j < 4; j++) - { - tmp2 = (tmp2 << 8) + tmpBuf[j]; - } - - if (!posNum) - { - tmp2 = mask[4] - tmp2; - } - - fprintf(ci.filePtr, "%09u", tmp2); - //printf("%09u", tmp2); - - tmpBuf += 4; - int32_t tmp3 = tmpBuf[0]; - - for (uint j = 1; j < (totalBytes - bytesBefore - 4); j++) - { - tmp3 = (tmp3 << 8) + tmpBuf[j]; - } - - int8_t digits = ci.columnTypes[colpos].scale - 9; //9 digits is a 4 bytes chunk - - if ( digits < 0 ) - digits = ci.columnTypes[colpos].scale; - - if (!posNum) - { - tmp3 = mask[totalBytes - bytesBefore - 4] - tmp3; - } - - fprintf(ci.filePtr, "%0*u%c", digits, tmp3, ci.delimiter); - //printf("%0*u|", digits, tmp3); - } - } - } - - buf += totalBytes; - break; - } - - case CalpontSystemCatalog::VARBINARY: - { - // For a VARBINARY field, ci.columnTypes[colpos].colWidth == colWidthInBytes - if (nullVal && (ci.columnTypes[colpos].constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) - { - fprintf(ci.filePtr, "%c", ci.delimiter); - - if (ci.columnTypes[colpos].colWidth < 256) - { - buf++; - } - else - { - buf = buf + 2; - } - } - else - { - // Maximum number of bytes allowed for a VARBINARY - // field is 65532, so the max length fits in 2 bytes. - // dataLength is length in bytes, not length in chars - uint16_t dataLength = 0; - - if (ci.columnTypes[colpos].colWidth < 256) - { - dataLength = *(uint8_t*) buf; - buf++; - } - else - { - dataLength = *(uint16_t*) buf; - buf = buf + 2 ; - } - - const uchar* tmpBuf = buf; - - for (int32_t i = 0; i < dataLength; i++) - { - fprintf(ci.filePtr, "%02x", *(uint8_t*)tmpBuf); - tmpBuf++; - } - - fprintf(ci.filePtr, "%c", ci.delimiter); - } - - buf += ci.columnTypes[colpos].colWidth; - - break; - } - - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - { - // MCOL-4005 Note that we don't handle nulls as a special - // case here as we do for other datatypes, the below works - // as expected for nulls. - // dataLength is length in bytes, not length in chars - uint32_t dataLength = 0; - uintptr_t* dataptr; - uchar* ucharptr; - - bool isBlob = - ci.columnTypes[colpos].colDataType == CalpontSystemCatalog::BLOB; - - Field* field = table->field[colpos]; - - uint32_t colWidthInBytes = isBlob ? ci.columnTypes[colpos].colWidth : - ci.columnTypes[colpos].colWidth * field->charset()->mbmaxlen; - - if (!isBlob && field->char_length() == 65535) - { - // Special case for TEXT field without default length, - // such as: - // CREATE TABLE mcol4364 (a TEXT); - // Here, char_length() represents the number of bytes, - // not number of characters. - dataLength = *(uint16_t*) buf; - buf += 2; - } - else if (colWidthInBytes < 256) - { - dataLength = *(uint8_t*) buf; - buf++; - } - else if (colWidthInBytes < 65536) - { - dataLength = *(uint16_t*) buf; - buf += 2; - } - else if (colWidthInBytes < 16777216) - { - dataLength = *(uint16_t*) buf; - dataLength |= ((int) buf[2]) << 16; - buf += 3; - } - else - { - dataLength = *(uint32_t*) buf; - buf += 4; - } - - // buf contains pointer to blob, for example: - // (gdb) p (char*)*(uintptr_t*)buf - // $43 = 0x7f68500c58f8 "hello world" - - dataptr = (uintptr_t*)buf; - ucharptr = (uchar*)*dataptr; - buf += sizeof(uintptr_t); - - if (isBlob) - { - for (uint32_t i = 0; i < dataLength; i++) - { - fprintf(ci.filePtr, "%02x", *(uint8_t*)ucharptr); - ucharptr++; - } - - fprintf(ci.filePtr, "%c", ci.delimiter); - } - else - { - // TEXT Column - escape.assign((char*)ucharptr, dataLength); - boost::replace_all(escape, "\\", "\\\\"); - fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, (int)escape.length(), escape.c_str(), ci.enclosed_by, ci.delimiter); - } - - break; - - } - - default: // treat as int64 - { - break; - } + datatypes::ColBatchWriter writer(ci.filePtr, ci.delimiter, + ci.enclosed_by, ci.utf8); + datatypes::WriteBatchFieldMariaDB field(table->field[colpos], colType); + idbassert(table == table->field[colpos]->table); + buf+= h->ColWriteBatch(&field, buf, nullVal, writer); } - colpos++; } } diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index 666bb467a..a5daf7048 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -3111,7 +3111,7 @@ CalpontSystemCatalog::ColType colType_MysqlToIDB (const Item* item) unsigned int precision = idp->decimal_precision(); unsigned int scale = idp->decimal_scale(); - datatypes::Decimal::setDecimalScalePrecision(ct, precision, scale); + ct.setDecimalScalePrecision(precision, scale); break; } @@ -3608,7 +3608,7 @@ ArithmeticColumn* buildArithmeticColumn( const CalpontSystemCatalog::ColType& rightColType = pt->right()->data()->resultType(); // Only tinker with the type if all columns involved are decimal - if (datatypes::Decimal::isDecimalOperands(mysqlType.colDataType, + if (datatypes::isDecimalOperands(mysqlType.colDataType, leftColType.colDataType, rightColType.colDataType)) { int32_t leftColWidth = leftColType.colWidth; @@ -3632,7 +3632,7 @@ ArithmeticColumn* buildArithmeticColumn( unsigned int precision = idp->decimal_precision(); unsigned int scale = idp->decimal_scale(); - datatypes::Decimal::setDecimalScalePrecisionHeuristic(mysqlType, precision, scale); + mysqlType.setDecimalScalePrecisionHeuristic(precision, scale); if (mysqlType.scale < scale1) mysqlType.scale = scale1; @@ -4087,7 +4087,7 @@ ReturnedColumn* buildFunctionColumn( { for (size_t i = 0; i < funcParms.size(); i++) { - if (datatypes::Decimal::isWideDecimalType(funcParms[i]->data()->resultType())) + if (funcParms[i]->data()->resultType().isWideDecimalType()) { fc->resultType().colWidth = datatypes::MAXDECIMALWIDTH; break; @@ -4475,18 +4475,18 @@ SimpleColumn* buildSimpleColumn(Item_field* ifp, gp_walk_info& gwi) return buildSimpleColFromDerivedTable(gwi, ifp); CalpontSystemCatalog::ColType ct; - bool columnStore = true; + datatypes::SimpleColumnParam prm(gwi.sessionid, true); try { // check foreign engine if (ifp->cached_table && ifp->cached_table->table) - columnStore = isMCSTable(ifp->cached_table->table); + prm.columnStore(isMCSTable(ifp->cached_table->table)); // @bug4509. ifp->cached_table could be null for myisam sometimes else if (ifp->field && ifp->field->table) - columnStore = isMCSTable(ifp->field->table); + prm.columnStore(isMCSTable(ifp->field->table)); - if (columnStore) + if (prm.columnStore()) { ct = gwi.csc->colType( gwi.csc->lookupOID(make_tcn(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str))); @@ -4503,75 +4503,12 @@ SimpleColumn* buildSimpleColumn(Item_field* ifp, gp_walk_info& gwi) return NULL; } - SimpleColumn* sc = NULL; + const datatypes::DatabaseQualifiedColumnName name(ifp->db_name.str, + bestTableName(ifp), + ifp->field_name.str); + const datatypes::TypeHandler *h= ct.typeHandler(); + SimpleColumn *sc = h->newSimpleColumn(name, ct, prm); - switch (ct.colDataType) - { - case CalpontSystemCatalog::TINYINT: - if (ct.scale == 0) - sc = new SimpleColumn_INT<1>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - else - { - sc = new SimpleColumn_Decimal<1>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - ct.colDataType = CalpontSystemCatalog::DECIMAL; - } - - break; - - case CalpontSystemCatalog::SMALLINT: - if (ct.scale == 0) - sc = new SimpleColumn_INT<2>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - else - { - sc = new SimpleColumn_Decimal<2>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - ct.colDataType = CalpontSystemCatalog::DECIMAL; - } - - break; - - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::MEDINT: - if (ct.scale == 0) - sc = new SimpleColumn_INT<4>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - else - { - sc = new SimpleColumn_Decimal<4>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - ct.colDataType = CalpontSystemCatalog::DECIMAL; - } - - break; - - case CalpontSystemCatalog::BIGINT: - if (ct.scale == 0) - sc = new SimpleColumn_INT<8>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - else - { - sc = new SimpleColumn_Decimal<8>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - ct.colDataType = CalpontSystemCatalog::DECIMAL; - } - - break; - - case CalpontSystemCatalog::UTINYINT: - sc = new SimpleColumn_UINT<1>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - break; - - case CalpontSystemCatalog::USMALLINT: - sc = new SimpleColumn_UINT<2>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - break; - - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UMEDINT: - sc = new SimpleColumn_UINT<4>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - break; - - case CalpontSystemCatalog::UBIGINT: - sc = new SimpleColumn_UINT<8>(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - break; - - default: - sc = new SimpleColumn(ifp->db_name.str, bestTableName(ifp), ifp->field_name.str, columnStore, gwi.sessionid); - } sc->resultType(ct); sc->charsetNumber(ifp->collation.collation->number); string tbname(ifp->table_name.str); @@ -4588,10 +4525,10 @@ SimpleColumn* buildSimpleColumn(Item_field* ifp, gp_walk_info& gwi) sc->viewName(getViewName(ifp->cached_table), lower_case_table_names); sc->alias(ifp->name.str); - sc->isColumnStore(columnStore); + sc->isColumnStore(prm.columnStore()); sc->timeZone(gwi.thd->variables.time_zone->get_name()->ptr()); - if (!columnStore && ifp->field) + if (!prm.columnStore() && ifp->field) sc->oid(ifp->field->field_index + 1); // ExeMgr requires offset started from 1 if (ifp->depended_from) @@ -4983,7 +4920,7 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) isp->sum_func() == Item_sum::SUM_DISTINCT_FUNC) { CalpontSystemCatalog::ColType ct = parm->resultType(); - if (datatypes::Decimal::isWideDecimalType(ct)) + if (ct.isWideDecimalType()) { uint32_t precision = ct.precision; uint32_t scale = ct.scale; @@ -5051,7 +4988,7 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) else if (bIsConst && hasDecimalConst && isAvg) { CalpontSystemCatalog::ColType ct = parm->resultType(); - if (datatypes::Decimal::isWideDecimalType(constValPrecision)) + if (datatypes::Decimal::isWideDecimalTypeByPrecision(constValPrecision)) { ct.precision = constValPrecision; ct.scale = constValScale; @@ -5065,8 +5002,7 @@ ReturnedColumn* buildAggregateColumn(Item* item, gp_walk_info& gwi) } // adjust decimal result type according to internalDecimalScale - bool isWideDecimal = - datatypes::Decimal::isWideDecimalType(ac->resultType()); + bool isWideDecimal = ac->resultType().isWideDecimalType(); // This must be also valid for UDECIMAL if (!isWideDecimal && gwi.internalDecimalScale >= 0 && ac->resultType().colDataType == CalpontSystemCatalog::DECIMAL) @@ -7413,7 +7349,7 @@ int getSelectPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, gwi.returnedCols[i]->hasAggregate(true); } - gwi.returnedCols[i]->resultType(dataconvert::DataConvert::convertUnionColType(coltypes)); + gwi.returnedCols[i]->resultType(CalpontSystemCatalog::ColType::convertUnionColType(coltypes)); } } @@ -9163,7 +9099,7 @@ int getGroupPlan(gp_walk_info& gwi, SELECT_LEX& select_lex, SCSEP& csep, cal_gro gwi.returnedCols[i]->hasAggregate(true); } - gwi.returnedCols[i]->resultType(dataconvert::DataConvert::convertUnionColType(coltypes)); + gwi.returnedCols[i]->resultType(CalpontSystemCatalog::ColType::convertUnionColType(coltypes)); } } diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 2724d2ba1..e94f19595 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -138,6 +138,8 @@ using namespace funcexp; #include "columnstoreversion.h" #include "ha_mcs_sysvars.h" +#include "ha_mcs_datatype.h" + namespace cal_impl_if { extern bool nonConstFunc(Item_func* ifp); @@ -249,108 +251,6 @@ void force_close_fep_conn(THD *thd, cal_connection_info* ci, bool check_prev_rc ci->cal_conn_hndl = 0; } -void storeNumericField(Field** f, int64_t value, CalpontSystemCatalog::ColType& ct) -{ - // unset null bit first - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - // For unsigned, use the ColType returned in the row rather than the - // unsigned_flag set by mysql. This is because mysql gets it wrong for SUM() - // Hopefully, in all other cases we get it right. - switch ((*f)->type()) - { - case MYSQL_TYPE_NEWDECIMAL: - { - // @bug4388 stick to InfiniDB's scale in case mysql gives wrong scale due - // to create vtable limitation. - //if (f2->dec < ct.scale) - // f2->dec = ct.scale; - - // WIP MCOL-641 - // This is too much - char buf[256]; - dataconvert::DataConvert::decimalToString(value, (unsigned)ct.scale, buf, 256, ct.colDataType); - (*f)->store(buf, strlen(buf), (*f)->charset()); - break; - } - - case MYSQL_TYPE_TINY: //TINYINT type - { - Field_tiny* f2 = (Field_tiny*)*f; - longlong int_val = (longlong)value; - (*f)->store(int_val, f2->unsigned_flag); - break; - } - - case MYSQL_TYPE_SHORT: //SMALLINT type - { - Field_short* f2 = (Field_short*)*f; - longlong int_val = (longlong)value; - (*f)->store(int_val, f2->unsigned_flag); - break; - } - - case MYSQL_TYPE_INT24: //MEDINT type - { - Field_medium* f2 = (Field_medium*)*f; - longlong int_val = (longlong)value; - (*f)->store(int_val, f2->unsigned_flag); - break; - } - - case MYSQL_TYPE_LONG: //INT type - { - Field_long* f2 = (Field_long*)*f; - longlong int_val = (longlong)value; - (*f)->store(int_val, f2->unsigned_flag); - break; - } - - case MYSQL_TYPE_LONGLONG: //BIGINT type - { - Field_longlong* f2 = (Field_longlong*)*f; - longlong int_val = (longlong)value; - (*f)->store(int_val, f2->unsigned_flag); - break; - } - - case MYSQL_TYPE_FLOAT: // FLOAT type - { - float float_val = *(float*)(&value); - (*f)->store(float_val); - break; - } - - case MYSQL_TYPE_DOUBLE: // DOUBLE type - { - double double_val = *(double*)(&value); - (*f)->store(double_val); - break; - } - - case MYSQL_TYPE_VARCHAR: - { - char tmp[25]; - if (ct.colDataType == CalpontSystemCatalog::DECIMAL) - dataconvert::DataConvert::decimalToString(value, (unsigned)ct.scale, tmp, 25, ct.colDataType); - else - snprintf(tmp, 25, "%lld", (long long)value); - - (*f)->store(tmp, strlen(tmp), (*f)->charset()); - break; - } - - default: - { - Field_longlong* f2 = (Field_longlong*)*f; - longlong int_val = (longlong)value; - (*f)->store(int_val, f2->unsigned_flag); - break; - } - } -} - // // @bug 2244. Log exception related to lost connection to ExeMgr. // Log exception error from calls to sm::tpl_scan_fetch in fetchNextRow() @@ -391,19 +291,6 @@ void tpl_scan_fetch_LogException( cal_table_info& ti, cal_connection_info* ci, s sesID << "; " << connHndl << "; rowsReturned: " << rowsRet << endl; } -const char hexdig[] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', }; - -int vbin2hex(const uint8_t* p, const unsigned l, char* o) -{ - for (unsigned i = 0; i < l; i++, p++) - { - *o++ = hexdig[*p >> 4]; - *o++ = hexdig[*p & 0xf]; - } - - return 0; -} - // Table Map is used by both cond_push and table mode processing // Entries made by cond_push don't have csep though. // When @@ -421,6 +308,7 @@ bool onlyOneTableinTM(cal_impl_if::cal_connection_info* ci) return true; } + int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool handler_flag = false) { int rc = HA_ERR_END_OF_FILE; @@ -473,9 +361,6 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h } std::vector& colTypes = ti.tpl_scan_ctx->ctp; - int64_t intColVal = 0; - uint64_t uintColVal = 0; - char tmp[256]; RowGroup* rowGroup = ti.tpl_scan_ctx->rowGroup; @@ -548,319 +433,25 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h colType.colDataType == CalpontSystemCatalog::VARCHAR || colType.colDataType == CalpontSystemCatalog::VARBINARY) { - (*f)->store(tmp, 0, (*f)->charset()); + (*f)->store("", 0, (*f)->charset()); } continue; } - // fetch and store data - switch (colType.colDataType) + const datatypes::TypeHandler *h= colType.typeHandler(); + if (!h) { - case CalpontSystemCatalog::DATE: - { - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - intColVal = row.getUintField<4>(s); - DataConvert::dateToString(intColVal, tmp, 255); - (*f)->store(tmp, strlen(tmp), (*f)->charset()); - break; - } - - case CalpontSystemCatalog::DATETIME: - { - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - intColVal = row.getUintField<8>(s); - DataConvert::datetimeToString(intColVal, tmp, 255, colType.precision); - (*f)->store(tmp, strlen(tmp), (*f)->charset()); - break; - } - - case CalpontSystemCatalog::TIME: - { - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - intColVal = row.getUintField<8>(s); - DataConvert::timeToString(intColVal, tmp, 255, colType.precision); - (*f)->store(tmp, strlen(tmp), (*f)->charset()); - break; - } - - case CalpontSystemCatalog::TIMESTAMP: - { - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - intColVal = row.getUintField<8>(s); - DataConvert::timestampToString(intColVal, tmp, 255, current_thd->variables.time_zone->get_name()->ptr(), colType.precision); - (*f)->store(tmp, strlen(tmp), (*f)->charset()); - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - { - switch (colType.colWidth) - { - case 1: - intColVal = row.getUintField<1>(s); - (*f)->store((char*)(&intColVal), strlen((char*)(&intColVal)), (*f)->charset()); - break; - - case 2: - intColVal = row.getUintField<2>(s); - (*f)->store((char*)(&intColVal), strlen((char*)(&intColVal)), (*f)->charset()); - break; - - case 4: - intColVal = row.getUintField<4>(s); - (*f)->store((char*)(&intColVal), strlen((char*)(&intColVal)), (*f)->charset()); - break; - - case 8: - //make sure we don't send strlen off into the weeds... - intColVal = row.getUintField<8>(s); - memcpy(tmp, &intColVal, 8); - tmp[8] = 0; - (*f)->store(tmp, strlen(tmp), (*f)->charset()); - break; - - default: - (*f)->store((const char*)row.getStringPointer(s), row.getStringLength(s), (*f)->charset()); - } - - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - break; - } - - case CalpontSystemCatalog::VARBINARY: - { - if (get_varbin_always_hex(current_thd)) - { - uint32_t l; - const uint8_t* p = row.getVarBinaryField(l, s); - uint32_t ll = l * 2; - boost::scoped_array sca(new char[ll]); - vbin2hex(p, l, sca.get()); - (*f)->store(sca.get(), ll, (*f)->charset()); - } - else - (*f)->store((const char*)row.getVarBinaryField(s), row.getVarBinaryLength(s), (*f)->charset()); - - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - break; - } - - case CalpontSystemCatalog::BIGINT: - { - intColVal = row.getIntField<8>(s); - storeNumericField(f, intColVal, colType); - break; - } - - case CalpontSystemCatalog::UBIGINT: - { - uintColVal = row.getUintField<8>(s); - storeNumericField(f, uintColVal, colType); - break; - } - - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::MEDINT: - { - intColVal = row.getIntField<4>(s); - storeNumericField(f, intColVal, colType); - break; - } - - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UMEDINT: - { - uintColVal = row.getUintField<4>(s); - storeNumericField(f, uintColVal, colType); - break; - } - - case CalpontSystemCatalog::SMALLINT: - { - intColVal = row.getIntField<2>(s); - storeNumericField(f, intColVal, colType); - break; - } - - case CalpontSystemCatalog::USMALLINT: - { - uintColVal = row.getUintField<2>(s); - storeNumericField(f, uintColVal, colType); - break; - } - - case CalpontSystemCatalog::TINYINT: - { - intColVal = row.getIntField<1>(s); - storeNumericField(f, intColVal, colType); - break; - } - - case CalpontSystemCatalog::UTINYINT: - { - uintColVal = row.getUintField<1>(s); - storeNumericField(f, uintColVal, colType); - break; - } - - //In this case, we're trying to load a double output column with float data. This is the - // case when you do sum(floatcol), e.g. - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - float dl = row.getFloatField(s); - - if (dl == std::numeric_limits::infinity()) - continue; - - // bug 3485, reserve enough space for the longest float value - // -3.402823466E+38 to -1.175494351E-38, 0, and - // 1.175494351E-38 to 3.402823466E+38. - (*f)->field_length = 40; - (*f)->store(dl); - - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - break; - } - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - double dl = row.getDoubleField(s); - - if (dl == std::numeric_limits::infinity()) - continue; - - if ((*f)->type() == MYSQL_TYPE_NEWDECIMAL) - { - char buf[310]; - // reserve enough space for the longest double value - // -1.7976931348623157E+308 to -2.2250738585072014E-308, 0, and - // 2.2250738585072014E-308 to 1.7976931348623157E+308. - snprintf(buf, 310, "%.18g", dl); - (*f)->store(buf, strlen(buf), (*f)->charset()); - } - else - { - // The server converts dl=-0 to dl=0 in (*f)->store(). - // This happens in the call to truncate_double(). - // This is an unexpected behaviour, so we directly store the - // double value using the lower level float8store() function. - // TODO Remove this when (*f)->store() handles this properly. - (*f)->field_length = 310; - if (dl == 0) - float8store((*f)->ptr,dl); - else - (*f)->store(dl); - } - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - long double dl = row.getLongDoubleField(s); - if (dl == std::numeric_limits::infinity()) - { - continue; - } - - if ((*f)->type() == MYSQL_TYPE_NEWDECIMAL) - { - char buf[310]; - snprintf(buf, 310, "%.20Lg", dl); - (*f)->store(buf, strlen(buf), (*f)->charset()); - } - else - { - // reserve enough space for the longest double value - // -1.7976931348623157E+308 to -2.2250738585072014E-308, 0, and - // 2.2250738585072014E-308 to 1.7976931348623157E+308. - (*f)->field_length = 310; - (*f)->store(static_cast(dl)); - } - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - break; - } - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - if (LIKELY(colType.colWidth == datatypes::MAXDECIMALWIDTH)) - { - // unset null bit first - // Might be redundant - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - int128_t* dec; - // We won't have more than [+-][0][.] + up to 38 digits - char buf[utils::MAXLENGTH16BYTES]; - - dec = row.getBinaryField(s); - dataconvert::DataConvert::decimalToString(dec, - (unsigned)colType.scale, buf, - sizeof(buf), colType.colDataType); - - Field_new_decimal* f2 = (Field_new_decimal*)*f; - f2->store(buf, strlen(buf), f2->charset()); - } - else - { - intColVal = row.getIntField(s); - storeNumericField(f, intColVal, colType); - } - break; - } - - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - { - Field_blob* f2 = (Field_blob*)*f; - f2->set_ptr(row.getVarBinaryLength(s), (unsigned char*)row.getVarBinaryField(s)); - - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - break; - } - case CalpontSystemCatalog::BINARY: - { - Field_varstring* f2 = (Field_varstring*)*f; - // WIP MCOL-641 Binary representation could contain \0. - char* binaryString = row.getBinaryField(s); - f2->store(binaryString, colType.colWidth, f2->charset()); - - if ((*f)->null_ptr) - *(*f)->null_ptr &= ~(*f)->null_bit; - - break; - } - default: // treat as int64 - { - intColVal = row.getUintField<8>(s); - storeNumericField(f, intColVal, colType); - break; - } + idbassert(0); + (*f)->reset(); + (*f)->set_null(); + } + else + { + // fetch and store data + (*f)->set_notnull(); + datatypes::StoreFieldMariaDB mf(*f, colType); + h->storeValueToField(row, s, &mf); } } diff --git a/dbcon/mysql/ha_mcs_partition.cpp b/dbcon/mysql/ha_mcs_partition.cpp index 17ba58992..c057eecdf 100644 --- a/dbcon/mysql/ha_mcs_partition.cpp +++ b/dbcon/mysql/ha_mcs_partition.cpp @@ -65,12 +65,31 @@ using namespace logging; #include using namespace boost; -#include "mcs_decimal.h" namespace { -const uint8_t ROUND_POS = 0x01; -const uint8_t ROUND_NEG = 0x80; + +datatypes::SimpleValue getStartVal(const datatypes::SessionParam &sp, + const CalpontSystemCatalog::ColType &ct, + const char *val, + datatypes::round_style_t & rfMin) +{ + const datatypes::TypeHandler *h= ct.typeHandler(); + return val ? h->toSimpleValue(sp, ct, val, rfMin) : + h->getMinValueSimple(); +} + + +datatypes::SimpleValue getEndVal(const datatypes::SessionParam &sp, + const CalpontSystemCatalog::ColType &ct, + const char *val, + datatypes::round_style_t & rfMax) +{ + const datatypes::TypeHandler *h= ct.typeHandler(); + return val ? h->toSimpleValue(sp, ct, val, rfMax) : + h->getMaxValueSimple(); +} + //convenience fcn inline uint32_t tid2sid(const uint32_t tid) @@ -101,375 +120,27 @@ void push_warnings(THD* thd, string& warnings) } } + string name(CalpontSystemCatalog::ColType& ct) { - switch (ct.colDataType) - { - case CalpontSystemCatalog::INT: - return "INT"; - - case CalpontSystemCatalog::TINYINT: - return "TINYINT"; - - case CalpontSystemCatalog::MEDINT: - return "MEDINT"; - - case CalpontSystemCatalog::SMALLINT: - return "SMALLINT"; - - case CalpontSystemCatalog::BIGINT: - return "BIGINT"; - - case CalpontSystemCatalog::DATE: - return "DATE"; - - case CalpontSystemCatalog::DATETIME: - return "DATETIME"; - - case CalpontSystemCatalog::TIME: - return "TIME"; - - case CalpontSystemCatalog::TIMESTAMP: - return "TIMESTAMP"; - - case CalpontSystemCatalog::DECIMAL: - return "DECIMAL"; - - case CalpontSystemCatalog::CHAR: - { - ostringstream oss; - oss << "CHAR(" << ct.colWidth << ")"; - return oss.str(); - } - - case CalpontSystemCatalog::VARCHAR: - { - ostringstream oss; - oss << "VARCHAR(" << ct.colWidth << ")"; - return oss.str(); - } - - case CalpontSystemCatalog::FLOAT: - return "FLOAT"; - - case CalpontSystemCatalog::DOUBLE: - return "DOUBLE"; - - case CalpontSystemCatalog::BIT: - return "BIT"; - - case CalpontSystemCatalog::VARBINARY: - return "VARBINARY"; - - case CalpontSystemCatalog::BLOB: - return "BLOB"; - - case CalpontSystemCatalog::TEXT: - return "TEXT"; - - case CalpontSystemCatalog::CLOB: - return "CLOB"; - - case CalpontSystemCatalog::UINT: - return "UINT"; - - case CalpontSystemCatalog::UTINYINT: - return "UTINYINT"; - - case CalpontSystemCatalog::UMEDINT: - return "UMEDINT"; - - case CalpontSystemCatalog::USMALLINT: - return "USMALLINT"; - - case CalpontSystemCatalog::UBIGINT: - return "UBIGINT"; - - case CalpontSystemCatalog::UDECIMAL: - return "UDECIMAL"; - - case CalpontSystemCatalog::UFLOAT: - return "UFLOAT"; - - case CalpontSystemCatalog::UDOUBLE: - return "UDOUBLE"; - - case CalpontSystemCatalog::LONGDOUBLE: - return "LONGDOUBLE"; - - default: - return "Unknown Type"; - } + const datatypes::TypeHandler *h= ct.typeHandler(); + if (!h) + return "Unknown Type"; + return h->print(ct); } + bool CP_type(CalpontSystemCatalog::ColType& ct) { - if (ct.colDataType == CalpontSystemCatalog::INT || - ct.colDataType == CalpontSystemCatalog::TINYINT || - ct.colDataType == CalpontSystemCatalog::MEDINT || - ct.colDataType == CalpontSystemCatalog::SMALLINT || - ct.colDataType == CalpontSystemCatalog::BIGINT || - ct.colDataType == CalpontSystemCatalog::DATE || - ct.colDataType == CalpontSystemCatalog::DATETIME || - ct.colDataType == CalpontSystemCatalog::TIME || - ct.colDataType == CalpontSystemCatalog::TIMESTAMP || - ct.colDataType == CalpontSystemCatalog::DECIMAL || - ct.colDataType == CalpontSystemCatalog::UTINYINT || - ct.colDataType == CalpontSystemCatalog::USMALLINT || - ct.colDataType == CalpontSystemCatalog::UMEDINT || - ct.colDataType == CalpontSystemCatalog::UINT || - ct.colDataType == CalpontSystemCatalog::UBIGINT || - ct.colDataType == CalpontSystemCatalog::UDECIMAL || - (ct.colDataType == CalpontSystemCatalog::CHAR && ct.colWidth <= 8) || - (ct.colDataType == CalpontSystemCatalog::VARCHAR && ct.colWidth <= 7)) - { - return true; - } - - return false; + const datatypes::TypeHandler *h= ct.typeHandler(); + if (!h) + return false; + return h->CP_type(ct); } -const uint64_t ET_DISABLED = 0x0002; -const uint64_t CPINVALID = 0x0004; -struct PartitionInfo -{ - int64_t min; - int64_t max; - union - { - int128_t int128Min; - int64_t min_; - }; - union - { - int128_t int128Max; - int64_t max_; - }; - uint64_t status; - PartitionInfo(): min((uint64_t)0x8000000000000001ULL), - max((uint64_t) - 0x8000000000000001LL), - status(0) - { - int128Min = datatypes::Decimal::minInt128; - int128Max = datatypes::Decimal::maxInt128; - }; -}; +typedef map PartitionMap; -typedef map PartitionMap; - -template -const string format(T v, CalpontSystemCatalog::ColType& ct) -{ - ostringstream oss; - - switch (ct.colDataType) - { - case CalpontSystemCatalog::DATE: - oss << DataConvert::dateToString(v); - break; - - case CalpontSystemCatalog::DATETIME: - oss << DataConvert::datetimeToString(v); - break; - - case CalpontSystemCatalog::TIMESTAMP: - oss << DataConvert::timestampToString(v, current_thd->variables.time_zone->get_name()->ptr()); - break; - - case CalpontSystemCatalog::TIME: - oss << DataConvert::timeToString(v); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - { - // swap again to retain the string byte order - uint64_t tmp = uint64ToStr(v); - oss << (char*)(&tmp); - break; - } - - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - if (ct.colWidth <= 8) - { - if (ct.scale > 0) - { - double d = ((double)(v) / (double)pow((double)10, ct.scale)); - oss << setprecision(ct.scale) << fixed << d; - } - else - { - oss << (int64_t) v; - } - } - else - { - char buf[datatypes::MAXLENGTH16BYTES]; - DataConvert::decimalToString((int128_t*)&v, (unsigned)ct.scale, buf, sizeof(buf), ct.colDataType); - oss << buf; - } - - break; - } - - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - oss << static_cast(v); - break; - - case CalpontSystemCatalog::VARBINARY: - oss << "N/A"; - break; - - default: - oss << (int64_t) v; - break; - } - - return oss.str(); -} - -template -T IDB_format(char* str, CalpontSystemCatalog::ColType& ct, uint8_t& rf) -{ - T v = 0; - bool pushWarning = false; - rf = 0; - boost::any anyVal = DataConvert::convertColumnData(ct, str, pushWarning, current_thd->variables.time_zone->get_name()->ptr(), false, true, false); - - switch (ct.colDataType) - { - case CalpontSystemCatalog::BIT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::TINYINT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::UTINYINT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::SMALLINT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::USMALLINT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: -#ifdef _MSC_VER - v = boost::any_cast(anyVal); -#else - v = boost::any_cast(anyVal); -#endif - break; - - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::BIGINT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::UBIGINT: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::TEXT: - case CalpontSystemCatalog::CLOB: - { - string i = boost::any_cast(anyVal); - // bug 1932, pad nulls up to the size of v - i.resize(sizeof(v), 0); - v = uint64ToStr(*((uint64_t*) i.data())); - - if (pushWarning) - rf = ROUND_POS; - } - break; - - case CalpontSystemCatalog::DATE: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::DATETIME: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::TIME: - v = boost::any_cast(anyVal); - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - v = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) - v = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::FOUR_BYTE) -#ifdef _MSC_VER - v = boost::any_cast(anyVal); - -#else - v = boost::any_cast(anyVal); -#endif - else if (ct.colWidth == execplan::CalpontSystemCatalog::EIGHT_BYTE) - v = boost::any_cast(anyVal); - else - v = boost::any_cast(anyVal); - - break; - - default: - break; - } - - if ((ct.colDataType == CalpontSystemCatalog::TINYINT || - ct.colDataType == CalpontSystemCatalog::SMALLINT || - ct.colDataType == CalpontSystemCatalog::MEDINT || - ct.colDataType == CalpontSystemCatalog::INT || - ct.colDataType == CalpontSystemCatalog::BIGINT || - ct.colDataType == CalpontSystemCatalog::DECIMAL || - ct.colDataType == CalpontSystemCatalog::UDECIMAL) && - pushWarning) - { - // get rid of leading white spaces and parentheses - string data(str); - size_t fpos = data.find_first_of(" \t()"); - - while (string::npos != fpos) - { - data.erase(fpos, 1); - fpos = data.find_first_of(" \t()"); - } - - rf = (data[0] == '-') ? ROUND_NEG : ROUND_POS; - } - - return v; -} void parsePartitionString(UDF_ARGS* args, @@ -644,6 +315,35 @@ int processPartition ( SqlStatement* stmt) return rc; } + +static void addPartition(const CalpontSystemCatalog::ColType& ct, + DBRM &em, + const BRM::EMEntry &entry, + PartitionMap &partMap, + const LogicalPartition &logicalPartNum) +{ + const datatypes::TypeHandler *h= ct.typeHandler(); + int state; + datatypes::MinMaxPartitionInfo partInfo= h->getExtentPartitionInfo(ct, em, entry, &state); + + PartitionMap::iterator mapit = partMap.find(logicalPartNum); + if (mapit == partMap.end()) + { + if (state != CP_VALID) + partInfo.set_invalid(); + + partMap[logicalPartNum] = partInfo; + } + else + { + if (mapit->second.is_invalid()) + return; + + mapit->second.MinMaxInfo::operator=(h->widenMinMaxInfo(ct, mapit->second, partInfo)); + } +} + + void partitionByValue_common(UDF_ARGS* args, // input string& errMsg, // output CalpontSystemCatalog::TableName& tableName, // output @@ -655,13 +355,7 @@ void partitionByValue_common(UDF_ARGS* args, // input vector entries; vector::iterator iter; PartitionMap partMap; - PartitionMap::iterator mapit; - int32_t seqNum; string schema, table, column; - CalpontSystemCatalog::ColType ct; - int64_t startVal = 0, endVal = 0; - int128_t int128StartVal = 0, int128EndVal = 0; - uint8_t rfMin = 0, rfMax = 0; if (args->arg_count == 5) { @@ -709,7 +403,12 @@ void partitionByValue_common(UDF_ARGS* args, // input CalpontSystemCatalog::TableColName tcn = make_tcn(schema, table, column, lower_case_table_names); csc->identity(CalpontSystemCatalog::FE); OID_t oid = csc->lookupOID(tcn); - ct = csc->colType(oid); + CalpontSystemCatalog::ColType ct = csc->colType(oid); + datatypes::SessionParam sp(current_thd->variables.time_zone->get_name()->ptr()); + datatypes::SimpleValue startVal; + datatypes::SimpleValue endVal; + datatypes::round_style_t rfMin = datatypes::round_style_t::NONE; + datatypes::round_style_t rfMax = datatypes::round_style_t::NONE; if (oid == -1) { @@ -731,261 +430,39 @@ void partitionByValue_common(UDF_ARGS* args, // input if (args->arg_count == 4) { - if (!args->args[2]) - { - if (!datatypes::Decimal::isWideDecimalType(ct)) - { - if (isUnsigned(ct.colDataType)) - { - startVal = 0; - } - else - { - startVal = numeric_limits::min(); - } - } - else - { - if (isUnsigned(ct.colDataType)) - { - int128StartVal = 0; - } - else - { - int128StartVal = datatypes::Decimal::minInt128; - } - } - } - else - { - if (!datatypes::Decimal::isWideDecimalType(ct)) - startVal = IDB_format((char*) args->args[2], ct, rfMin); - else - int128StartVal = IDB_format((char*) args->args[2], ct, rfMin); - } - - if (!args->args[3]) - { - if (!datatypes::Decimal::isWideDecimalType(ct)) - { - if (isUnsigned(ct.colDataType)) - { - endVal = static_cast(numeric_limits::max()); - } - else - { - endVal = numeric_limits::max(); - } - } - else - { - if (isUnsigned(ct.colDataType)) - { - int128EndVal = -1; - } - else - { - int128EndVal = datatypes::Decimal::maxInt128; - } - } - } - else - { - if (!datatypes::Decimal::isWideDecimalType(ct)) - endVal = IDB_format((char*) args->args[3], ct, rfMax); - else - int128EndVal = IDB_format((char*) args->args[3], ct, rfMax); - } + startVal = getStartVal(sp, ct, args->args[2], rfMin); + endVal = getEndVal(sp, ct, args->args[3], rfMax); } else { - if (!args->args[3]) - { - if (!datatypes::Decimal::isWideDecimalType(ct)) - { - if (isUnsigned(ct.colDataType)) - { - startVal = 0; - } - else - { - startVal = numeric_limits::min(); - } - } - else - { - if (isUnsigned(ct.colDataType)) - { - int128StartVal = 0; - } - else - { - int128StartVal = datatypes::Decimal::minInt128; - } - } - } - else - { - if (!datatypes::Decimal::isWideDecimalType(ct)) - startVal = IDB_format((char*) args->args[3], ct, rfMin); - else - int128StartVal = IDB_format((char*) args->args[3], ct, rfMin); - } - - if (!args->args[4]) - { - if (!datatypes::Decimal::isWideDecimalType(ct)) - { - if (isUnsigned(ct.colDataType)) - { - endVal = static_cast(numeric_limits::max()); - } - else - { - endVal = numeric_limits::max(); - } - } - else - { - if (isUnsigned(ct.colDataType)) - { - int128EndVal = -1; - } - else - { - int128EndVal = datatypes::Decimal::maxInt128; - } - } - } - else - { - if (!datatypes::Decimal::isWideDecimalType(ct)) - endVal = IDB_format((char*) args->args[4], ct, rfMax); - else - int128EndVal = IDB_format((char*) args->args[4], ct, rfMax); - } + startVal = getStartVal(sp, ct, args->args[3], rfMin); + endVal = getEndVal(sp, ct, args->args[4], rfMax); } CHECK(em.getExtents(oid, entries, false, false, true)); if (entries.size() > 0) { - LogicalPartition logicalPartNum; for (iter = entries.begin(); iter != entries.end(); ++iter) { - PartitionInfo partInfo; + LogicalPartition logicalPartNum; logicalPartNum.dbroot = (*iter).dbRoot; logicalPartNum.pp = (*iter).partitionNum; logicalPartNum.seg = (*iter).segmentNum; - - if (iter->status == EXTENTOUTOFSERVICE) - partInfo.status |= ET_DISABLED; - - mapit = partMap.find(logicalPartNum); - - int state; - - if (!datatypes::Decimal::isWideDecimalType(ct)) - state = em.getExtentMaxMin(iter->range.start, partInfo.max, partInfo.min, seqNum); - else - state = em.getExtentMaxMin(iter->range.start, partInfo.int128Max, partInfo.int128Min, seqNum); - - // char column order swap - if ((ct.colDataType == CalpontSystemCatalog::CHAR && ct.colWidth <= 8) || - (ct.colDataType == CalpontSystemCatalog::VARCHAR && ct.colWidth <= 7)) - { - partInfo.max = uint64ToStr(partInfo.max); - partInfo.min = uint64ToStr(partInfo.min); - } - - if (mapit == partMap.end()) - { - if (state != CP_VALID) - partInfo.status |= CPINVALID; - - partMap[logicalPartNum] = partInfo; - } - else - { - if (mapit->second.status & CPINVALID) - continue; - - if (!datatypes::Decimal::isWideDecimalType(ct)) - { - if (isUnsigned(ct.colDataType)) - { - mapit->second.min = - (static_cast(partInfo.min) < static_cast(mapit->second.min) ? partInfo.min : mapit->second.min); - mapit->second.max = - (static_cast(partInfo.max) > static_cast(mapit->second.max) ? partInfo.max : mapit->second.max); - } - else - { - mapit->second.min = (partInfo.min < mapit->second.min ? partInfo.min : mapit->second.min); - mapit->second.max = (partInfo.max > mapit->second.max ? partInfo.max : mapit->second.max); - } - } - else - { - mapit->second.int128Min = (partInfo.int128Min < mapit->second.int128Min ? partInfo.int128Min : mapit->second.int128Min); - mapit->second.int128Max = (partInfo.int128Max > mapit->second.int128Max ? partInfo.int128Max : mapit->second.int128Max); - } - } + addPartition(ct, em, *iter, partMap, logicalPartNum); } // check col value range - for (mapit = partMap.begin(); mapit != partMap.end(); ++mapit) + for (PartitionMap::iterator mapit = partMap.begin(); mapit != partMap.end(); ++mapit) { + if (mapit->second.is_invalid()) + continue; + + const datatypes::TypeHandler *h= ct.typeHandler(); // @bug 4595. check empty/null case - if (!datatypes::Decimal::isWideDecimalType(ct)) - { - if (isUnsigned(ct.colDataType)) - { - if (!(mapit->second.status & CPINVALID) && - static_cast(mapit->second.min) >= static_cast(startVal) && - static_cast(mapit->second.max) <= static_cast(endVal) && - !(static_cast(mapit->second.min) == numeric_limits::max() && - static_cast(mapit->second.max == 0))) - { - if (rfMin == ROUND_POS && mapit->second.min == startVal) - continue; - - if (rfMax == ROUND_NEG && mapit->second.max == endVal) - continue; - - partSet.insert(mapit->first); - } - } - else - { - if (!(mapit->second.status & CPINVALID) && mapit->second.min >= startVal && mapit->second.max <= endVal && - !(mapit->second.min == numeric_limits::max() && mapit->second.max == numeric_limits::min())) - { - if (rfMin == ROUND_POS && mapit->second.min == startVal) - continue; - - if (rfMax == ROUND_NEG && mapit->second.max == endVal) - continue; - - partSet.insert(mapit->first); - } - } - } - else - { - if (!(mapit->second.status & CPINVALID) && mapit->second.int128Min >= int128StartVal && mapit->second.int128Max <= int128EndVal && - !(mapit->second.int128Min == datatypes::Decimal::maxInt128 && mapit->second.int128Max == datatypes::Decimal::minInt128)) - { - if (rfMin == ROUND_POS && mapit->second.int128Min == int128StartVal) - continue; - - if (rfMax == ROUND_NEG && mapit->second.int128Max == int128EndVal) - continue; - - partSet.insert(mapit->first); - } - } + if (h->isSuitablePartition(ct, mapit->second, startVal, rfMin, endVal, rfMax)) + partSet.insert(mapit->first); } } } @@ -1135,7 +612,6 @@ extern "C" vector::iterator iter; vector::iterator end; PartitionMap partMap; - int32_t seqNum; string schema, table, column; CalpontSystemCatalog::ColType ct; string errMsg; @@ -1194,54 +670,10 @@ extern "C" for (; iter != end; ++iter) { - PartitionInfo partInfo; logicalPartNum.dbroot = (*iter).dbRoot; logicalPartNum.pp = (*iter).partitionNum; logicalPartNum.seg = (*iter).segmentNum; - - if (iter->status == EXTENTOUTOFSERVICE) - partInfo.status |= ET_DISABLED; - - mapit = partMap.find(logicalPartNum); - - int state = CP_INVALID; - - if (!datatypes::Decimal::isWideDecimalType(ct)) - state = em.getExtentMaxMin(iter->range.start, partInfo.max, partInfo.min, seqNum); - else - state = em.getExtentMaxMin(iter->range.start, partInfo.int128Max, partInfo.int128Min, seqNum); - - // char column order swap for compare - if ((ct.colDataType == CalpontSystemCatalog::CHAR && ct.colWidth <= 8) || - (ct.colDataType == CalpontSystemCatalog::VARCHAR && ct.colWidth <= 7)) - { - partInfo.max = uint64ToStr(partInfo.max); - partInfo.min = uint64ToStr(partInfo.min); - } - - if (mapit == partMap.end()) - { - if (state != CP_VALID) - partInfo.status |= CPINVALID; - - partMap[logicalPartNum] = partInfo; - } - else - { - if (mapit->second.status & CPINVALID) - continue; - - if (!datatypes::Decimal::isWideDecimalType(ct)) - { - mapit->second.min = (partInfo.min < mapit->second.min ? partInfo.min : mapit->second.min); - mapit->second.max = (partInfo.max > mapit->second.max ? partInfo.max : mapit->second.max); - } - else - { - mapit->second.int128Min = (partInfo.int128Min < mapit->second.int128Min ? partInfo.int128Min : mapit->second.int128Min); - mapit->second.int128Max = (partInfo.int128Max > mapit->second.int128Max ? partInfo.int128Max : mapit->second.int128Max); - } - } + addPartition(ct, em, *iter, partMap, logicalPartNum); } } } @@ -1258,35 +690,14 @@ extern "C" return result; } + const datatypes::TypeHandler *h= ct.typeHandler(); + uint8_t valueCharLength= h->PartitionValueCharLength(ct); ostringstream output; output.setf(ios::left, ios::adjustfield); - if (!datatypes::Decimal::isWideDecimalType(ct)) - { - output << setw(10) << "Part#" - << setw(30) << "Min" - << setw(30) << "Max" << "Status"; - } - else - { - output << setw(10) << "Part#" - << setw(datatypes::MAXLENGTH16BYTES) << "Min" - << setw(datatypes::MAXLENGTH16BYTES) << "Max" << "Status"; - } - int64_t maxLimit = numeric_limits::max(); - int64_t minLimit = numeric_limits::min(); - - int128_t int128MaxLimit, int128MinLimit; - int128MaxLimit = datatypes::Decimal::maxInt128; - int128MinLimit = datatypes::Decimal::minInt128; - - // char column order swap for compare in subsequent loop - if ((ct.colDataType == CalpontSystemCatalog::CHAR && ct.colWidth <= 8) || - (ct.colDataType == CalpontSystemCatalog::VARCHAR && ct.colWidth <= 7)) - { - maxLimit = uint64ToStr(maxLimit); - minLimit = uint64ToStr(minLimit); - } + output << setw(10) << "Part#" + << setw(valueCharLength) << "Min" + << setw(valueCharLength) << "Max" << "Status"; PartitionMap::const_iterator partIt; @@ -1296,54 +707,18 @@ extern "C" oss << partIt->first; output << "\n " << setw(10) << oss.str(); - if (partIt->second.status & CPINVALID) + if (partIt->second.is_invalid()) { - if (!datatypes::Decimal::isWideDecimalType(ct)) - output << setw(30) << "N/A" << setw(30) << "N/A"; - else - output << setw(datatypes::MAXLENGTH16BYTES) << "N/A" << setw(datatypes::MAXLENGTH16BYTES) << "N/A"; + output << setw(valueCharLength) << "N/A" + << setw(valueCharLength) << "N/A"; } else { - if ((isUnsigned(ct.colDataType))) - { - if (!datatypes::Decimal::isWideDecimalType(ct)) - { - if (static_cast(partIt->second.min) == numeric_limits::max() - && static_cast(partIt->second.max) == numeric_limits::min()) - output << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; - else - output << setw(30) << format(partIt->second.min, ct) << setw(30) << format(partIt->second.max, ct); - } - else - { - if (partIt->second.int128Min == int128MaxLimit - && partIt->second.int128Max == int128MinLimit) - output << setw(datatypes::MAXLENGTH16BYTES) << "Empty/Null" << setw(datatypes::MAXLENGTH16BYTES) << "Empty/Null"; - else - output << setw(datatypes::MAXLENGTH16BYTES) << format(partIt->second.int128Min, ct) << setw(datatypes::MAXLENGTH16BYTES) << format(partIt->second.int128Max, ct); - } - } - else - { - if (!datatypes::Decimal::isWideDecimalType(ct)) - { - if (partIt->second.min == maxLimit && partIt->second.max == minLimit) - output << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; - else - output << setw(30) << format(partIt->second.min, ct) << setw(30) << format(partIt->second.max, ct); - } - else - { - if (partIt->second.int128Min == int128MaxLimit && partIt->second.int128Max == int128MinLimit) - output << setw(datatypes::MAXLENGTH16BYTES) << "Empty/Null" << setw(datatypes::MAXLENGTH16BYTES) << "Empty/Null"; - else - output << setw(datatypes::MAXLENGTH16BYTES) << format(partIt->second.int128Min, ct) << setw(datatypes::MAXLENGTH16BYTES) << format(partIt->second.int128Max, ct); - } - } + const datatypes::TypeHandler *h= ct.typeHandler(); + oss << h->formatPartitionInfo(ct, partIt->second); } - if (partIt->second.status & ET_DISABLED) + if (partIt->second.is_disabled()) output << "Disabled"; else output << "Enabled"; @@ -1884,6 +1259,9 @@ extern "C" delete initid->ptr; } + + + #ifdef _MSC_VER __declspec(dllexport) #endif @@ -1897,13 +1275,14 @@ extern "C" vector::iterator end; PartitionMap partMap; PartitionMap::iterator mapit; - int32_t seqNum; string schema, table, column; CalpontSystemCatalog::ColType ct; string errMsg; - int64_t startVal = 0, endVal = 0; - int128_t int128StartVal = 0, int128EndVal = 0; - uint8_t rfMin = 0, rfMax = 0; + datatypes::SessionParam sp(current_thd->variables.time_zone->get_name()->ptr()); + datatypes::SimpleValue startVal; + datatypes::SimpleValue endVal; + datatypes::round_style_t rfMin = datatypes::round_style_t::NONE; + datatypes::round_style_t rfMax = datatypes::round_style_t::NONE; try { @@ -1959,139 +1338,13 @@ extern "C" if (args->arg_count == 4) { - if (!args->args[2]) - { - if (!datatypes::Decimal::isWideDecimalType(ct)) - { - if (isUnsigned(ct.colDataType)) - { - startVal = 0; - } - else - { - startVal = numeric_limits::min(); - } - } - else - { - if (isUnsigned(ct.colDataType)) - { - int128StartVal = 0; - } - else - { - int128StartVal = datatypes::Decimal::minInt128; - } - } - } - else - { - if (!datatypes::Decimal::isWideDecimalType(ct)) - startVal = IDB_format((char*) args->args[2], ct, rfMin); - else - int128StartVal = IDB_format((char*) args->args[2], ct, rfMin); - } - - if (!args->args[3]) - { - if (!datatypes::Decimal::isWideDecimalType(ct)) - { - if (isUnsigned(ct.colDataType)) - { - endVal = static_cast(numeric_limits::max()); - } - else - { - endVal = numeric_limits::max(); - } - } - else - { - if (isUnsigned(ct.colDataType)) - { - int128EndVal = -1; - } - else - { - int128EndVal = datatypes::Decimal::maxInt128; - } - } - } - else - { - if (!datatypes::Decimal::isWideDecimalType(ct)) - endVal = IDB_format((char*) args->args[3], ct, rfMax); - else - int128EndVal = IDB_format((char*) args->args[3], ct, rfMax); - } + startVal= getStartVal(sp, ct, args->args[2], rfMin); + endVal= getEndVal(sp, ct, args->args[3], rfMax); } else { - if (!args->args[3]) - { - if (!datatypes::Decimal::isWideDecimalType(ct)) - { - if (isUnsigned(ct.colDataType)) - { - startVal = 0; - } - else - { - startVal = numeric_limits::min(); - } - } - else - { - if (isUnsigned(ct.colDataType)) - { - int128StartVal = 0; - } - else - { - int128StartVal = datatypes::Decimal::minInt128; - } - } - } - else - { - if (!datatypes::Decimal::isWideDecimalType(ct)) - startVal = IDB_format((char*) args->args[3], ct, rfMin); - else - int128StartVal = IDB_format((char*) args->args[3], ct, rfMin); - } - - if (!args->args[4]) - { - if (!datatypes::Decimal::isWideDecimalType(ct)) - { - if (isUnsigned(ct.colDataType)) - { - endVal = static_cast(numeric_limits::max()); - } - else - { - endVal = numeric_limits::max(); - } - } - else - { - if (isUnsigned(ct.colDataType)) - { - int128EndVal = -1; - } - else - { - int128EndVal = datatypes::Decimal::maxInt128; - } - } - } - else - { - if (!datatypes::Decimal::isWideDecimalType(ct)) - endVal = IDB_format((char*) args->args[4], ct, rfMax); - else - int128EndVal = IDB_format((char*) args->args[4], ct, rfMax); - } + startVal= getStartVal(sp, ct, args->args[3], rfMin); + endVal= getEndVal(sp, ct, args->args[4], rfMax); } CHECK(em.getExtents(oid, entries, false, false, true)); @@ -2104,64 +1357,10 @@ extern "C" for (; iter != end; ++iter) { - PartitionInfo partInfo; logicalPartNum.dbroot = (*iter).dbRoot; logicalPartNum.pp = (*iter).partitionNum; logicalPartNum.seg = (*iter).segmentNum; - - if (iter->status == EXTENTOUTOFSERVICE) - partInfo.status |= ET_DISABLED; - - mapit = partMap.find(logicalPartNum); - - int state; - - if (!datatypes::Decimal::isWideDecimalType(ct)) - state = em.getExtentMaxMin(iter->range.start, partInfo.max, partInfo.min, seqNum); - else - state = em.getExtentMaxMin(iter->range.start, partInfo.int128Max, partInfo.int128Min, seqNum); - - // char column order swap - if ((ct.colDataType == CalpontSystemCatalog::CHAR && ct.colWidth <= 8) || - (ct.colDataType == CalpontSystemCatalog::VARCHAR && ct.colWidth <= 7)) - { - partInfo.max = uint64ToStr(partInfo.max); - partInfo.min = uint64ToStr(partInfo.min); - } - - if (mapit == partMap.end()) - { - if (state != CP_VALID) - partInfo.status |= CPINVALID; - - partMap[logicalPartNum] = partInfo; - } - else - { - if (mapit->second.status & CPINVALID) - continue; - - if (!datatypes::Decimal::isWideDecimalType(ct)) - { - if (isUnsigned(ct.colDataType)) - { - mapit->second.min = - (static_cast(partInfo.min) < static_cast(mapit->second.min) ? partInfo.min : mapit->second.min); - mapit->second.max = - (static_cast(partInfo.max) > static_cast(mapit->second.max) ? partInfo.max : mapit->second.max); - } - else - { - mapit->second.min = (partInfo.min < mapit->second.min ? partInfo.min : mapit->second.min); - mapit->second.max = (partInfo.max > mapit->second.max ? partInfo.max : mapit->second.max); - } - } - else - { - mapit->second.int128Min = (partInfo.int128Min < mapit->second.int128Min ? partInfo.int128Min : mapit->second.int128Min); - mapit->second.int128Max = (partInfo.int128Max > mapit->second.int128Max ? partInfo.int128Max : mapit->second.int128Max); - } - } + addPartition(ct, em, *iter, partMap, logicalPartNum); } } } @@ -2193,107 +1392,41 @@ extern "C" for (mapit = partMap.begin(); mapit != partMap.end(); ++mapit) { + const datatypes::TypeHandler *h= ct.typeHandler(); + uint8_t valueCharLength= h->PartitionValueCharLength(ct); + if (mapit->second.is_invalid()) + { + output << setw(valueCharLength) << "N/A" + << setw(valueCharLength) << "N/A"; + continue; + } // @bug 4595. check empty/null case - if (!datatypes::Decimal::isWideDecimalType(ct)) + string tmp= h->PrintPartitionValue(ct, mapit->second, + startVal, rfMin, + endVal, rfMax); + if (tmp == "") + continue; + + // print header + if (noPartFound) { - if (!(mapit->second.status & CPINVALID) && mapit->second.min >= startVal && mapit->second.max <= endVal && - !(mapit->second.min == numeric_limits::max() && mapit->second.max == numeric_limits::min())) - { - if (rfMin == ROUND_POS && mapit->second.min == startVal) - continue; - - if (rfMax == ROUND_NEG && mapit->second.max == endVal) - continue; - - // print header - if (noPartFound) - { - output.setf(ios::left, ios::adjustfield); - output << setw(10) << "Part#" - << setw(30) << "Min" - << setw(30) << "Max" << "Status"; - } - - noPartFound = false; - - // print part info - ostringstream oss; - oss << mapit->first; - output << "\n " << setw(10) << oss.str(); - - if (mapit->second.status & CPINVALID) - { - output << setw(30) << "N/A" << setw(30) << "N/A"; - } - else - { - if ((isUnsigned(ct.colDataType))) - { - if (static_cast(mapit->second.min) > static_cast(mapit->second.max)) - output << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; - else - output << setw(30) << format(mapit->second.min, ct) << setw(30) << format(mapit->second.max, ct); - } - else - { - if (mapit->second.min > mapit->second.max) - output << setw(30) << "Empty/Null" << setw(30) << "Empty/Null"; - else - output << setw(30) << format(mapit->second.min, ct) << setw(30) << format(mapit->second.max, ct); - } - } - - if (mapit->second.status & ET_DISABLED) - output << "Disabled"; - else - output << "Enabled"; - } + output.setf(ios::left, ios::adjustfield); + output << setw(10) << "Part#" + << setw(valueCharLength) << "Min" + << setw(valueCharLength) << "Max" << "Status"; + noPartFound = false; } + + // print part info + ostringstream oss; + oss << mapit->first; + output << "\n " << setw(10) << oss.str() << tmp; + + if (mapit->second.is_disabled()) + output << "Disabled"; else - { - if (!(mapit->second.status & CPINVALID) && mapit->second.int128Min >= int128StartVal && mapit->second.int128Max <= int128EndVal && - !(mapit->second.int128Min == datatypes::Decimal::maxInt128 && mapit->second.int128Max == datatypes::Decimal::minInt128)) - { - if (rfMin == ROUND_POS && mapit->second.int128Min == int128StartVal) - continue; + output << "Enabled"; - if (rfMax == ROUND_NEG && mapit->second.int128Max == int128EndVal) - continue; - - // print header - if (noPartFound) - { - output.setf(ios::left, ios::adjustfield); - output << setw(10) << "Part#" - << setw(datatypes::MAXLENGTH16BYTES) << "Min" - << setw(datatypes::MAXLENGTH16BYTES) << "Max" << "Status"; - } - - noPartFound = false; - - // print part info - ostringstream oss; - oss << mapit->first; - output << "\n " << setw(10) << oss.str(); - - if (mapit->second.status & CPINVALID) - { - output << setw(datatypes::MAXLENGTH16BYTES) << "N/A" << setw(datatypes::MAXLENGTH16BYTES) << "N/A"; - } - else - { - if (mapit->second.int128Min > mapit->second.int128Max) - output << setw(datatypes::MAXLENGTH16BYTES) << "Empty/Null" << setw(datatypes::MAXLENGTH16BYTES) << "Empty/Null"; - else - output << setw(datatypes::MAXLENGTH16BYTES) << format(mapit->second.int128Min, ct) << setw(datatypes::MAXLENGTH16BYTES) << format(mapit->second.int128Max, ct); - } - - if (mapit->second.status & ET_DISABLED) - output << "Disabled"; - else - output << "Enabled"; - } - } } if (noPartFound) diff --git a/dbcon/mysql/is_columnstore_extents.cpp b/dbcon/mysql/is_columnstore_extents.cpp index 7f48b0d41..7e6f01165 100644 --- a/dbcon/mysql/is_columnstore_extents.cpp +++ b/dbcon/mysql/is_columnstore_extents.cpp @@ -28,7 +28,6 @@ #include "objectidmanager.h" #include "is_columnstore.h" #include "mcs_decimal.h" -#include "widedecimalutils.h" #include "dataconvert.h" // Required declaration as it isn't in a MairaDB include @@ -114,10 +113,10 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM* emp, TABLE* table, THD* th { table->field[4]->set_notnull(); - char buf[utils::MAXLENGTH16BYTES]; + char buf[datatypes::Decimal::MAXLENGTH16BYTES]; dataconvert::DataConvert::decimalToString( &iter->partition.cprange.bigLoVal, - 0, buf, sizeof(buf), execplan::CalpontSystemCatalog::DECIMAL); + 0, buf, (uint8_t) sizeof(buf), datatypes::SystemCatalog::DECIMAL); table->field[4]->store(buf, strlen(buf), table->field[4]->charset()); } @@ -129,10 +128,10 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM* emp, TABLE* table, THD* th { table->field[5]->set_notnull(); - char buf[utils::MAXLENGTH16BYTES]; + char buf[datatypes::Decimal::MAXLENGTH16BYTES]; dataconvert::DataConvert::decimalToString( &iter->partition.cprange.bigHiVal, - 0, buf, sizeof(buf), execplan::CalpontSystemCatalog::DECIMAL); + 0, buf, (uint8_t) sizeof(buf), datatypes::SystemCatalog::DECIMAL); table->field[5]->store(buf, strlen(buf), table->field[5]->charset()); } } diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index 62ac4f66f..fbedc0c6a 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -288,7 +288,7 @@ void ColumnCommand::issuePrimitive() bpp->lbidForCP = lbid; if (UNLIKELY(utils::isWide(colType.colWidth))) { - if (datatypes::Decimal::isWideDecimalType(colType)) + if (colType.isWideDecimalType()) { bpp->hasWideColumnOut = true; // colWidth is int32 and wideColumnWidthOut's diff --git a/primitives/primproc/filtercommand.cpp b/primitives/primproc/filtercommand.cpp index 5bf5d0b05..ac9f8c79c 100644 --- a/primitives/primproc/filtercommand.cpp +++ b/primitives/primproc/filtercommand.cpp @@ -250,7 +250,7 @@ void FilterCommand::setColTypes(const execplan::CalpontSystemCatalog::ColType& l leftColType = left; rightColType = right; - if (datatypes::Decimal::isWideDecimalType(left) || datatypes::Decimal::isWideDecimalType(right)) + if (left.isWideDecimalType() || right.isWideDecimalType()) hasWideColumns = true; } @@ -280,7 +280,7 @@ void FilterCommand::doFilter() if ((this->*compareFunc)(i, j) == true) { bpp->relRids[bpp->ridCount] = bpp->fFiltCmdRids[0][i]; - if (datatypes::Decimal::isWideDecimalType(leftColType)) + if (leftColType.isWideDecimalType()) bpp->wide128Values[bpp->ridCount] = bpp->fFiltCmdBinaryValues[0][i]; else bpp->values[bpp->ridCount] = bpp->fFiltCmdValues[0][i]; @@ -342,7 +342,7 @@ bool FilterCommand::binaryCompare(uint64_t i, uint64_t j) // not int128_t int128_t leftVal, rightVal; - if (datatypes::Decimal::isWideDecimalType(leftColType)) + if (leftColType.isWideDecimalType()) { if (execplan::isNull(bpp->fFiltCmdBinaryValues[0][i], leftColType)) return false; @@ -355,7 +355,7 @@ bool FilterCommand::binaryCompare(uint64_t i, uint64_t j) leftVal = bpp->fFiltCmdValues[0][i]; } - if (datatypes::Decimal::isWideDecimalType(rightColType)) + if (rightColType.isWideDecimalType()) { if (execplan::isNull(bpp->fFiltCmdBinaryValues[1][j], rightColType)) return false; diff --git a/primitives/primproc/pseudocc.cpp b/primitives/primproc/pseudocc.cpp index 4fb3ba8d8..588723452 100644 --- a/primitives/primproc/pseudocc.cpp +++ b/primitives/primproc/pseudocc.cpp @@ -58,7 +58,7 @@ void PseudoCC::resetCommand(messageqcpp::ByteStream& bs) { if (function == PSEUDO_EXTENTMAX || function == PSEUDO_EXTENTMIN) { - if (!datatypes::Decimal::isWideDecimalType(colType)) + if (!colType.isWideDecimalType()) bs >> valueFromUM; else bs >> bigValueFromUM; @@ -67,7 +67,7 @@ void PseudoCC::resetCommand(messageqcpp::ByteStream& bs) { bs >> valueFromUM; - if (datatypes::Decimal::isWideDecimalType(colType)) + if (colType.isWideDecimalType()) bigValueFromUM = valueFromUM; } diff --git a/tests/dataconvert-tests.cpp b/tests/dataconvert-tests.cpp index 07b191b1a..17c816c98 100644 --- a/tests/dataconvert-tests.cpp +++ b/tests/dataconvert-tests.cpp @@ -20,8 +20,6 @@ using namespace std; #include "gtest/gtest.h" -#include "calpontsystemcatalog.h" -using namespace execplan; #include "dataconvert.h" using namespace dataconvert; #include "joblisttypes.h" @@ -143,7 +141,8 @@ TEST(DataConvertTest, Strtoll128) TEST(DataConvertTest, NumberIntValue) { - CalpontSystemCatalog::ColType ct; + datatypes::SystemCatalog::TypeAttributesStd ct; + int128_t res, valMax; string data; bool noRoundup = false; @@ -153,7 +152,7 @@ TEST(DataConvertTest, NumberIntValue) // tests for signed decimal // behaviour of number_int_value for unsigned decimal // is similar to the signed case. - ct.colDataType = CalpontSystemCatalog::DECIMAL; + datatypes::SystemCatalog::ColDataType typecode = datatypes::SystemCatalog::DECIMAL; // test with decimal(38,0) ct.precision = 38; ct.scale = 0; @@ -161,39 +160,39 @@ TEST(DataConvertTest, NumberIntValue) //data = ""; data = "0"; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); EXPECT_EQ(res, 0); EXPECT_FALSE(pushWarning); data = "1234"; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); EXPECT_EQ(res, 1234); EXPECT_FALSE(pushWarning); data = "12.0"; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); EXPECT_EQ(res, 12); EXPECT_FALSE(pushWarning); data = "12.34"; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); EXPECT_EQ(res, 12); EXPECT_TRUE(pushWarning); data = "-1234"; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); EXPECT_EQ(res, -1234); EXPECT_FALSE(pushWarning); data = "-12.34"; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); EXPECT_EQ(res, -12); EXPECT_TRUE(pushWarning); // test max data = "99999999999999999999999999999999999999"; valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -206,7 +205,7 @@ TEST(DataConvertTest, NumberIntValue) valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; valMax = -valMax; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -217,12 +216,12 @@ TEST(DataConvertTest, NumberIntValue) // test rounding data = "12.56"; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); EXPECT_EQ(res, 13); EXPECT_TRUE(pushWarning); data = "-12.56"; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); EXPECT_EQ(res, -13); EXPECT_TRUE(pushWarning); // test saturation @@ -230,7 +229,7 @@ TEST(DataConvertTest, NumberIntValue) // valMax has 38 9's valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -243,7 +242,7 @@ TEST(DataConvertTest, NumberIntValue) valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; valMax = -valMax; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -255,7 +254,7 @@ TEST(DataConvertTest, NumberIntValue) data = "1.23e37"; valMax = ((((((((int128_t)123000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 100) + 0; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -266,7 +265,7 @@ TEST(DataConvertTest, NumberIntValue) data = "1.23e38"; valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -279,39 +278,39 @@ TEST(DataConvertTest, NumberIntValue) ct.scale = 10; data = "0"; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); EXPECT_EQ(res, 0); EXPECT_FALSE(pushWarning); data = "1234"; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); EXPECT_EQ(res, 12340000000000); EXPECT_FALSE(pushWarning); data = "12.0"; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); EXPECT_EQ(res, 120000000000); EXPECT_FALSE(pushWarning); data = "12.34"; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); EXPECT_EQ(res, 123400000000); EXPECT_FALSE(pushWarning); data = "-1234"; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); EXPECT_EQ(res, -12340000000000); EXPECT_FALSE(pushWarning); data = "-12.34"; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); EXPECT_EQ(res, -123400000000); EXPECT_FALSE(pushWarning); // test max data = "9999999999999999999999999999.9999999999"; valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -324,7 +323,7 @@ TEST(DataConvertTest, NumberIntValue) valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; valMax = -valMax; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -335,12 +334,12 @@ TEST(DataConvertTest, NumberIntValue) // test rounding data = "12.11111111119"; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); EXPECT_EQ(res, 121111111112); EXPECT_TRUE(pushWarning); data = "-12.11111111119"; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); EXPECT_EQ(res, -121111111112); EXPECT_TRUE(pushWarning); // test saturation @@ -348,7 +347,7 @@ TEST(DataConvertTest, NumberIntValue) // valMax has 38 9's valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -361,7 +360,7 @@ TEST(DataConvertTest, NumberIntValue) valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; valMax = -valMax; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -373,7 +372,7 @@ TEST(DataConvertTest, NumberIntValue) data = "1.23e9"; valMax = ((((int128_t)123000000 * 1000000000) + 0) * 100); pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -384,7 +383,7 @@ TEST(DataConvertTest, NumberIntValue) data = "1.23e28"; valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -396,13 +395,13 @@ TEST(DataConvertTest, NumberIntValue) ct.scale = 38; data = "0"; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); EXPECT_EQ(res, 0); EXPECT_FALSE(pushWarning); data = "1.234"; valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -413,7 +412,7 @@ TEST(DataConvertTest, NumberIntValue) data = "0.123"; valMax = ((((((((int128_t)123000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 100) + 0; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -425,7 +424,7 @@ TEST(DataConvertTest, NumberIntValue) valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; valMax = -valMax; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -437,7 +436,7 @@ TEST(DataConvertTest, NumberIntValue) valMax = ((((((((int128_t)123000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 100) + 0; valMax = -valMax; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -449,7 +448,7 @@ TEST(DataConvertTest, NumberIntValue) data = "0.99999999999999999999999999999999999999"; valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -462,7 +461,7 @@ TEST(DataConvertTest, NumberIntValue) valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; valMax = -valMax; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -474,7 +473,7 @@ TEST(DataConvertTest, NumberIntValue) data = "0.199999999999999999999999999999999999999"; valMax = ((((((((int128_t)200000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 100) + 0; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -486,7 +485,7 @@ TEST(DataConvertTest, NumberIntValue) valMax = ((((((((int128_t)200000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 100) + 0; valMax = -valMax; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -499,7 +498,7 @@ TEST(DataConvertTest, NumberIntValue) // valMax has 38 9's valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -512,7 +511,7 @@ TEST(DataConvertTest, NumberIntValue) valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; valMax = -valMax; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -524,7 +523,7 @@ TEST(DataConvertTest, NumberIntValue) data = "123e-4"; valMax = ((((((((int128_t)123000000 * 1000000000) + 0) * 1000000000) + 0) * 1000000000 ) + 0) * 10) + 0; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -535,7 +534,7 @@ TEST(DataConvertTest, NumberIntValue) data = "123e-2"; valMax = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; pushWarning = false; - number_int_value(data, ct, pushWarning, noRoundup, res); + number_int_value(data, typecode, ct, pushWarning, noRoundup, res); b1 = *(reinterpret_cast(&res)); b2 = *(reinterpret_cast(&res) + 1); b3 = *(reinterpret_cast(&valMax)); @@ -547,8 +546,8 @@ TEST(DataConvertTest, NumberIntValue) TEST(DataConvertTest, DecimalToStringCheckScale0) { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::DECIMAL; + datatypes::SystemCatalog::TypeHolderStd ct; + ct.colDataType = datatypes::SystemCatalog::DECIMAL; char buf[42]; string input, expected; ct.precision = 38; @@ -599,8 +598,8 @@ TEST(DataConvertTest, DecimalToStringCheckScale0) } TEST(DataConvertTest, DecimalToStringCheckScale10) { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::DECIMAL; + datatypes::SystemCatalog::TypeHolderStd ct; + ct.colDataType = datatypes::SystemCatalog::DECIMAL; char buf[42]; string input, expected; ct.precision = 38; @@ -670,8 +669,8 @@ TEST(DataConvertTest, DecimalToStringCheckScale10) TEST(DataConvertTest, DecimalToStringCheckScale38) { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::DECIMAL; + datatypes::SystemCatalog::TypeHolderStd ct; + ct.colDataType = datatypes::SystemCatalog::DECIMAL; char buf[42]; string input, expected; ct.precision = 38; @@ -731,8 +730,8 @@ TEST(DataConvertTest, DecimalToStringCheckScale38) TEST(DataConvertTest, DecimalToStringCheckScale37) { - CalpontSystemCatalog::ColType ct; - ct.colDataType = CalpontSystemCatalog::DECIMAL; + datatypes::SystemCatalog::TypeHolderStd ct; + ct.colDataType = datatypes::SystemCatalog::DECIMAL; char buf[42]; string expected; ct.precision = 38; diff --git a/tests/mcs_decimal-tests.cpp b/tests/mcs_decimal-tests.cpp index 53a40c533..762db7365 100644 --- a/tests/mcs_decimal-tests.cpp +++ b/tests/mcs_decimal-tests.cpp @@ -21,9 +21,7 @@ #include "treenode.h" #include "mcs_decimal.h" -#include "widedecimalutils.h" #include "dataconvert.h" -#include "calpontsystemcatalog.h" TEST(Decimal, compareCheck) { @@ -86,8 +84,7 @@ TEST(Decimal, compareCheck) TEST(Decimal, additionNoOverflowCheck) { - execplan::CalpontSystemCatalog::ColType ct; - ct.colDataType = execplan::CalpontSystemCatalog::DECIMAL; + datatypes::SystemCatalog::ColDataType colDataType = datatypes::SystemCatalog::DECIMAL; char buf[42]; // Addition w/o overflow check execplan::IDB_Decimal l, r, result; @@ -146,7 +143,7 @@ TEST(Decimal, additionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("0.00000000000042000000000000000000000042", std::string(buf)); // same precision, L scale > R scale, both negative values @@ -155,7 +152,7 @@ TEST(Decimal, additionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("-0.00000000000042000000000000000000000042", std::string(buf)); // same precision, L scale > R scale, +- values @@ -164,7 +161,7 @@ TEST(Decimal, additionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("-0.00000000000041999999999999999999999958", std::string(buf)); // same precision, L scale > R scale, both 0 @@ -189,7 +186,7 @@ TEST(Decimal, additionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("0.00000000000004200000000000000000000420", std::string(buf)); // same precision, L scale < R scale, both negative values @@ -198,7 +195,7 @@ TEST(Decimal, additionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("-0.00000000000004200000000000000000000420", std::string(buf)); // same precision, L scale < R scale, +- values @@ -207,7 +204,7 @@ TEST(Decimal, additionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("0.00000000000004199999999999999999999580", std::string(buf)); // same precision, L scale < R scale, both 0 @@ -223,8 +220,7 @@ TEST(Decimal, divisionNoOverflowCheck) { // DIVISION // same precision, same scale, both positive values - execplan::CalpontSystemCatalog::ColType ct; - ct.colDataType = execplan::CalpontSystemCatalog::DECIMAL; + datatypes::SystemCatalog::ColDataType colDataType = datatypes::SystemCatalog::DECIMAL; char buf[42]; execplan::IDB_Decimal l, r, result; @@ -241,7 +237,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("9.7674418605", std::string(buf)); // same precision, same scale, both negative values @@ -250,7 +246,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("9.7674418605", std::string(buf)); // same precision, same scale, +- values @@ -259,7 +255,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("-1157.8947368421", std::string(buf)); // same precision, same scale, l = 0 @@ -282,7 +278,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("115789473684210526315789.4736842105", std::string(buf)); // same precision, L scale > R scale, both negative values @@ -291,7 +287,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("86363636363636363636363.6363636364", std::string(buf)); // same precision, L scale > R scale, +- values @@ -300,7 +296,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("-115789473684210526315789.4736842105", std::string(buf)); // same precision, L scale > R scale, R = 0 @@ -317,7 +313,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("100000000000000000000000.0000000000", std::string(buf)); // same precision, L scale > R scale, both MIN negative values @@ -326,7 +322,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("100000000000000000000000.0000000000", std::string(buf)); // same precision, L scale < R scale, both positive values @@ -343,7 +339,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("0.00000000000000000000001000000000000000", std::string(buf)); // same precision, L scale < R scale, both negative values @@ -352,7 +348,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("0.00000000000000000000000863636363636364", std::string(buf)); // same precision, L scale < R scale, +- values @@ -361,7 +357,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("-0.00000000000000000000000863636363636364", std::string(buf)); // same precision, L scale < R scale, R = 0 @@ -379,7 +375,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("0.00000000000000000000001000000000000000", std::string(buf)); // same precision, L scale < R scale, both MIN negative values @@ -388,7 +384,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("0.00000000000000000000001000000000000000", std::string(buf)); // same precision, L scale < R scale, result.scale < (r.scale-l.scale) @@ -406,7 +402,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("1", std::string(buf)); // same precision, L scale < R scale, result.scale < (r.scale-l.scale) @@ -416,7 +412,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("9", std::string(buf)); // same precision, L scale < R scale, result.scale < (r.scale-l.scale) @@ -426,7 +422,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("-9", std::string(buf)); // same precision, L scale < R scale, result.scale < (r.scale-l.scale) @@ -445,7 +441,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("0", std::string(buf)); // same precision, L scale < R scale, result.scale < (r.scale-l.scale) @@ -455,7 +451,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("0", std::string(buf)); } @@ -468,8 +464,7 @@ void doDiv(const execplan::IDB_Decimal& l, TEST(Decimal, divisionWithOverflowCheck) { - execplan::CalpontSystemCatalog::ColType ct; - ct.colDataType = execplan::CalpontSystemCatalog::DECIMAL; + datatypes::SystemCatalog::ColDataType colDataType = datatypes::SystemCatalog::DECIMAL; char buf[42]; // Divide min int128 by -1 execplan::IDB_Decimal l, r, result; @@ -508,7 +503,7 @@ TEST(Decimal, divisionWithOverflowCheck) result.s128Value = 0; EXPECT_NO_THROW(doDiv(l, r, result)); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("9223372036854775809", std::string(buf)); } @@ -521,8 +516,7 @@ void doAdd(const execplan::IDB_Decimal& l, TEST(Decimal, additionWithOverflowCheck) { - execplan::CalpontSystemCatalog::ColType ct; - ct.colDataType = execplan::CalpontSystemCatalog::DECIMAL; + datatypes::SystemCatalog::ColDataType colDataType = datatypes::SystemCatalog::DECIMAL; char buf[42]; // Add two max ints execplan::IDB_Decimal l, r, result; @@ -561,14 +555,13 @@ TEST(Decimal, additionWithOverflowCheck) result.s128Value = 0; EXPECT_NO_THROW(doAdd(l, r, result)); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("-170141183460469231713240559642174554113", std::string(buf)); } TEST(Decimal, subtractionNoOverflowCheck) { - execplan::CalpontSystemCatalog::ColType ct; - ct.colDataType = execplan::CalpontSystemCatalog::DECIMAL; + datatypes::SystemCatalog::ColDataType colDataType = datatypes::SystemCatalog::DECIMAL; char buf[42]; // Subtractio w/o overflow check execplan::IDB_Decimal l, r, result; @@ -586,7 +579,7 @@ TEST(Decimal, subtractionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::subtraction(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("-0.00000000000000000000000000000000000378", std::string(buf)); // same precision, same scale, both negative values @@ -595,7 +588,7 @@ TEST(Decimal, subtractionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::subtraction(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("0.00000000000000000000000000000000000378", std::string(buf)); // same precision, same scale, +- values @@ -604,7 +597,7 @@ TEST(Decimal, subtractionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::subtraction(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("0.00000000000000000000000000000000000462", std::string(buf)); // same precision, same scale, both 0 @@ -630,7 +623,7 @@ TEST(Decimal, subtractionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::subtraction(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("-0.00000000000041999999999999999999999958", std::string(buf)); // same precision, L scale > R scale, both negative values @@ -639,7 +632,7 @@ TEST(Decimal, subtractionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::subtraction(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("0.00000000000041999999999999999999999958", std::string(buf)); // same precision, L scale > R scale, +- values @@ -648,7 +641,7 @@ TEST(Decimal, subtractionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::subtraction(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("0.00000000000042000000000000000000000042", std::string(buf)); // same precision, L scale > R scale, both 0 @@ -673,7 +666,7 @@ TEST(Decimal, subtractionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::subtraction(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("0.00000000000004199999999999999999999580", std::string(buf)); // same precision, L scale < R scale, both negative values @@ -682,7 +675,7 @@ TEST(Decimal, subtractionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::subtraction(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("-0.00000000000004199999999999999999999580", std::string(buf)); // same precision, L scale < R scale, +- values @@ -691,7 +684,7 @@ TEST(Decimal, subtractionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::subtraction(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("0.00000000000004200000000000000000000420", std::string(buf)); // same precision, L scale < R scale, both 0 @@ -712,8 +705,7 @@ void doSubtract(const execplan::IDB_Decimal& l, TEST(Decimal, subtractionWithOverflowCheck) { - execplan::CalpontSystemCatalog::ColType ct; - ct.colDataType = execplan::CalpontSystemCatalog::DECIMAL; + datatypes::SystemCatalog::ColDataType colDataType = datatypes::SystemCatalog::DECIMAL; char buf[42]; // Subtract a max int from a min int execplan::IDB_Decimal l, r, result; @@ -752,7 +744,7 @@ TEST(Decimal, subtractionWithOverflowCheck) result.s128Value = 0; EXPECT_NO_THROW(doSubtract(l, r, result)); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("170141183460469231713240559642174554112", std::string(buf)); } @@ -760,8 +752,7 @@ TEST(Decimal, multiplicationNoOverflowCheck) { // Multiplication // same precision, l.scale + r.scale = result.scale, both positive values - execplan::CalpontSystemCatalog::ColType ct; - ct.colDataType = execplan::CalpontSystemCatalog::DECIMAL; + datatypes::SystemCatalog::ColDataType colDataType = datatypes::SystemCatalog::DECIMAL; char buf[42]; execplan::IDB_Decimal l, r, result; @@ -778,7 +769,7 @@ TEST(Decimal, multiplicationNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::multiplication(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("0.85070591730234615861231965839514664960", std::string(buf)); // same precision, l.scale + r.scale = result.scale, both negative values @@ -788,7 +779,7 @@ TEST(Decimal, multiplicationNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::multiplication(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("0.85070591730234615861231965839514664960", std::string(buf)); // same precision, l.scale + r.scale = result.scale, +- values @@ -797,7 +788,7 @@ TEST(Decimal, multiplicationNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::multiplication(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("-0.85070591730234615861231965839514664960", std::string(buf)); // same precision, l.scale + r.scale = result.scale, l = 0 @@ -822,7 +813,7 @@ TEST(Decimal, multiplicationNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::multiplication(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("0.66461399789245793645190353014017228800", std::string(buf)); // same precision, l.scale + r.scale < result.scale, both negative values @@ -832,7 +823,7 @@ TEST(Decimal, multiplicationNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::multiplication(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("0.66461399789245793645190353014017228800", std::string(buf)); // same precision, l.scale + r.scale < result.scale, +- values @@ -841,7 +832,7 @@ TEST(Decimal, multiplicationNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::multiplication(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("-0.66461399789245793645190353014017228800", std::string(buf)); // same precision, l.scale + r.scale < result.scale, l = 0 @@ -868,7 +859,7 @@ TEST(Decimal, multiplicationNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::multiplication(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("0.01524157875323883675019051998750190521", std::string(buf)); // same precision, l.scale + r.scale > result.scale, both negative values @@ -877,7 +868,7 @@ TEST(Decimal, multiplicationNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::multiplication(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("0.01524157875323883675019051998750190521", std::string(buf)); // same precision, l.scale + r.scale > result.scale, +- values @@ -885,7 +876,7 @@ TEST(Decimal, multiplicationNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::multiplication(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("-0.01524157875323883675019051998750190521", std::string(buf)); // same precision, l.scale + r.scale > result.scale, l = 0 @@ -905,8 +896,7 @@ void doMultiply(const execplan::IDB_Decimal& l, TEST(Decimal, multiplicationWithOverflowCheck) { - execplan::CalpontSystemCatalog::ColType ct; - ct.colDataType = execplan::CalpontSystemCatalog::DECIMAL; + datatypes::SystemCatalog::ColDataType colDataType = datatypes::SystemCatalog::DECIMAL; char buf[42]; execplan::IDB_Decimal l, r, result; // result.scale >= l.scale + r.scale @@ -950,6 +940,6 @@ TEST(Decimal, multiplicationWithOverflowCheck) result.s128Value = 0; EXPECT_NO_THROW(doMultiply(l, r, result)); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, ct.colDataType); + dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); EXPECT_EQ("21267647932558653966460912964485513216", std::string(buf)); } diff --git a/tests/rowgroup-tests.cpp b/tests/rowgroup-tests.cpp index d53356d3a..7232d4898 100644 --- a/tests/rowgroup-tests.cpp +++ b/tests/rowgroup-tests.cpp @@ -20,7 +20,6 @@ #include "rowgroup.h" #include "columnwidth.h" -#include "widedecimalutils.h" #include "joblisttypes.h" #include "dataconvert.h" @@ -98,7 +97,7 @@ protected: int128_t nullValue = 0; int128_t bigValue = 0; - utils::setWideDecimalNullValue(nullValue); + datatypes::Decimal::setWideDecimalNullValue(nullValue); bigValue = -static_cast(0xFFFFFFFF)*0xFFFFFFFFFFFFFFFF; sValueVector.push_back(nullValue); diff --git a/tools/editem/editem.cpp b/tools/editem/editem.cpp index 9ed40b2aa..dd39875df 100644 --- a/tools/editem/editem.cpp +++ b/tools/editem/editem.cpp @@ -44,7 +44,6 @@ using namespace config; #include "dataconvert.h" using namespace dataconvert; -#include "widedecimalutils.h" #include "mcs_decimal.h" #include "liboamcpp.h" @@ -192,12 +191,12 @@ const string fmt(T v) } else { - char buf[utils::MAXLENGTH16BYTES]; + char buf[datatypes::Decimal::MAXLENGTH16BYTES]; int128_t tmp = v; dataconvert::DataConvert::decimalToString( - &tmp, 0, buf, sizeof(buf), execplan::CalpontSystemCatalog::DECIMAL); + &tmp, 0, buf, (uint8_t) sizeof(buf), datatypes::SystemCatalog::DECIMAL); oss << buf; } @@ -219,12 +218,12 @@ const string fmt(T v) } else { - char buf[utils::MAXLENGTH16BYTES]; + char buf[datatypes::Decimal::MAXLENGTH16BYTES]; int128_t tmp = static_cast(v); dataconvert::DataConvert::decimalToString( - &tmp, 0, buf, sizeof(buf), execplan::CalpontSystemCatalog::DECIMAL); + &tmp, 0, buf, (uint8_t) sizeof(buf), datatypes::SystemCatalog::DECIMAL); oss << buf; } @@ -248,12 +247,12 @@ const string fmt(T v) } else { - char buf[utils::MAXLENGTH16BYTES]; + char buf[datatypes::Decimal::MAXLENGTH16BYTES]; int128_t tmp = v; dataconvert::DataConvert::decimalToString( - &tmp, 0, buf, sizeof(buf), execplan::CalpontSystemCatalog::DECIMAL); + &tmp, 0, buf, (uint8_t) sizeof(buf), datatypes::SystemCatalog::DECIMAL); oss << buf; } diff --git a/utils/cloudio/CMakeLists.txt b/utils/cloudio/CMakeLists.txt index 1e00662e8..f94924b99 100755 --- a/utils/cloudio/CMakeLists.txt +++ b/utils/cloudio/CMakeLists.txt @@ -1,6 +1,6 @@ include_directories(${ENGINE_COMMON_INCLUDES} ${ENGINE_SRC_DIR}/storage-manager/include) -set(cloudio_LIB_SRCS SMComm.cpp SMDataFile.cpp SMFileFactory.cpp SMFileSystem.cpp SocketPool.cpp cloud_plugin.cpp) +set(cloudio_LIB_SRCS SMComm.cpp SMDataFile.cpp SMFileFactory.cpp SMFileSystem.cpp SocketPool.cpp cloud_plugin.cpp ../../datatypes/mcs_datatype.cpp) add_library(cloudio SHARED ${cloudio_LIB_SRCS}) diff --git a/utils/common/emptyvaluemanip.cpp b/utils/common/emptyvaluemanip.cpp index fc97d2f14..3300bc690 100644 --- a/utils/common/emptyvaluemanip.cpp +++ b/utils/common/emptyvaluemanip.cpp @@ -15,7 +15,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -#include "widedecimalutils.h" #include "emptyvaluemanip.h" namespace utils @@ -81,7 +80,7 @@ void getEmptyRowValue(const execplan::CalpontSystemCatalog::ColDataType colDataT else if (width <= 8) *(uint64_t*)emptyVal = joblist::BIGINTEMPTYROW; else - setWideDecimalEmptyValue(*(reinterpret_cast(emptyVal))); + datatypes::Decimal::setWideDecimalEmptyValue(*(reinterpret_cast(emptyVal))); break; //case CalpontSystemCatalog::BINARY: diff --git a/utils/common/widedecimalutils.h b/utils/common/widedecimalutils.h index 6efa1b595..b6f1ad5e9 100644 --- a/utils/common/widedecimalutils.h +++ b/utils/common/widedecimalutils.h @@ -29,52 +29,10 @@ namespace utils const uint64_t BINARYNULLVALUEHIGH = 0x8000000000000000ULL; const uint64_t BINARYEMPTYVALUELOW = 1ULL; const uint64_t BINARYEMPTYVALUEHIGH = 0x8000000000000000ULL; - const uint8_t MAXLENGTH16BYTES = 42; - const uint8_t MAXLENGTH8BYTES = 23; const int128_t minInt128 = int128_t(0x8000000000000000LL) << 64; const int128_t maxInt128 = (int128_t(0x7FFFFFFFFFFFFFFFLL) << 64) + 0xFFFFFFFFFFFFFFFFLL; - inline bool isWideDecimalNullValue(const int128_t& val) - { - const uint64_t* ptr = reinterpret_cast(&val); - return (ptr[0] == BINARYNULLVALUELOW && ptr[1] == BINARYNULLVALUEHIGH); - } - - inline bool isWideDecimalEmptyValue(const int128_t& val) - { - const uint64_t* ptr = reinterpret_cast(&val); - return (ptr[0] == BINARYEMPTYVALUELOW && ptr[1] == BINARYEMPTYVALUEHIGH); - } - - inline void setWideDecimalNullValue(int128_t& val) - { - uint64_t* ptr = reinterpret_cast(&val); - ptr[0] = BINARYNULLVALUELOW; - ptr[1] = BINARYNULLVALUEHIGH; - } - - inline void setWideDecimalEmptyValue(int128_t& val) - { - uint64_t* ptr = reinterpret_cast(&val); - ptr[0] = BINARYEMPTYVALUELOW; - ptr[1] = BINARYEMPTYVALUEHIGH; - } - - inline void setWideDecimalNullValue(int128_t* val) - { - uint64_t* ptr = reinterpret_cast(val); - ptr[0] = BINARYNULLVALUELOW; - ptr[1] = BINARYNULLVALUEHIGH; - } - - inline void setWideDecimalEmptyValue(int128_t* val) - { - uint64_t* ptr = reinterpret_cast(val); - ptr[0] = BINARYEMPTYVALUELOW; - ptr[1] = BINARYEMPTYVALUEHIGH; - } - inline void int128Max(int128_t& val) { uint64_t* ptr = reinterpret_cast(&val); diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index b7a0f9985..3651e990f 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -34,10 +34,8 @@ using namespace std; #include using namespace boost::algorithm; #include -#include "calpontsystemcatalog.h" #include "calpontselectexecutionplan.h" #include "columnresult.h" -using namespace execplan; #include "joblisttypes.h" @@ -109,7 +107,8 @@ namespace dataconvert template void number_int_value(const string& data, - const CalpontSystemCatalog::ColType& ct, + cscDataType typeCode, + const datatypes::SystemCatalog::TypeAttributesStd& ct, bool& pushwarning, bool noRoundup, T& intVal) @@ -298,9 +297,9 @@ void number_int_value(const string& data, if (frnVal != 0) pushwarning = true; - switch (ct.colDataType) + switch (typeCode) { - case CalpontSystemCatalog::TINYINT: + case datatypes::SystemCatalog::TINYINT: if (intVal < MIN_TINYINT) { intVal = MIN_TINYINT; @@ -314,7 +313,7 @@ void number_int_value(const string& data, break; - case CalpontSystemCatalog::SMALLINT: + case datatypes::SystemCatalog::SMALLINT: if (intVal < MIN_SMALLINT) { intVal = MIN_SMALLINT; @@ -328,7 +327,7 @@ void number_int_value(const string& data, break; - case CalpontSystemCatalog::MEDINT: + case datatypes::SystemCatalog::MEDINT: if (intVal < MIN_MEDINT) { intVal = MIN_MEDINT; @@ -342,7 +341,7 @@ void number_int_value(const string& data, break; - case CalpontSystemCatalog::INT: + case datatypes::SystemCatalog::INT: if (intVal < MIN_INT) { intVal = MIN_INT; @@ -356,7 +355,7 @@ void number_int_value(const string& data, break; - case CalpontSystemCatalog::BIGINT: + case datatypes::SystemCatalog::BIGINT: if (intVal < MIN_BIGINT) { intVal = MIN_BIGINT; @@ -365,8 +364,8 @@ void number_int_value(const string& data, break; - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::UDECIMAL: if (LIKELY(ct.colWidth == 16)) { int128_t tmp; @@ -432,8 +431,8 @@ void number_int_value(const string& data, } // @ bug 3285 make sure the value is in precision range for decimal data type - if ( (ct.colDataType == CalpontSystemCatalog::DECIMAL) || - (ct.colDataType == CalpontSystemCatalog::UDECIMAL) || + if ( (typeCode == datatypes::SystemCatalog::DECIMAL) || + (typeCode == datatypes::SystemCatalog::UDECIMAL) || (ct.scale > 0)) { T rangeUp, rangeLow; @@ -467,20 +466,23 @@ void number_int_value(const string& data, // Explicit template instantiation template void number_int_value(const std::string& data, - const execplan::CalpontSystemCatalog::ColType& ct, + cscDataType typeCode, + const datatypes::SystemCatalog::TypeAttributesStd& ct, bool& pushwarning, bool noRoundup, int64_t& intVal); template void number_int_value(const std::string& data, - const execplan::CalpontSystemCatalog::ColType& ct, + cscDataType typeCode, + const datatypes::SystemCatalog::TypeAttributesStd& ct, bool& pushwarning, bool noRoundup, int128_t& intVal); uint64_t number_uint_value(const string& data, - const CalpontSystemCatalog::ColType& ct, + cscDataType typeCode, + const datatypes::SystemCatalog::TypeAttributesStd& ct, bool& pushwarning, bool noRoundup) { @@ -585,9 +587,9 @@ uint64_t number_uint_value(const string& data, if (frnVal != 0) pushwarning = true; - switch (ct.colDataType) + switch (typeCode) { - case CalpontSystemCatalog::UTINYINT: + case datatypes::SystemCatalog::UTINYINT: if (uintVal > MAX_UTINYINT) { uintVal = MAX_UTINYINT; @@ -596,7 +598,7 @@ uint64_t number_uint_value(const string& data, break; - case CalpontSystemCatalog::USMALLINT: + case datatypes::SystemCatalog::USMALLINT: if (uintVal > MAX_USMALLINT) { uintVal = MAX_USMALLINT; @@ -605,7 +607,7 @@ uint64_t number_uint_value(const string& data, break; - case CalpontSystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UMEDINT: if (uintVal > MAX_UMEDINT) { uintVal = MAX_UMEDINT; @@ -614,7 +616,7 @@ uint64_t number_uint_value(const string& data, break; - case CalpontSystemCatalog::UINT: + case datatypes::SystemCatalog::UINT: if (uintVal > MAX_UINT) { uintVal = MAX_UINT; @@ -623,7 +625,7 @@ uint64_t number_uint_value(const string& data, break; - case CalpontSystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UBIGINT: if (uintVal > MAX_UBIGINT) { uintVal = MAX_UBIGINT; @@ -1338,7 +1340,8 @@ void DataConvert::decimalToString(int128_t* dec, char* original_p = p; size_t written = 0; // Raise exception on NULL and EMPTY value - if (utils::isWideDecimalNullValue(*dec) || utils::isWideDecimalEmptyValue(*dec)) + if (datatypes::Decimal::isWideDecimalNullValue(*dec) || + datatypes::Decimal::isWideDecimalEmptyValue(*dec)) { throw QueryDataExcept("toString() char buffer overflow.", formatErr); } @@ -1370,7 +1373,8 @@ void DataConvert::decimalToString(int128_t* dec, } boost::any -DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, +DataConvert::convertColumnData(cscDataType typeCode, + const datatypes::SystemCatalog::TypeAttributesStd& colType, const std::string& dataOrig, bool& pushWarning, const std::string& timeZone, bool nulFlag, bool noRoundup, bool isUpdate) @@ -1380,14 +1384,13 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, // WIP std::string data( dataOrig ); pushWarning = false; - cscDataType type = colType.colDataType; //if ( !data.empty() ) if (!nulFlag) { - switch (type) + switch (typeCode) { - case CalpontSystemCatalog::BIT: + case datatypes::SystemCatalog::BIT: { unsigned int x = data.find("("); @@ -1405,7 +1408,7 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, int64_t tmp = 0; - number_int_value (data, colType, pushWarning, noRoundup, tmp); + number_int_value (data, typeCode, colType, pushWarning, noRoundup, tmp); if (tmp) { @@ -1423,52 +1426,52 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } break; - case CalpontSystemCatalog::TINYINT: - number_int_value(data, colType, pushWarning, noRoundup, val64); + case datatypes::SystemCatalog::TINYINT: + number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); value = (char) val64; break; - case CalpontSystemCatalog::SMALLINT: - number_int_value(data, colType, pushWarning, noRoundup, val64); + case datatypes::SystemCatalog::SMALLINT: + number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); value = (short) val64; break; - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - number_int_value(data, colType, pushWarning, noRoundup, val64); + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); value = (int) val64; break; - case CalpontSystemCatalog::BIGINT: - number_int_value(data, colType, pushWarning, noRoundup, val64); + case datatypes::SystemCatalog::BIGINT: + number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); value = (long long) val64; break; - case CalpontSystemCatalog::DECIMAL: + case datatypes::SystemCatalog::DECIMAL: if (LIKELY(colType.colWidth == 16)) { int128_t val128; - number_int_value(data, colType, pushWarning, noRoundup, val128); + number_int_value(data, typeCode, colType, pushWarning, noRoundup, val128); value = (int128_t) val128; } else if (colType.colWidth == 8) { - number_int_value(data, colType, pushWarning, noRoundup, val64); + number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); value = (long long) val64; } else if (colType.colWidth == 4) { - number_int_value(data, colType, pushWarning, noRoundup, val64); + number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); value = (int) val64; } else if (colType.colWidth == 2) { - number_int_value(data, colType, pushWarning, noRoundup, val64); + number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); value = (short) val64; } else if (colType.colWidth == 1) { - number_int_value(data, colType, pushWarning, noRoundup, val64); + number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); value = (char) val64; } //else if (colType.colWidth == 32) @@ -1476,17 +1479,17 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, break; - case CalpontSystemCatalog::UDECIMAL: + case datatypes::SystemCatalog::UDECIMAL: // UDECIMAL numbers may not be negative if (LIKELY(colType.colWidth == 16)) { int128_t val128; - number_int_value(data, colType, pushWarning, noRoundup, val128); + number_int_value(data, typeCode, colType, pushWarning, noRoundup, val128); if (val128 < 0 && - !utils::isWideDecimalNullValue(val128) && - !utils::isWideDecimalEmptyValue(val128)) + !datatypes::Decimal::isWideDecimalNullValue(val128) && + !datatypes::Decimal::isWideDecimalEmptyValue(val128)) { val128 = 0; pushWarning = true; @@ -1496,7 +1499,7 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } else if (colType.colWidth == 8) { - number_int_value(data, colType, pushWarning, noRoundup, val64); + number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); long long ival = static_cast(val64); if (ival < 0 && @@ -1511,7 +1514,7 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } else if (colType.colWidth == 4) { - number_int_value(data, colType, pushWarning, noRoundup, val64); + number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); int ival = static_cast(val64); if (ival < 0 && @@ -1526,7 +1529,7 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } else if (colType.colWidth == 2) { - number_int_value(data, colType, pushWarning, noRoundup, val64); + number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); short ival = (short) val64; if (ival < 0 && @@ -1541,7 +1544,7 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } else if (colType.colWidth == 1) { - number_int_value(data, colType, pushWarning, noRoundup, val64); + number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); char ival = (char) val64; if (ival < 0 && @@ -1557,8 +1560,8 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, break; - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::UFLOAT: { string::size_type x = data.find('('); @@ -1615,8 +1618,10 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, floatvalue = 0; } - if (floatvalue < 0.0 && type == CalpontSystemCatalog::UFLOAT && - floatvalue != joblist::FLOATEMPTYROW && floatvalue != joblist::FLOATNULL) + if (floatvalue < 0.0 && + typeCode == datatypes::SystemCatalog::UFLOAT && + floatvalue != joblist::FLOATEMPTYROW && + floatvalue != joblist::FLOATNULL) { value = 0.0; pushWarning = true; @@ -1629,8 +1634,8 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } break; - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UDOUBLE: { string::size_type x = data.find('('); @@ -1669,8 +1674,10 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, else value = doublevalue; - if (doublevalue < 0.0 && type == CalpontSystemCatalog::UDOUBLE && - doublevalue != joblist::DOUBLEEMPTYROW && doublevalue != joblist::DOUBLENULL) + if (doublevalue < 0.0 && + typeCode == datatypes::SystemCatalog::UDOUBLE && + doublevalue != joblist::DOUBLEEMPTYROW && + doublevalue != joblist::DOUBLENULL) { doublevalue = 0.0; pushWarning = true; @@ -1683,26 +1690,26 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } break; - case CalpontSystemCatalog::UTINYINT: - value = (uint8_t)number_uint_value(data, colType, pushWarning, noRoundup); + case datatypes::SystemCatalog::UTINYINT: + value = (uint8_t)number_uint_value(data, typeCode, colType, pushWarning, noRoundup); break; - case CalpontSystemCatalog::USMALLINT: - value = (uint16_t)number_uint_value(data, colType, pushWarning, noRoundup); + case datatypes::SystemCatalog::USMALLINT: + value = (uint16_t)number_uint_value(data, typeCode, colType, pushWarning, noRoundup); break; - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - value = (uint32_t)number_uint_value(data, colType, pushWarning, noRoundup); + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + value = (uint32_t)number_uint_value(data, typeCode, colType, pushWarning, noRoundup); break; - case CalpontSystemCatalog::UBIGINT: - value = (uint64_t)number_uint_value(data, colType, pushWarning, noRoundup); + case datatypes::SystemCatalog::UBIGINT: + value = (uint64_t)number_uint_value(data, typeCode, colType, pushWarning, noRoundup); break; - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::TEXT: + case datatypes::SystemCatalog::CHAR: + case datatypes::SystemCatalog::VARCHAR: + case datatypes::SystemCatalog::TEXT: { //check data length if ( data.length() > (unsigned int)colType.colWidth ) @@ -1723,7 +1730,7 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } break; - case CalpontSystemCatalog::DATE: + case datatypes::SystemCatalog::DATE: { Date aDay; @@ -1739,7 +1746,7 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } break; - case CalpontSystemCatalog::DATETIME: + case datatypes::SystemCatalog::DATETIME: { DateTime aDatetime; @@ -1755,7 +1762,7 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } break; - case CalpontSystemCatalog::TIME: + case datatypes::SystemCatalog::TIME: { Time aTime; @@ -1768,7 +1775,7 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } break; - case CalpontSystemCatalog::TIMESTAMP: + case datatypes::SystemCatalog::TIMESTAMP: { TimeStamp aTimestamp; @@ -1781,16 +1788,16 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } break; - case CalpontSystemCatalog::BLOB: - case CalpontSystemCatalog::CLOB: + case datatypes::SystemCatalog::BLOB: + case datatypes::SystemCatalog::CLOB: value = data; break; - case CalpontSystemCatalog::VARBINARY: + case datatypes::SystemCatalog::VARBINARY: value = data; break; - case CalpontSystemCatalog::BINARY: + case datatypes::SystemCatalog::BINARY: value = data; break; @@ -1801,241 +1808,11 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, } else //null { - switch (type) - { - case CalpontSystemCatalog::BIT: - { - //TODO: How to communicate with write engine? - } - break; - - case CalpontSystemCatalog::TINYINT: - { - char tinyintvalue = joblist::TINYINTNULL; - value = tinyintvalue; - } - break; - - case CalpontSystemCatalog::SMALLINT: - { - short smallintvalue = joblist::SMALLINTNULL; - value = smallintvalue; - } - break; - - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - { - int intvalue = joblist::INTNULL; - value = intvalue; - } - break; - - case CalpontSystemCatalog::BIGINT: - { - long long bigint = joblist::BIGINTNULL; - value = bigint; - } - break; - - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UDECIMAL: - { - if (LIKELY(colType.colWidth == 16)) - { - int128_t val; - utils::setWideDecimalNullValue(val); - value = val; - } - else if (colType.colWidth == CalpontSystemCatalog::EIGHT_BYTE) - { - long long eightbyte = joblist::BIGINTNULL; - value = eightbyte; - } - else if (colType.colWidth == CalpontSystemCatalog::FOUR_BYTE) - { - int intvalue = joblist::INTNULL; - value = intvalue; - } - else if (colType.colWidth == CalpontSystemCatalog::TWO_BYTE) - { - short smallintvalue = joblist::SMALLINTNULL; - value = smallintvalue; - } - else if (colType.colWidth == CalpontSystemCatalog::ONE_BYTE) - { - char tinyintvalue = joblist::TINYINTNULL; - value = tinyintvalue; - } - else - { - WriteEngine::Token nullToken; - value = nullToken; - } - } - break; - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::UFLOAT: - { - uint32_t tmp = joblist::FLOATNULL; - float* floatvalue = (float*)&tmp; - value = *floatvalue; - } - break; - - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UDOUBLE: - { - uint64_t tmp = joblist::DOUBLENULL; - double* doublevalue = (double*)&tmp; - value = *doublevalue; - } - break; - - case CalpontSystemCatalog::DATE: - { - uint32_t d = joblist::DATENULL; - value = d; - } - break; - - case CalpontSystemCatalog::DATETIME: - { - uint64_t d = joblist::DATETIMENULL; - value = d; - } - break; - - case CalpontSystemCatalog::TIMESTAMP: - { - uint64_t d = joblist::TIMESTAMPNULL; - value = d; - } - break; - - case CalpontSystemCatalog::TIME: - { - uint64_t d = joblist::TIMENULL; - value = d; - } - break; - - case CalpontSystemCatalog::CHAR: - { - std::string charnull; - - if (colType.colWidth == 1) - { - //charnull = joblist::CHAR1NULL; - charnull = '\376'; - value = charnull; - } - else if (colType.colWidth == 2) - { - //charnull = joblist::CHAR2NULL; - charnull = "\377\376"; - value = charnull; - } - else if (( colType.colWidth < 5 ) && ( colType.colWidth > 2 )) - { - //charnull = joblist::CHAR4NULL; - charnull = "\377\377\377\376"; - value = charnull; - } - else if (( colType.colWidth < 9 ) && ( colType.colWidth > 4 )) - { - //charnull = joblist::CHAR8NULL; - charnull = "\377\377\377\377\377\377\377\376"; - value = charnull; - } - else - { - WriteEngine::Token nullToken; - value = nullToken; - } - } - break; - - case CalpontSystemCatalog::VARCHAR: - case CalpontSystemCatalog::TEXT: - { - std::string charnull; - - if (colType.colWidth == 1 ) - { - //charnull = joblist::CHAR2NULL; - charnull = "\377\376"; - value = charnull; - } - else if ((colType.colWidth < 4) && (colType.colWidth > 1)) - { - //charnull = joblist::CHAR4NULL; - charnull = "\377\377\377\376"; - value = charnull; - } - else if ((colType.colWidth < 8) && (colType.colWidth > 3)) - { - //charnull = joblist::CHAR8NULL; - charnull = "\377\377\377\377\377\377\377\376"; - value = charnull; - } - else if ( colType.colWidth > 7 ) - { - WriteEngine::Token nullToken; - value = nullToken; - } - } - break; - - case CalpontSystemCatalog::VARBINARY: - case CalpontSystemCatalog::BLOB: - { - WriteEngine::Token nullToken; - value = nullToken; - } - break; - - case CalpontSystemCatalog::BINARY: - { - value = data; - } - break; - - case CalpontSystemCatalog::UTINYINT: - { - uint8_t utinyintvalue = joblist::UTINYINTNULL; - value = utinyintvalue; - } - break; - - case CalpontSystemCatalog::USMALLINT: - { - uint16_t usmallintvalue = joblist::USMALLINTNULL; - value = usmallintvalue; - } - break; - - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - { - uint32_t uintvalue = joblist::UINTNULL; - value = uintvalue; - } - break; - - case CalpontSystemCatalog::UBIGINT: - { - uint64_t ubigint = joblist::UBIGINTNULL; - value = ubigint; - } - break; - - default: - throw QueryDataExcept("convertColumnData: unknown column data type.", dataTypeErr); - break; - - } + const datatypes::TypeHandler *h= datatypes::TypeHandler::find(typeCode, colType); + if (!h) + throw QueryDataExcept("convertColumnData: unknown column data type.", dataTypeErr); + else + return h->getNullValueForType(colType); } return value; @@ -3349,441 +3126,435 @@ int64_t DataConvert::stringToTime(const string& data) return *(reinterpret_cast(&atime)); } -CalpontSystemCatalog::ColType DataConvert::convertUnionColType(vector& types) + +void +DataConvert::joinColTypeForUnion(datatypes::SystemCatalog::TypeHolderStd &unionedType, + const datatypes::SystemCatalog::TypeHolderStd &type) { - idbassert(types.size()); - - CalpontSystemCatalog::ColType unionedType = types[0]; - - for (uint64_t i = 1; i < types.size(); i++) + // limited support for VARBINARY, no implicit conversion. + if (type.colDataType == datatypes::SystemCatalog::VARBINARY || + unionedType.colDataType == datatypes::SystemCatalog::VARBINARY) { - // limited support for VARBINARY, no implicit conversion. - if (types[i].colDataType == CalpontSystemCatalog::VARBINARY || - unionedType.colDataType == CalpontSystemCatalog::VARBINARY) + if (type.colDataType != unionedType.colDataType || + type.colWidth != unionedType.colWidth) + throw runtime_error("VARBINARY in UNION must be the same width."); + } + + switch (type.colDataType) + { + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UDECIMAL: { - if (types[i].colDataType != unionedType.colDataType || - types[i].colWidth != unionedType.colWidth) - throw runtime_error("VARBINARY in UNION must be the same width."); + switch (unionedType.colDataType) + { + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UDECIMAL: + if (type.colWidth > unionedType.colWidth) + { + unionedType.colDataType = type.colDataType; + unionedType.colWidth = type.colWidth; + } + + // If same size and result is signed but source is unsigned... + if (type.colWidth == unionedType.colWidth && !isUnsigned(unionedType.colDataType) && isUnsigned(type.colDataType)) + { + unionedType.colDataType = type.colDataType; + } + + if (type.colDataType == datatypes::SystemCatalog::DECIMAL || type.colDataType == datatypes::SystemCatalog::UDECIMAL) + { + unionedType.colDataType = datatypes::SystemCatalog::DECIMAL; + } + + unionedType.scale = (type.scale > unionedType.scale) ? type.scale : unionedType.scale; + break; + + case datatypes::SystemCatalog::DATE: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.colWidth = 20; + break; + + case datatypes::SystemCatalog::TIME: + case datatypes::SystemCatalog::DATETIME: + case datatypes::SystemCatalog::TIMESTAMP: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.colWidth = 26; + break; + + case datatypes::SystemCatalog::CHAR: + if (unionedType.colWidth < 20) + unionedType.colWidth = 20; + + break; + + case datatypes::SystemCatalog::VARCHAR: + if (unionedType.colWidth < 21) + unionedType.colWidth = 21; + + break; + + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + case datatypes::SystemCatalog::LONGDOUBLE: + default: + break; + } + + break; } - switch (types[i].colDataType) + case datatypes::SystemCatalog::DATE: { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: + switch (unionedType.colDataType) { - switch (unionedType.colDataType) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: - if (types[i].colWidth > unionedType.colWidth) - { - unionedType.colDataType = types[i].colDataType; - unionedType.colWidth = types[i].colWidth; - } + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UDECIMAL: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + case datatypes::SystemCatalog::LONGDOUBLE: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.scale = 0; + unionedType.colWidth = 20; + break; - // If same size and result is signed but source is unsigned... - if (types[i].colWidth == unionedType.colWidth && !isUnsigned(unionedType.colDataType) && isUnsigned(types[i].colDataType)) - { - unionedType.colDataType = types[i].colDataType; - } + case datatypes::SystemCatalog::CHAR: + if (unionedType.colWidth < 10) + unionedType.colWidth = 10; - if (types[i].colDataType == CalpontSystemCatalog::DECIMAL || types[i].colDataType == CalpontSystemCatalog::UDECIMAL) - { - unionedType.colDataType = CalpontSystemCatalog::DECIMAL; - } + break; - unionedType.scale = (types[i].scale > unionedType.scale) ? types[i].scale : unionedType.scale; - break; + case datatypes::SystemCatalog::VARCHAR: + if (unionedType.colWidth < 11) + unionedType.colWidth = 11; - case CalpontSystemCatalog::DATE: - unionedType.colDataType = CalpontSystemCatalog::CHAR; + break; + + case datatypes::SystemCatalog::DATE: + case datatypes::SystemCatalog::DATETIME: + case datatypes::SystemCatalog::TIMESTAMP: + case datatypes::SystemCatalog::TIME: + default: + break; + } + + break; + } + + case datatypes::SystemCatalog::DATETIME: + { + switch (unionedType.colDataType) + { + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UDECIMAL: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + case datatypes::SystemCatalog::TIME: + case datatypes::SystemCatalog::LONGDOUBLE: + case datatypes::SystemCatalog::TIMESTAMP: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.scale = 0; + unionedType.colWidth = 26; + break; + + case datatypes::SystemCatalog::DATE: + unionedType.colDataType = datatypes::SystemCatalog::DATETIME; + unionedType.colWidth = type.colWidth; + break; + + case datatypes::SystemCatalog::CHAR: + if (unionedType.colWidth < 26) + unionedType.colWidth = 26; + + break; + + case datatypes::SystemCatalog::VARCHAR: + if (unionedType.colWidth < 27) + unionedType.colWidth = 27; + + break; + + case datatypes::SystemCatalog::DATETIME: + default: + break; + } + + break; + } + + case datatypes::SystemCatalog::TIMESTAMP: + { + switch (unionedType.colDataType) + { + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UDECIMAL: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + case datatypes::SystemCatalog::TIME: + case datatypes::SystemCatalog::DATETIME: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.scale = 0; + unionedType.colWidth = 26; + break; + + case datatypes::SystemCatalog::DATE: + unionedType.colDataType = datatypes::SystemCatalog::TIMESTAMP; + unionedType.colWidth = type.colWidth; + break; + + case datatypes::SystemCatalog::CHAR: + if (unionedType.colWidth < 26) + unionedType.colWidth = 26; + + break; + + case datatypes::SystemCatalog::VARCHAR: + if (unionedType.colWidth < 27) + unionedType.colWidth = 27; + + break; + + case datatypes::SystemCatalog::TIMESTAMP: + default: + break; + } + + break; + } + + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + { + switch (unionedType.colDataType) + { + case datatypes::SystemCatalog::DATE: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.scale = 0; + unionedType.colWidth = 20; + break; + + case datatypes::SystemCatalog::DATETIME: + case datatypes::SystemCatalog::TIMESTAMP: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.scale = 0; + unionedType.colWidth = 26; + break; + + case datatypes::SystemCatalog::CHAR: + if (unionedType.colWidth < 20) unionedType.colWidth = 20; - break; - case CalpontSystemCatalog::TIME: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - unionedType.colDataType = CalpontSystemCatalog::CHAR; - unionedType.colWidth = 26; - break; + break; - case CalpontSystemCatalog::CHAR: - if (unionedType.colWidth < 20) - unionedType.colWidth = 20; + case datatypes::SystemCatalog::VARCHAR: + if (unionedType.colWidth < 21) + unionedType.colWidth = 21; - break; + break; - case CalpontSystemCatalog::VARCHAR: - if (unionedType.colWidth < 21) - unionedType.colWidth = 21; + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UDECIMAL: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + unionedType.colDataType = datatypes::SystemCatalog::DOUBLE; + unionedType.scale = 0; + unionedType.colWidth = sizeof(double); + break; - break; - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UFLOAT: - case CalpontSystemCatalog::UDOUBLE: - case CalpontSystemCatalog::LONGDOUBLE: - default: - break; - } - - break; + default: + break; } - case CalpontSystemCatalog::DATE: + break; + } + + case datatypes::SystemCatalog::LONGDOUBLE: + { + switch (unionedType.colDataType) { - switch (unionedType.colDataType) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: - case CalpontSystemCatalog::UFLOAT: - case CalpontSystemCatalog::UDOUBLE: - case CalpontSystemCatalog::LONGDOUBLE: - unionedType.colDataType = CalpontSystemCatalog::CHAR; - unionedType.scale = 0; + case datatypes::SystemCatalog::DATE: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.scale = 0; + unionedType.colWidth = 20; + break; + + case datatypes::SystemCatalog::DATETIME: + unionedType.colDataType = datatypes::SystemCatalog::CHAR; + unionedType.scale = 0; + unionedType.colWidth = 26; + break; + + case datatypes::SystemCatalog::CHAR: + if (unionedType.colWidth < 20) unionedType.colWidth = 20; - break; - case CalpontSystemCatalog::CHAR: - if (unionedType.colWidth < 10) - unionedType.colWidth = 10; + break; - break; + case datatypes::SystemCatalog::VARCHAR: + if (unionedType.colWidth < 21) + unionedType.colWidth = 21; - case CalpontSystemCatalog::VARCHAR: - if (unionedType.colWidth < 11) - unionedType.colWidth = 11; + break; - break; + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UDECIMAL: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + case datatypes::SystemCatalog::LONGDOUBLE: + unionedType.colDataType = datatypes::SystemCatalog::LONGDOUBLE; + unionedType.scale = (type.scale > unionedType.scale) ? type.scale : unionedType.scale; + unionedType.colWidth = sizeof(long double); + unionedType.precision = -1; + break; - case CalpontSystemCatalog::DATE: - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - case CalpontSystemCatalog::TIME: - default: - break; - } - - break; + default: + break; } - case CalpontSystemCatalog::DATETIME: + break; + } + + case datatypes::SystemCatalog::CHAR: + case datatypes::SystemCatalog::VARCHAR: + { + switch (unionedType.colDataType) { - switch (unionedType.colDataType) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: - case CalpontSystemCatalog::UFLOAT: - case CalpontSystemCatalog::UDOUBLE: - case CalpontSystemCatalog::TIME: - case CalpontSystemCatalog::LONGDOUBLE: - case CalpontSystemCatalog::TIMESTAMP: - unionedType.colDataType = CalpontSystemCatalog::CHAR; - unionedType.scale = 0; - unionedType.colWidth = 26; - break; + case datatypes::SystemCatalog::TINYINT: + case datatypes::SystemCatalog::SMALLINT: + case datatypes::SystemCatalog::MEDINT: + case datatypes::SystemCatalog::INT: + case datatypes::SystemCatalog::BIGINT: + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::FLOAT: + case datatypes::SystemCatalog::DOUBLE: + case datatypes::SystemCatalog::UTINYINT: + case datatypes::SystemCatalog::USMALLINT: + case datatypes::SystemCatalog::UMEDINT: + case datatypes::SystemCatalog::UINT: + case datatypes::SystemCatalog::UBIGINT: + case datatypes::SystemCatalog::UDECIMAL: + case datatypes::SystemCatalog::UFLOAT: + case datatypes::SystemCatalog::UDOUBLE: + case datatypes::SystemCatalog::LONGDOUBLE: + unionedType.scale = 0; + unionedType.colWidth = (type.colWidth > 20) ? type.colWidth : 20; + break; - case CalpontSystemCatalog::DATE: - unionedType.colDataType = CalpontSystemCatalog::DATETIME; - unionedType.colWidth = types[i].colWidth; - break; + case datatypes::SystemCatalog::DATE: + unionedType.colWidth = (type.colWidth > 10) ? type.colWidth : 10; + break; - case CalpontSystemCatalog::CHAR: - if (unionedType.colWidth < 26) - unionedType.colWidth = 26; + case datatypes::SystemCatalog::DATETIME: + case datatypes::SystemCatalog::TIMESTAMP: + unionedType.colWidth = (type.colWidth > 26) ? type.colWidth : 26; + break; - break; + case datatypes::SystemCatalog::CHAR: + case datatypes::SystemCatalog::VARCHAR: - case CalpontSystemCatalog::VARCHAR: - if (unionedType.colWidth < 27) - unionedType.colWidth = 27; + // VARCHAR will fit in CHAR of the same width + if (unionedType.colWidth < type.colWidth) + unionedType.colWidth = type.colWidth; - break; + break; - case CalpontSystemCatalog::DATETIME: - default: - break; - } - - break; + default: + break; } - case CalpontSystemCatalog::TIMESTAMP: - { - switch (unionedType.colDataType) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: - case CalpontSystemCatalog::UFLOAT: - case CalpontSystemCatalog::UDOUBLE: - case CalpontSystemCatalog::TIME: - case CalpontSystemCatalog::DATETIME: - unionedType.colDataType = CalpontSystemCatalog::CHAR; - unionedType.scale = 0; - unionedType.colWidth = 26; - break; + // MariaDB bug 651. Setting to CHAR broke union in subquery + unionedType.colDataType = datatypes::SystemCatalog::VARCHAR; + break; + } - case CalpontSystemCatalog::DATE: - unionedType.colDataType = CalpontSystemCatalog::TIMESTAMP; - unionedType.colWidth = types[i].colWidth; - break; - - case CalpontSystemCatalog::CHAR: - if (unionedType.colWidth < 26) - unionedType.colWidth = 26; - - break; - - case CalpontSystemCatalog::VARCHAR: - if (unionedType.colWidth < 27) - unionedType.colWidth = 27; - - break; - - case CalpontSystemCatalog::TIMESTAMP: - default: - break; - } - - break; - } - - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UFLOAT: - case CalpontSystemCatalog::UDOUBLE: - { - switch (unionedType.colDataType) - { - case CalpontSystemCatalog::DATE: - unionedType.colDataType = CalpontSystemCatalog::CHAR; - unionedType.scale = 0; - unionedType.colWidth = 20; - break; - - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - unionedType.colDataType = CalpontSystemCatalog::CHAR; - unionedType.scale = 0; - unionedType.colWidth = 26; - break; - - case CalpontSystemCatalog::CHAR: - if (unionedType.colWidth < 20) - unionedType.colWidth = 20; - - break; - - case CalpontSystemCatalog::VARCHAR: - if (unionedType.colWidth < 21) - unionedType.colWidth = 21; - - break; - - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: - case CalpontSystemCatalog::UFLOAT: - case CalpontSystemCatalog::UDOUBLE: - unionedType.colDataType = CalpontSystemCatalog::DOUBLE; - unionedType.scale = 0; - unionedType.colWidth = sizeof(double); - break; - - default: - break; - } - - break; - } - - case CalpontSystemCatalog::LONGDOUBLE: - { - switch (unionedType.colDataType) - { - case CalpontSystemCatalog::DATE: - unionedType.colDataType = CalpontSystemCatalog::CHAR; - unionedType.scale = 0; - unionedType.colWidth = 20; - break; - - case CalpontSystemCatalog::DATETIME: - unionedType.colDataType = CalpontSystemCatalog::CHAR; - unionedType.scale = 0; - unionedType.colWidth = 26; - break; - - case CalpontSystemCatalog::CHAR: - if (unionedType.colWidth < 20) - unionedType.colWidth = 20; - - break; - - case CalpontSystemCatalog::VARCHAR: - if (unionedType.colWidth < 21) - unionedType.colWidth = 21; - - break; - - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: - case CalpontSystemCatalog::UFLOAT: - case CalpontSystemCatalog::UDOUBLE: - case CalpontSystemCatalog::LONGDOUBLE: - unionedType.colDataType = CalpontSystemCatalog::LONGDOUBLE; - unionedType.scale = (types[i].scale > unionedType.scale) ? types[i].scale : unionedType.scale; - unionedType.colWidth = sizeof(long double); - unionedType.precision = -1; - break; - - default: - break; - } - - break; - } - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - { - switch (unionedType.colDataType) - { - case CalpontSystemCatalog::TINYINT: - case CalpontSystemCatalog::SMALLINT: - case CalpontSystemCatalog::MEDINT: - case CalpontSystemCatalog::INT: - case CalpontSystemCatalog::BIGINT: - case CalpontSystemCatalog::DECIMAL: - case CalpontSystemCatalog::FLOAT: - case CalpontSystemCatalog::DOUBLE: - case CalpontSystemCatalog::UTINYINT: - case CalpontSystemCatalog::USMALLINT: - case CalpontSystemCatalog::UMEDINT: - case CalpontSystemCatalog::UINT: - case CalpontSystemCatalog::UBIGINT: - case CalpontSystemCatalog::UDECIMAL: - case CalpontSystemCatalog::UFLOAT: - case CalpontSystemCatalog::UDOUBLE: - case CalpontSystemCatalog::LONGDOUBLE: - unionedType.scale = 0; - unionedType.colWidth = (types[i].colWidth > 20) ? types[i].colWidth : 20; - break; - - case CalpontSystemCatalog::DATE: - unionedType.colWidth = (types[i].colWidth > 10) ? types[i].colWidth : 10; - break; - - case CalpontSystemCatalog::DATETIME: - case CalpontSystemCatalog::TIMESTAMP: - unionedType.colWidth = (types[i].colWidth > 26) ? types[i].colWidth : 26; - break; - - case CalpontSystemCatalog::CHAR: - case CalpontSystemCatalog::VARCHAR: - - // VARCHAR will fit in CHAR of the same width - if (unionedType.colWidth < types[i].colWidth) - unionedType.colWidth = types[i].colWidth; - - break; - - default: - break; - } - - // MariaDB bug 651. Setting to CHAR broke union in subquery - unionedType.colDataType = CalpontSystemCatalog::VARCHAR; - break; - } - - default: - { - break; - } - } // switch - } // for - - return unionedType; + default: + { + break; + } + } // switch } } // namespace dataconvert diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 2636919e0..4b1dfa96b 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -47,12 +47,11 @@ #include #endif -#include "calpontsystemcatalog.h" +#include "mcs_datatype.h" #include "columnresult.h" #include "exceptclasses.h" #include "common/branchpred.h" -#include "widedecimalutils.h" // remove this block if the htonll is defined in library #ifdef __linux__ @@ -87,7 +86,7 @@ inline uint64_t uint64ToStr(uint64_t n) return htonll(n); } -using cscDataType = execplan::CalpontSystemCatalog::ColDataType; +using cscDataType = datatypes::SystemCatalog::ColDataType; #if defined(_MSC_VER) && defined(xxxDATACONVERT_DLLEXPORT) #define EXPORT __declspec(dllexport) @@ -883,7 +882,8 @@ uint64_t string_to_ull( const std::string& data, bool& bSaturate ) template void number_int_value(const std::string& data, - const execplan::CalpontSystemCatalog::ColType& ct, + cscDataType typeCode, + const datatypes::SystemCatalog::TypeAttributesStd &ct, bool& pushwarning, bool noRoundup, T& intVal); @@ -901,7 +901,9 @@ public: * @param type the columns data type * @param data the columns string representation of it's data */ - EXPORT static boost::any convertColumnData( const execplan::CalpontSystemCatalog::ColType& colType, + EXPORT static boost::any convertColumnData( + cscDataType typecode, + const datatypes::SystemCatalog::TypeAttributesStd& attr, const std::string& dataOrig, bool& bSaturate, const std::string& timeZone, bool nulFlag = false, bool noRoundup = false, bool isUpdate = false); @@ -1046,7 +1048,6 @@ public: EXPORT static bool isColumnTimeValid( int64_t time ); EXPORT static bool isColumnTimeStampValid( int64_t timeStamp ); - EXPORT static bool isNullData(execplan::ColumnResult* cr, int rownum, execplan::CalpontSystemCatalog::ColType colType); static inline std::string decimalToString(int64_t value, uint8_t scale, cscDataType colDataType); static inline void decimalToString(int64_t value, uint8_t scale, char* buf, unsigned int buflen, cscDataType colDataType); @@ -1083,7 +1084,8 @@ public: EXPORT static int64_t timeToInt(const std::string& time); EXPORT static int64_t stringToTime (const std::string& data); // bug4388, union type conversion - EXPORT static execplan::CalpontSystemCatalog::ColType convertUnionColType(std::vector&); + EXPORT static void joinColTypeForUnion(datatypes::SystemCatalog::TypeHolderStd &unionedType, + const datatypes::SystemCatalog::TypeHolderStd &type); }; inline void DataConvert::dateToString( int datevalue, char* buf, unsigned int buflen) diff --git a/utils/funcexp/func_round.cpp b/utils/funcexp/func_round.cpp index 780b499b0..4e3ad260c 100644 --- a/utils/funcexp/func_round.cpp +++ b/utils/funcexp/func_round.cpp @@ -116,7 +116,7 @@ int64_t Func_round::getIntVal(Row& row, { IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); - if (!datatypes::Decimal::isWideDecimalType(op_ct)) + if (!op_ct.isWideDecimalType()) { if (x.scale > 0) { @@ -207,7 +207,7 @@ double Func_round::getDoubleVal(Row& row, double d; - if (!datatypes::Decimal::isWideDecimalType(op_ct)) + if (!op_ct.isWideDecimalType()) d = x.value; else d = datatypes::Decimal::getDoubleFromWideDecimal(x.s128Value); @@ -274,7 +274,7 @@ long double Func_round::getLongDoubleVal(Row& row, double d; - if (!datatypes::Decimal::isWideDecimalType(op_ct)) + if (!op_ct.isWideDecimalType()) d = x.value; else d = datatypes::Decimal::getDoubleFromWideDecimal(x.s128Value); @@ -313,7 +313,7 @@ IDB_Decimal Func_round::getDecimalVal(Row& row, int64_t d = 0; decimal = parm[0]->data()->getDecimalVal(row, isNull); - if (!datatypes::Decimal::isWideDecimalType(op_ct)) + if (!op_ct.isWideDecimalType()) { //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. volatile int64_t p = 1; @@ -720,12 +720,12 @@ string Func_round::getStrVal(Row& row, break; } - if (!datatypes::Decimal::isWideDecimalType(op_ct)) + if (!op_ct.isWideDecimalType()) return dataconvert::DataConvert::decimalToString(x.value, x.scale, op_ct.colDataType); else { - char buf[utils::MAXLENGTH16BYTES]; - dataconvert::DataConvert::decimalToString( &x.s128Value, x.scale, buf, utils::MAXLENGTH16BYTES, op_ct.colDataType); + char buf[datatypes::Decimal::MAXLENGTH16BYTES]; + dataconvert::DataConvert::decimalToString( &x.s128Value, x.scale, buf, (uint8_t) sizeof(buf), op_ct.colDataType); return string(buf); } } diff --git a/utils/funcexp/func_truncate.cpp b/utils/funcexp/func_truncate.cpp index d60fa6e70..9ab383774 100644 --- a/utils/funcexp/func_truncate.cpp +++ b/utils/funcexp/func_truncate.cpp @@ -119,7 +119,7 @@ int64_t Func_truncate::getIntVal(Row& row, { IDB_Decimal x = getDecimalVal(row, parm, isNull, op_ct); - if (!datatypes::Decimal::isWideDecimalType(op_ct)) + if (!op_ct.isWideDecimalType()) { if (x.scale > 0) { @@ -230,7 +230,7 @@ double Func_truncate::getDoubleVal(Row& row, double d; - if (!datatypes::Decimal::isWideDecimalType(op_ct)) + if (!op_ct.isWideDecimalType()) d = x.value; else d = datatypes::Decimal::getDoubleFromWideDecimal(x.s128Value); @@ -290,7 +290,7 @@ long double Func_truncate::getLongDoubleVal(Row& row, double d; - if (!datatypes::Decimal::isWideDecimalType(op_ct)) + if (!op_ct.isWideDecimalType()) d = x.value; else d = datatypes::Decimal::getDoubleFromWideDecimal(x.s128Value); @@ -334,7 +334,7 @@ IDB_Decimal Func_truncate::getDecimalVal(Row& row, int64_t d = 0; decimal = parm[0]->data()->getDecimalVal(row, isNull); - if (!datatypes::Decimal::isWideDecimalType(op_ct)) + if (!op_ct.isWideDecimalType()) { //@Bug 3101 - GCC 4.5.1 optimizes too aggressively here. Mark as volatile. volatile int64_t p = 1; @@ -739,12 +739,12 @@ string Func_truncate::getStrVal(Row& row, break; } - if (!datatypes::Decimal::isWideDecimalType(op_ct)) + if (!op_ct.isWideDecimalType()) return dataconvert::DataConvert::decimalToString(x.value, x.scale, op_ct.colDataType); else { - char buf[utils::MAXLENGTH16BYTES]; - dataconvert::DataConvert::decimalToString( &x.s128Value, x.scale, buf, utils::MAXLENGTH16BYTES, op_ct.colDataType); + char buf[datatypes::Decimal::MAXLENGTH16BYTES]; + dataconvert::DataConvert::decimalToString( &x.s128Value, x.scale, buf, (uint8_t) sizeof(buf), op_ct.colDataType); return string(buf); } } diff --git a/utils/joiner/tuplejoiner.cpp b/utils/joiner/tuplejoiner.cpp index ee405dd89..7c6793bf9 100644 --- a/utils/joiner/tuplejoiner.cpp +++ b/utils/joiner/tuplejoiner.cpp @@ -31,7 +31,6 @@ #include "spinlock.h" #include "vlarray.h" -#include "widedecimalutils.h" using namespace std; using namespace rowgroup; @@ -110,7 +109,7 @@ TupleJoiner::TupleJoiner( if (smallRG.isUnsigned(smallKeyColumns[0])) { - if (datatypes::Decimal::isWideDecimalType( + if (datatypes::isWideDecimalType( smallRG.getColType(smallKeyColumns[0]), smallRG.getColumnWidth(smallKeyColumns[0]))) { @@ -125,7 +124,7 @@ TupleJoiner::TupleJoiner( } else { - if (datatypes::Decimal::isWideDecimalType( + if (datatypes::isWideDecimalType( smallRG.getColType(smallKeyColumns[0]), smallRG.getColumnWidth(smallKeyColumns[0]))) { @@ -225,7 +224,7 @@ TupleJoiner::TupleJoiner( discreteValues[i] = false; if (isUnsigned(smallRG.getColTypes()[smallKeyColumns[i]])) { - if (datatypes::Decimal::isWideDecimalType( + if (datatypes::isWideDecimalType( smallRG.getColType(smallKeyColumns[i]), smallRG.getColumnWidth(smallKeyColumns[i]))) { @@ -240,7 +239,7 @@ TupleJoiner::TupleJoiner( } else { - if (datatypes::Decimal::isWideDecimalType( + if (datatypes::isWideDecimalType( smallRG.getColType(smallKeyColumns[i]), smallRG.getColumnWidth(smallKeyColumns[i]))) { @@ -802,7 +801,7 @@ void TupleJoiner::doneInserting() } } } - else if (datatypes::Decimal::isWideDecimalType( + else if (datatypes::isWideDecimalType( smallRow.getColType(smallKeyColumns[col]), smallRow.getColumnWidth(smallKeyColumns[col]))) { @@ -1165,7 +1164,7 @@ void TupleJoiner::updateCPData(const Row& r) } } } - else if (datatypes::Decimal::isWideDecimalType( + else if (datatypes::isWideDecimalType( r.getColType(colIdx), r.getColumnWidth(colIdx))) { @@ -1205,7 +1204,7 @@ void TupleJoiner::updateCPData(const Row& r) } } } - else if (datatypes::Decimal::isWideDecimalType( + else if (datatypes::isWideDecimalType( r.getColType(colIdx), r.getColumnWidth(colIdx))) { @@ -1745,7 +1744,7 @@ boost::shared_ptr TupleJoiner::copyForDiskJoin() ret->discreteValues[i] = false; if (isUnsigned(smallRG.getColTypes()[smallKeyColumns[i]])) { - if (datatypes::Decimal::isWideDecimalType( + if (datatypes::isWideDecimalType( smallRG.getColType(smallKeyColumns[i]), smallRG.getColumnWidth(smallKeyColumns[i]))) { @@ -1760,7 +1759,7 @@ boost::shared_ptr TupleJoiner::copyForDiskJoin() } else { - if (datatypes::Decimal::isWideDecimalType( + if (datatypes::isWideDecimalType( smallRG.getColType(smallKeyColumns[i]), smallRG.getColumnWidth(smallKeyColumns[i]))) { diff --git a/utils/regr/moda.cpp b/utils/regr/moda.cpp index 7bc5b427d..aa095ab53 100644 --- a/utils/regr/moda.cpp +++ b/utils/regr/moda.cpp @@ -158,7 +158,7 @@ mcsv1_UDAF::ReturnCode moda::init(mcsv1Context* context, return mcsv1_UDAF::ERROR; } - if (!(execplan::isNumeric(colTypes[0].dataType))) + if (!(datatypes::isNumeric(colTypes[0].dataType))) { // The error message will be prepended with // "The storage engine for the table doesn't support " diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index 04fe593ef..643b934b5 100755 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -53,7 +53,6 @@ #include "vlarray.h" #include "collation.h" -#include "widedecimalutils.h" //..comment out NDEBUG to enable assertions, uncomment NDEBUG to disable //#define NDEBUG @@ -2752,7 +2751,7 @@ void RowAggregationUM::calculateAvgColumns() uint32_t precision = fRow.getPrecision(colOut); bool isWideDecimal = - datatypes::Decimal::isWideDecimalType(precision); + datatypes::Decimal::isWideDecimalTypeByPrecision(precision); if (LIKELY(!isWideDecimal)) { @@ -3687,12 +3686,12 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData auto width = fRow.getColumnWidth(colOut); if (width == datatypes::MAXDECIMALWIDTH) { - ColTypeAlias colType; + execplan::CalpontSystemCatalog::TypeHolderStd colType; colType.colWidth = width; colType.precision = fRow.getPrecision(i); colType.scale = fRow.getScale(i); colType.colDataType = colDataType; - fRow.setInt128Field(Dec::int128FromString(aggData.fConstValue, colType), colOut); + fRow.setInt128Field(colType.decimal128FromString(aggData.fConstValue), colOut); } else if (width <= datatypes::MAXLEGACYWIDTH) { @@ -3812,13 +3811,12 @@ void RowAggregationUM::doNotNullConstantAggregate(const ConstantAggData& aggData auto width = fRow.getColumnWidth(colOut); if (width == datatypes::MAXDECIMALWIDTH) { - ColTypeAlias colType; + execplan::CalpontSystemCatalog::TypeHolderStd colType; colType.colWidth = width; colType.precision = fRow.getPrecision(i); colType.scale = fRow.getScale(i); colType.colDataType = colDataType; - int128_t constValue = Dec::int128FromString(aggData.fConstValue, - colType); + int128_t constValue = colType.decimal128FromString(aggData.fConstValue); int128_t sum; datatypes::MultiplicationOverflowCheck multOp; diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 731cf61e5..5b460e27d 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -44,7 +44,6 @@ using namespace execplan; #include "rowgroup.h" #include "dataconvert.h" #include "columnwidth.h" -#include "widedecimalutils.h" #include "collation.h" @@ -641,7 +640,7 @@ string Row::toString() const case CalpontSystemCatalog::UDECIMAL: if (colWidths[i] == datatypes::MAXDECIMALWIDTH) { - unsigned int buflen = utils::MAXLENGTH16BYTES; + unsigned int buflen = datatypes::Decimal::MAXLENGTH16BYTES; char *buf = (char*)alloca(buflen); // empty the buffer dataconvert::DataConvert::decimalToString(getBinaryField(i), @@ -851,7 +850,7 @@ void Row::initToNull() break; case 16 : - utils::setWideDecimalNullValue(reinterpret_cast(data[offsets[i]])); + datatypes::Decimal::setWideDecimalNullValue(reinterpret_cast(data[offsets[i]])); break; default: *((int64_t*) &data[offsets[i]]) = static_cast(joblist::BIGINTNULL); @@ -879,7 +878,7 @@ void Row::initToNull() break; case CalpontSystemCatalog::BINARY: { - utils::setWideDecimalNullValue(reinterpret_cast(data[offsets[i]])); + datatypes::Decimal::setWideDecimalNullValue(reinterpret_cast(data[offsets[i]])); } break; @@ -924,7 +923,7 @@ Row::isNullValue_offset( uint32_t offset) const { const int128_t *intPtr = reinterpret_cast(&data[offset]); - return utils::isWideDecimalNullValue (*intPtr); + return datatypes::Decimal::isWideDecimalNullValue (*intPtr); } template<> @@ -933,7 +932,7 @@ Row::isNullValue_offset( uint32_t offset) const { const int128_t *intPtr = reinterpret_cast(&data[offset]); - return utils::isWideDecimalNullValue (*intPtr); + return datatypes::Decimal::isWideDecimalNullValue (*intPtr); } template<> @@ -1203,7 +1202,7 @@ bool Row::equals(const Row& r2, const std::vector& keyCols) const if (getLongDoubleField(col) != r2.getLongDoubleField(col)) return false; } - else if (UNLIKELY(datatypes::Decimal::isWideDecimalType(columnType, colWidths[col]))) + else if (UNLIKELY(datatypes::isWideDecimalType(columnType, colWidths[col]))) { if (*getBinaryField(col) != *r2.getBinaryField(col)) return false; @@ -1262,7 +1261,7 @@ bool Row::equals(const Row& r2, uint32_t lastCol) const if (getLongDoubleField(col) != r2.getLongDoubleField(col)) return false; } - else if (UNLIKELY(datatypes::Decimal::isWideDecimalType(columnType, colWidths[col]))) + else if (UNLIKELY(datatypes::isWideDecimalType(columnType, colWidths[col]))) { if (*getBinaryField(col) != *r2.getBinaryField(col)) return false; @@ -1652,7 +1651,7 @@ void applyMapping(const int* mapping, const Row& in, Row* out) // WIP this doesn't look right b/c we can pushdown colType // Migrate to offset based methods here // code precision 2 width convertor - else if (UNLIKELY(datatypes::Decimal::isWideDecimalType(in.getColTypes()[i], + else if (UNLIKELY(datatypes::isWideDecimalType(in.getColTypes()[i], in.getColumnWidth(i)))) out->setBinaryField_offset(in.getBinaryField(i), 16, out->getOffset(mapping[i])); diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 9064aa8b8..94f72776a 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -653,12 +653,12 @@ inline uint32_t Row::getCharsetNumber(uint32_t col) const inline bool Row::isCharType(uint32_t colIndex) const { - return execplan::isCharType(types[colIndex]); + return datatypes::isCharType(types[colIndex]); } inline bool Row::isUnsigned(uint32_t colIndex) const { - return execplan::isUnsigned(types[colIndex]); + return datatypes::isUnsigned(types[colIndex]); } inline bool Row::isShortString(uint32_t colIndex) const @@ -1277,7 +1277,7 @@ inline void Row::copyField(Row& out, uint32_t destIndex, uint32_t srcIndex) cons { out.setLongDoubleField(getLongDoubleField(srcIndex), destIndex); } - else if (UNLIKELY(datatypes::Decimal::isWideDecimalType( + else if (UNLIKELY(datatypes::isWideDecimalType( types[srcIndex], colWidths[srcIndex]))) { copyBinaryField(out, destIndex, srcIndex); @@ -1724,12 +1724,12 @@ inline uint64_t RowGroup::getSizeWithStrings() const inline bool RowGroup::isCharType(uint32_t colIndex) const { - return execplan::isCharType(types[colIndex]); + return datatypes::isCharType(types[colIndex]); } inline bool RowGroup::isUnsigned(uint32_t colIndex) const { - return execplan::isUnsigned(types[colIndex]); + return datatypes::isUnsigned(types[colIndex]); } inline bool RowGroup::isShortString(uint32_t colIndex) const @@ -1931,7 +1931,7 @@ inline void copyRow(const Row& in, Row* out, uint32_t colCount) { out->setLongDoubleField(in.getLongDoubleField(i), i); } - else if (UNLIKELY(datatypes::Decimal::isWideDecimalType( + else if (UNLIKELY(datatypes::isWideDecimalType( in.getColType(i), in.getColumnWidth(i)))) { in.copyBinaryField(*out, i, i); diff --git a/utils/udfsdk/mcsv1_udaf.cpp b/utils/udfsdk/mcsv1_udaf.cpp index 2ad8905a8..dea99c125 100755 --- a/utils/udfsdk/mcsv1_udaf.cpp +++ b/utils/udfsdk/mcsv1_udaf.cpp @@ -151,7 +151,7 @@ const std::string mcsv1Context::toString() const std::ostringstream output; output << "mcsv1Context: " << getName() << std::endl; output << " RunFlags=" << fRunFlags << " ContextFlags=" << fContextFlags << std::endl; - output << " UserDataSize=" << fUserDataSize << " ResultType=" << colDataTypeToString(fResultType) << std::endl; + output << " UserDataSize=" << fUserDataSize << " ResultType=" << execplan::colDataTypeToString(fResultType) << std::endl; output << " Resultscale=" << fResultscale << " ResultPrecision=" << fResultPrecision << std::endl; output << " ErrorMsg=" << errorMsg << std::endl; output << " bInterrupted=" << bInterrupted << std::endl; diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index 17aa122a9..738c2bd4e 100644 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -793,7 +793,7 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos) break; case 16: - utils::setWideDecimalNullValue(int128Null); + datatypes::Decimal::setWideDecimalNullValue(int128Null); v = &int128Null; break; diff --git a/versioning/BRM/CMakeLists.txt b/versioning/BRM/CMakeLists.txt index 6f39c645d..db6b809b5 100644 --- a/versioning/BRM/CMakeLists.txt +++ b/versioning/BRM/CMakeLists.txt @@ -28,7 +28,8 @@ set(brm_LIB_SRCS transactionnode.cpp undoable.cpp vbbm.cpp - vss.cpp) + vss.cpp + ../../datatypes/mcs_datatype.cpp) add_library(brm SHARED ${brm_LIB_SRCS}) diff --git a/versioning/BRM/dbrm.cpp b/versioning/BRM/dbrm.cpp index bd27c3aec..ad0f96c9f 100644 --- a/versioning/BRM/dbrm.cpp +++ b/versioning/BRM/dbrm.cpp @@ -31,7 +31,6 @@ #include #include "dataconvert.h" -#include "widedecimalutils.h" #include "oamcache.h" #include "rwlock.h" #include "mastersegmenttable.h" @@ -4541,7 +4540,7 @@ void DBRM::invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN aInfo.isBinaryColumn = isBinaryColumn; if (!isBinaryColumn) { - if (execplan::isUnsigned(colType.colDataType)) + if (datatypes::isUnsigned(colType.colDataType)) { aInfo.max = 0; aInfo.min = numeric_limits::max(); @@ -4554,7 +4553,7 @@ void DBRM::invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN } else { - if (execplan::isUnsigned(colType.colDataType)) + if (datatypes::isUnsigned(colType.colDataType)) { aInfo.bigMax = 0; aInfo.bigMin = -1; diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index 42b452c4f..20fde35c9 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -55,7 +55,6 @@ namespace bi = boost::interprocess; #include "mastersegmenttable.h" #include "blocksize.h" #include "dataconvert.h" -#include "widedecimalutils.h" #include "mcs_decimal.h" #include "oamcache.h" #include "IDBDataFile.h" diff --git a/writeengine/bulk/we_brmreporter.cpp b/writeengine/bulk/we_brmreporter.cpp index f86490bec..229a8c333 100644 --- a/writeengine/bulk/we_brmreporter.cpp +++ b/writeengine/bulk/we_brmreporter.cpp @@ -34,7 +34,6 @@ #include "we_log.h" #include "cacheutils.h" #include "IDBPolicy.h" -#include "widedecimalutils.h" #include "mcs_decimal.h" #include "dataconvert.h" @@ -301,7 +300,7 @@ void BRMReporter::sendCPToFile( ) { if (fCPInfo.size() > 0) { - char buf[utils::MAXLENGTH16BYTES]; + char buf[datatypes::Decimal::MAXLENGTH16BYTES]; std::ostringstream oss; oss << "Writing " << fCPInfo.size() << " CP updates for table " << fTableName << " to report file " << fRptFileName; @@ -309,7 +308,7 @@ void BRMReporter::sendCPToFile( ) for (unsigned int i = 0; i < fCPInfo.size(); i++) { - if (!datatypes::Decimal::isWideDecimalType(fCPInfo[i].type, fCPInfo[i].colWidth)) + if (!datatypes::isWideDecimalType(fCPInfo[i].type, fCPInfo[i].colWidth)) { fRptFile << "CP: " << fCPInfo[i].startLbid << ' ' << fCPInfo[i].max << ' ' << @@ -322,10 +321,10 @@ void BRMReporter::sendCPToFile( ) { std::string bigMin, bigMax; - dataconvert::DataConvert::decimalToString(&fCPInfo[i].bigMin, 0, buf, utils::MAXLENGTH16BYTES, fCPInfo[i].type); + dataconvert::DataConvert::decimalToString(&fCPInfo[i].bigMin, 0, buf, (uint8_t) sizeof(buf), fCPInfo[i].type); bigMin = buf; - dataconvert::DataConvert::decimalToString(&fCPInfo[i].bigMax, 0, buf, utils::MAXLENGTH16BYTES, fCPInfo[i].type); + dataconvert::DataConvert::decimalToString(&fCPInfo[i].bigMax, 0, buf, (uint8_t) sizeof(buf), fCPInfo[i].type); bigMax = buf; fRptFile << "CP: " << fCPInfo[i].startLbid << ' ' << diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index 18e6bae00..250356156 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -40,7 +40,6 @@ #include "dataconvert.h" #include "exceptclasses.h" #include "mcs_decimal.h" -#include "widedecimalutils.h" #include "joblisttypes.h" diff --git a/writeengine/bulk/we_bulkloadbuffer.h b/writeengine/bulk/we_bulkloadbuffer.h index bbdb2378d..ffccc2aeb 100644 --- a/writeengine/bulk/we_bulkloadbuffer.h +++ b/writeengine/bulk/we_bulkloadbuffer.h @@ -31,7 +31,6 @@ #include "we_columninfo.h" #include "calpontsystemcatalog.h" #include "dataconvert.h" -#include "widedecimalutils.h" namespace WriteEngine { diff --git a/writeengine/bulk/we_colextinf.cpp b/writeengine/bulk/we_colextinf.cpp index f94433689..a4116df9b 100644 --- a/writeengine/bulk/we_colextinf.cpp +++ b/writeengine/bulk/we_colextinf.cpp @@ -27,7 +27,6 @@ #include "we_colextinf.h" #include "dataconvert.h" -#include "widedecimalutils.h" #include #include diff --git a/writeengine/bulk/we_colextinf.h b/writeengine/bulk/we_colextinf.h index e0e516616..58de88cc2 100644 --- a/writeengine/bulk/we_colextinf.h +++ b/writeengine/bulk/we_colextinf.h @@ -41,7 +41,6 @@ #include "brmtypes.h" #include "we_type.h" #include "dataconvert.h" -#include "widedecimalutils.h" namespace WriteEngine { diff --git a/writeengine/server/CMakeLists.txt b/writeengine/server/CMakeLists.txt index c4128df7e..1165dde47 100644 --- a/writeengine/server/CMakeLists.txt +++ b/writeengine/server/CMakeLists.txt @@ -15,7 +15,8 @@ set(WriteEngineServer_SRCS we_cleartablelockcmd.cpp we_cpifeederthread.cpp we_getfilesizes.cpp - ../../utils/common/crashtrace.cpp) + ../../utils/common/crashtrace.cpp + ../../datatypes/mcs_datatype.cpp) add_executable(WriteEngineServer ${WriteEngineServer_SRCS}) diff --git a/writeengine/server/we_ddlcommandproc.cpp b/writeengine/server/we_ddlcommandproc.cpp index 8356ffc22..d60ff033f 100644 --- a/writeengine/server/we_ddlcommandproc.cpp +++ b/writeengine/server/we_ddlcommandproc.cpp @@ -213,12 +213,12 @@ uint8_t WE_DDLCommandProc::writeSystable(ByteStream& bs, std::string& err) else if (INIT_COL == column.tableColName.column) { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } else if (NEXT_COL == column.tableColName.column) { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } else if (AUTOINC_COL == column.tableColName.column) { @@ -226,7 +226,7 @@ uint8_t WE_DDLCommandProc::writeSystable(ByteStream& bs, std::string& err) } else { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } colStruct.dataOid = column.oid; @@ -588,7 +588,7 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err else { tmpStr = ""; - //colTuple.data = getNullValueForType(column.colType); + //colTuple.data = column.colType.getNullValueForType(); } } @@ -629,16 +629,16 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err } else { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } } else if (LISTOBJID_COL == column.tableColName.column) { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } else if (TREEOBJID_COL == column.tableColName.column) { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } else if (MINVAL_COL == column.tableColName.column) { @@ -664,7 +664,7 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err } else { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } colStruct.dataOid = column.oid; @@ -982,7 +982,7 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string& err) else { tmpStr = ""; - //colTuple.data = getNullValueForType(column.colType); + //colTuple.data = column.colType.getNullValueForType(); } } @@ -1023,16 +1023,16 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string& err) } else { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } } else if (LISTOBJID_COL == column.tableColName.column) { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } else if (TREEOBJID_COL == column.tableColName.column) { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } else if (MINVAL_COL == column.tableColName.column) { @@ -1058,7 +1058,7 @@ uint8_t WE_DDLCommandProc::writeSyscolumn(ByteStream& bs, std::string& err) } else { - colTuple.data = getNullValueForType(column.colType); + colTuple.data = column.colType.getNullValueForType(); } colStruct.dataOid = column.oid; @@ -3646,7 +3646,7 @@ uint8_t WE_DDLCommandProc::fillNewColumn(ByteStream& bs, std::string& err) colType.scale = scale; colType.precision = precision; bool pushWarning = false; - defaultVal.data = DataConvert::convertColumnData(colType, defaultValStr, pushWarning, timeZone, isNULL, false, false); + defaultVal.data = colType.convertColumnData(defaultValStr, pushWarning, timeZone, isNULL, false, false); fWEWrapper.setTransId(txnID); fWEWrapper.setIsInsert(true); fWEWrapper.setBulkFlag(true); diff --git a/writeengine/server/we_ddlcommon.h b/writeengine/server/we_ddlcommon.h index d477880f1..b1741a440 100644 --- a/writeengine/server/we_ddlcommon.h +++ b/writeengine/server/we_ddlcommon.h @@ -143,239 +143,6 @@ inline void getColumnsForTable(uint32_t sessionID, std::string schema, std::str } -inline boost::any getNullValueForType(const execplan::CalpontSystemCatalog::ColType& colType) -{ - boost::any value; - - switch (colType.colDataType) - { - case execplan::CalpontSystemCatalog::BIT: - break; - - case execplan::CalpontSystemCatalog::TINYINT: - { - char tinyintvalue = joblist::TINYINTNULL; - value = tinyintvalue; - - } - break; - - case execplan::CalpontSystemCatalog::UTINYINT: - { - uint8_t tinyintvalue = joblist::UTINYINTNULL; - value = tinyintvalue; - - } - break; - - case execplan::CalpontSystemCatalog::SMALLINT: - { - short smallintvalue = joblist::SMALLINTNULL; - value = smallintvalue; - } - break; - - case execplan::CalpontSystemCatalog::USMALLINT: - { - uint16_t smallintvalue = joblist::USMALLINTNULL; - value = smallintvalue; - } - break; - - case execplan::CalpontSystemCatalog::MEDINT: - case execplan::CalpontSystemCatalog::INT: - { - int intvalue = joblist::INTNULL; - value = intvalue; - } - break; - - case execplan::CalpontSystemCatalog::UMEDINT: - case execplan::CalpontSystemCatalog::UINT: - { - uint32_t intvalue = joblist::UINTNULL; - value = intvalue; - } - break; - - case execplan::CalpontSystemCatalog::BIGINT: - { - long long bigint = joblist::BIGINTNULL; - value = bigint; - } - break; - - case execplan::CalpontSystemCatalog::UBIGINT: - { - uint64_t bigint = joblist::UBIGINTNULL; - value = bigint; - } - break; - - case execplan::CalpontSystemCatalog::DECIMAL: - case execplan::CalpontSystemCatalog::UDECIMAL: - { - if (colType.colWidth <= execplan::CalpontSystemCatalog::FOUR_BYTE) - { - short smallintvalue = joblist::SMALLINTNULL; - value = smallintvalue; - } - else if (colType.colWidth <= 9) - { - int intvalue = joblist::INTNULL; - value = intvalue; - } - else if (colType.colWidth <= 18) - { - long long eightbyte = joblist::BIGINTNULL; - value = eightbyte; - } - else - { - WriteEngine::Token nullToken; - value = nullToken; - } - } - break; - - case execplan::CalpontSystemCatalog::FLOAT: - case execplan::CalpontSystemCatalog::UFLOAT: - { - uint32_t jlfloatnull = joblist::FLOATNULL; - float* fp = reinterpret_cast(&jlfloatnull); - value = *fp; - } - break; - - case execplan::CalpontSystemCatalog::DOUBLE: - case execplan::CalpontSystemCatalog::UDOUBLE: - { - uint64_t jldoublenull = joblist::DOUBLENULL; - double* dp = reinterpret_cast(&jldoublenull); - value = *dp; - } - break; - - case execplan::CalpontSystemCatalog::DATE: - { - int d = joblist::DATENULL; - value = d; - } - break; - - case execplan::CalpontSystemCatalog::DATETIME: - { - long long d = joblist::DATETIMENULL; - value = d; - } - break; - - case execplan::CalpontSystemCatalog::TIME: - { - long long d = joblist::TIMENULL; - value = d; - } - break; - - case execplan::CalpontSystemCatalog::TIMESTAMP: - { - long long d = joblist::TIMESTAMPNULL; - value = d; - } - break; - - case execplan::CalpontSystemCatalog::CHAR: - { - std::string charnull; - - if (colType.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - { - //charnull = joblist::CHAR1NULL; - charnull = "\376"; - value = charnull; - } - else if (colType.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) - { - //charnull = joblist::CHAR2NULL; - charnull = "\377\376"; - value = charnull; - } - else if (colType.colWidth <= execplan::CalpontSystemCatalog::FOUR_BYTE) - { - //charnull = joblist::CHAR4NULL; - charnull = "\377\377\377\376"; - value = charnull; - } - else - { - WriteEngine::Token nullToken; - value = nullToken; - } - - } - break; - - case execplan::CalpontSystemCatalog::VARCHAR: - { - std::string charnull; - - if (colType.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - { - //charnull = joblist::CHAR2NULL; - charnull = "\377\376"; - value = charnull; - } - else if (colType.colWidth < execplan::CalpontSystemCatalog::FOUR_BYTE) - { - //charnull = joblist::CHAR4NULL; - charnull = "\377\377\377\376"; - value = charnull; - } - else - { - WriteEngine::Token nullToken; - value = nullToken; - } - - } - break; - - case execplan::CalpontSystemCatalog::BLOB: - case execplan::CalpontSystemCatalog::TEXT: - case execplan::CalpontSystemCatalog::VARBINARY: - { - std::string charnull; - - if (colType.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - { - //charnull = joblist::CHAR2NULL; - charnull = "\377\376"; - value = charnull; - } - else if (colType.colWidth < execplan::CalpontSystemCatalog::FOUR_BYTE) - { - //charnull = joblist::CHAR4NULL; - charnull = "\377\377\377\376"; - value = charnull; - } - else - { - WriteEngine::Token nullToken; - value = nullToken; - } - - } - break; - - - default: - throw std::runtime_error("getNullValueForType: unkown column data type"); - break; - - } - - return value; -} inline int convertDataType(int dataType) { diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 0a17d5128..f9c577dbd 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -370,7 +370,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: try { - datavalue = DataConvert::convertColumnData(colType, indata, pushWarning, insertPkg.get_TimeZone(), isNULL, false, false); + datavalue = colType.convertColumnData(indata, pushWarning, insertPkg.get_TimeZone(), isNULL, false, false); } catch (exception&) { @@ -1245,7 +1245,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std:: try { - datavalue = DataConvert::convertColumnData(colType, indata, pushWarning, insertPkg.get_TimeZone(), isNULL, false, false); + datavalue = colType.convertColumnData(indata, pushWarning, insertPkg.get_TimeZone(), isNULL, false, false); } catch (exception&) { @@ -2996,12 +2996,12 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, if (fetchColColwidths[fetchColPos] == datatypes::MAXDECIMALWIDTH) { int128_t* dec; - char buf[utils::MAXLENGTH16BYTES]; + char buf[datatypes::Decimal::MAXLENGTH16BYTES]; dec = row.getBinaryField(fetchColPos); dataconvert::DataConvert::decimalToString(dec, (unsigned)fetchColScales[fetchColPos], buf, - sizeof(buf), fetchColTypes[fetchColPos]); + (uint8_t) sizeof(buf), fetchColTypes[fetchColPos]); value.assign(buf); @@ -3362,12 +3362,12 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, { // WIP MCOL-641 int128_t* dec; - char buf[utils::MAXLENGTH16BYTES]; + char buf[datatypes::Decimal::MAXLENGTH16BYTES]; dec = row.getBinaryField(fetchColPos); dataconvert::DataConvert::decimalToString(dec, (unsigned)fetchColScales[fetchColPos], buf, - sizeof(buf), fetchColTypes[fetchColPos]); + (uint8_t) sizeof(buf), fetchColTypes[fetchColPos]); value = buf; @@ -3527,7 +3527,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, try { - datavalue = DataConvert::convertColumnData(colType, colType.defaultValue, pushWarn, timeZone, isNull, false, false); + datavalue = colType.convertColumnData(colType.defaultValue, pushWarn, timeZone, isNull, false, false); } catch (exception&) { @@ -3559,7 +3559,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, { try { - datavalue = DataConvert::convertColumnData(colType, value, pushWarn, timeZone, isNull, false, false); + datavalue = colType.convertColumnData(value, pushWarn, timeZone, isNull, false, false); } catch (exception&) { @@ -3656,7 +3656,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, try { - datavalue = DataConvert::convertColumnData(colType, inData, pushWarn, timeZone, isNull, false, false); + datavalue = colType.convertColumnData(inData, pushWarn, timeZone, isNull, false, false); } catch (exception&) { @@ -3702,7 +3702,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, { try { - datavalue = DataConvert::convertColumnData(colType, colType.defaultValue, pushWarn, timeZone, isNull, false, false); + datavalue = colType.convertColumnData(colType.defaultValue, pushWarn, timeZone, isNull, false, false); } catch (exception&) { @@ -3735,7 +3735,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, { try { - datavalue = DataConvert::convertColumnData(colType, inData, pushWarn, timeZone, isNull, false, true); + datavalue = colType.convertColumnData(inData, pushWarn, timeZone, isNull, false, true); } catch (exception& ex) { diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index bc9f0666e..3b14b8aae 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -40,7 +40,6 @@ using namespace std; using namespace execplan; #include "dataconvert.h" -#include "widedecimalutils.h" #include "IDBDataFile.h" #include "IDBPolicy.h" diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index da4c25bf4..2b5948a92 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -58,7 +58,6 @@ using namespace execplan; #include "MonitorProcMem.h" using namespace idbdatafile; #include "dataconvert.h" -#include "widedecimalutils.h" #ifdef _MSC_VER #define isnan _isnan @@ -3898,9 +3897,9 @@ void WriteEngineWrapper::printInputValue(const ColStructList& colStructList, else if (curTuple.data.type() == typeid(int128_t)) { // WIP replace with a single call - char buf[utils::MAXLENGTH16BYTES]; + char buf[datatypes::Decimal::MAXLENGTH16BYTES]; int128_t val = boost::any_cast(curTuple.data); - dataconvert::DataConvert::decimalToString(&val, 0, buf, utils::MAXLENGTH16BYTES, curColStruct.colDataType); + dataconvert::DataConvert::decimalToString(&val, 0, buf, (uint8_t) sizeof(buf), curColStruct.colDataType); curStr.assign(buf); } else if (curTuple.data.type() == typeid(double)) From 916950d1e93cb0319e180debed0ba85f6cf46451 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Tue, 3 Nov 2020 21:18:20 +0400 Subject: [PATCH 67/78] Fixing RelWithDebInfo warnings introduced in MCOL-4174 --- datatypes/mcs_datatype.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/datatypes/mcs_datatype.h b/datatypes/mcs_datatype.h index 1e161b7ae..358d69c04 100644 --- a/datatypes/mcs_datatype.h +++ b/datatypes/mcs_datatype.h @@ -393,8 +393,8 @@ public: :m_str(str), m_length(length) { } const char *str() const { return m_str; } - const size_t length() const { return m_length; } const char *end() const { return m_str + m_length; } + size_t length() const { return m_length; } void bin2hex(char *o) { static const char hexdig[] = { '0', '1', '2', '3', '4', '5', '6', '7', @@ -532,8 +532,8 @@ public: } void widenUInt64(const MinMaxInfo &partInfo) { - min = static_cast(partInfo.min < static_cast(min) ? partInfo.min : min); - max = static_cast(partInfo.max > static_cast(max) ? partInfo.max : max); + min = static_cast(static_cast(partInfo.min) < static_cast(min) ? partInfo.min : min); + max = static_cast(static_cast(partInfo.max) > static_cast(max) ? partInfo.max : max); } void widenSInt128(const MinMaxInfo &partInfo) { @@ -578,8 +578,8 @@ public: { }; MinMaxPartitionInfo(const BRM::EMEntry &entry); void set_invalid() { m_status|= CPINVALID; } - const bool is_invalid() const { return m_status & CPINVALID; } - const bool is_disabled() const { return m_status & ET_DISABLED; } + bool is_invalid() const { return m_status & CPINVALID; } + bool is_disabled() const { return m_status & ET_DISABLED; } bool isSuitableSInt64(const SimpleValue &startVal, round_style_t rfMin, @@ -703,6 +703,7 @@ public: class StoreField { public: + virtual ~StoreField() {} virtual int32_t colWidth() const = 0; virtual int32_t precision() const = 0; virtual int store_date(int64_t val) = 0; @@ -724,6 +725,7 @@ public: class WriteBatchField { public: + virtual ~WriteBatchField() { } virtual size_t ColWriteBatchDate(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; virtual size_t ColWriteBatchDatetime(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; virtual size_t ColWriteBatchTime(const unsigned char *buf, bool nullVal, ColBatchWriter &ci) = 0; From d5c6645ba1273599ef1508fe48e273445d4cd181 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Fri, 6 Nov 2020 18:05:50 +0400 Subject: [PATCH 68/78] Adding mcs_basic_types.h For now it consists of only: using int128_t = __int128; using uint128_t = unsigned __int128; All new privitive data types should go into this file in the future. --- datatypes/mcs_datatype.h | 4 -- datatypes/mcs_decimal.h | 3 +- dbcon/execplan/calpontsystemcatalog.h | 1 + dbcon/joblist/batchprimitiveprocessor-jl.cpp | 4 +- dbcon/joblist/batchprimitiveprocessor-jl.h | 3 +- dbcon/joblist/lbidlist.cpp | 50 ++++++++++---------- dbcon/joblist/lbidlist.h | 4 +- dbcon/joblist/primitivestep.h | 3 +- dbcon/joblist/tuple-bps.cpp | 20 ++++---- dbcon/mysql/ha_mcs_impl.cpp | 2 +- tests/arithmeticoperator-tests.cpp | 3 +- tests/comparators-tests.cpp | 1 - tests/rowgroup-tests.cpp | 2 - utils/common/any.hpp | 5 +- utils/common/hasher.h | 4 +- utils/common/mcs_basic_types.h | 28 +++++++++++ utils/common/widedecimalutils.h | 4 +- utils/dataconvert/dataconvert.h | 6 +-- utils/messageqcpp/bytestream.h | 3 +- utils/rowgroup/rowaggregation.cpp | 4 +- utils/rowgroup/rowgroup-tests.cpp | 2 - utils/rowgroup/rowgroup.h | 3 -- utils/udfsdk/mcsv1_udaf.cpp | 5 +- utils/windowfunction/idborderby.cpp | 3 -- utils/windowfunction/wf_udaf.cpp | 4 +- versioning/BRM/brmtypes.h | 17 +++---- versioning/BRM/dbrm.cpp | 4 +- versioning/BRM/extentmap.cpp | 30 ++++++------ versioning/BRM/extentmap.h | 6 +-- versioning/BRM/slavecomm.cpp | 2 +- writeengine/bulk/we_bulkloadbuffer.cpp | 2 +- writeengine/bulk/we_bulkloadbuffer.h | 4 +- writeengine/bulk/we_colextinf.cpp | 14 +++--- writeengine/bulk/we_colextinf.h | 18 +++---- writeengine/shared/we_type.h | 1 - writeengine/wrapper/we_colop.cpp | 2 - 36 files changed, 138 insertions(+), 133 deletions(-) create mode 100644 utils/common/mcs_basic_types.h diff --git a/datatypes/mcs_datatype.h b/datatypes/mcs_datatype.h index 358d69c04..abd8a21b9 100644 --- a/datatypes/mcs_datatype.h +++ b/datatypes/mcs_datatype.h @@ -32,10 +32,6 @@ typedef int32_t mcs_sint32_t; #endif -using int128_t = __int128; -using uint128_t = unsigned __int128; - - // Because including my_sys.h in a Columnstore header causes too many conflicts struct charset_info_st; typedef const struct charset_info_st CHARSET_INFO; diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index dcef8b947..b2fd8a225 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -21,11 +21,10 @@ #include #include #include +#include "mcs_basic_types.h" #include "exceptclasses.h" #include "widedecimalutils.h" -using int128_t = __int128; -using uint128_t = unsigned __int128; namespace datatypes { diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index 29113afc8..b40ca48aa 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -39,6 +39,7 @@ #include #include +#include "mcs_basic_types.h" #include "../../writeengine/shared/we_typeext.h" #include "columnresult.h" #include "bytestream.h" diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.cpp b/dbcon/joblist/batchprimitiveprocessor-jl.cpp index 929252d56..aff8f4e8b 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.cpp +++ b/dbcon/joblist/batchprimitiveprocessor-jl.cpp @@ -731,12 +731,12 @@ void BatchPrimitiveProcessorJL::deserializeAggregateResult(ByteStream* in, } void BatchPrimitiveProcessorJL::getRowGroupData(ByteStream& in, vector* out, - bool* validCPData, uint64_t* lbid, __int128* min, __int128* max, + bool* validCPData, uint64_t* lbid, int128_t* min, int128_t* max, uint32_t* cachedIO, uint32_t* physIO, uint32_t* touchedBlocks, bool* countThis, uint32_t threadID, bool* hasWideColumn, const execplan::CalpontSystemCatalog::ColType& colType) const { uint64_t tmp64; - unsigned __int128 tmp128; + uint128_t tmp128; uint8_t tmp8; RGData rgData; uint32_t rowCount; diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.h b/dbcon/joblist/batchprimitiveprocessor-jl.h index 0a8f1f841..fc535664d 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.h +++ b/dbcon/joblist/batchprimitiveprocessor-jl.h @@ -37,6 +37,7 @@ #include #include +#include "mcs_basic_types.h" #include "primitivemsg.h" #include "serializeable.h" #include "primitivestep.h" @@ -168,7 +169,7 @@ public: void deserializeAggregateResults(messageqcpp::ByteStream* in, std::vector* out) const; void getRowGroupData(messageqcpp::ByteStream& in, std::vector* out, - bool* validCPData, uint64_t* lbid, __int128* min, __int128* max, + bool* validCPData, uint64_t* lbid, int128_t* min, int128_t* max, uint32_t* cachedIO, uint32_t* physIO, uint32_t* touchedBlocks, bool* countThis, uint32_t threadID, bool* hasBinaryColumn, const execplan::CalpontSystemCatalog::ColType& colType) const; void deserializeAggregateResult(messageqcpp::ByteStream* in, diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index 4f2affc89..36725dc8e 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -236,7 +236,7 @@ bool LBIDList::GetMinMax(T& min, T& max, int64_t& seq, int64_t lbid, if (isUnsigned(colDataType)) { - if (typeid(T) == typeid(__int128)) + if (typeid(T) == typeid(int128_t)) { mmp->bigMax = 0; mmp->bigMin = -1; @@ -249,7 +249,7 @@ bool LBIDList::GetMinMax(T& min, T& max, int64_t& seq, int64_t lbid, } else { - if (typeid(T) == typeid(__int128)) + if (typeid(T) == typeid(int128_t)) { utils::int128Min(mmp->bigMax); utils::int128Max(mmp->bigMin); @@ -296,7 +296,7 @@ bool LBIDList::GetMinMax(T* min, T* max, int64_t* seq, if (isUnsigned(colDataType)) { - if (typeid(T) == typeid(__int128)) + if (typeid(T) == typeid(int128_t)) { mmp->bigMax = 0; mmp->bigMin = -1; @@ -309,7 +309,7 @@ bool LBIDList::GetMinMax(T* min, T* max, int64_t* seq, } else { - if (typeid(T) == typeid(__int128)) + if (typeid(T) == typeid(int128_t)) { utils::int128Min(mmp->bigMax); utils::int128Max(mmp->bigMin); @@ -327,7 +327,7 @@ bool LBIDList::GetMinMax(T* min, T* max, int64_t* seq, return false; } - if (typeid(T) == typeid(__int128)) + if (typeid(T) == typeid(int128_t)) { *min = entry.partition.cprange.bigLoVal; *max = entry.partition.cprange.bigHiVal; @@ -356,7 +356,7 @@ int LBIDList::getMinMaxFromEntries(T& min, T& max, int32_t& seq, if (lbid >= EMEntries[i].range.start && lbid <= lastLBID) { - if (typeid(T) == typeid(__int128)) + if (typeid(T) == typeid(int128_t)) { min = EMEntries[i].partition.cprange.bigLoVal; max = EMEntries[i].partition.cprange.bigHiVal; @@ -426,12 +426,12 @@ void LBIDList::UpdateMinMax(T min, T max, int64_t lbid, CalpontSystemCatalog::Co } else if (datatypes::isUnsigned(type)) { - if (typeid(T) == typeid(__int128)) + if (typeid(T) == typeid(int128_t)) { - if (static_cast(min) < static_cast(mmp->bigMin)) + if (static_cast(min) < static_cast(mmp->bigMin)) mmp->bigMin = min; - if (static_cast(max) > static_cast(mmp->bigMax)) + if (static_cast(max) > static_cast(mmp->bigMax)) mmp->bigMax = max; } else @@ -445,7 +445,7 @@ void LBIDList::UpdateMinMax(T min, T max, int64_t lbid, CalpontSystemCatalog::Co } else { - if (typeid(T) == typeid(__int128)) + if (typeid(T) == typeid(int128_t)) { if (min < mmp->bigMin) mmp->bigMin = min; @@ -686,7 +686,7 @@ bool LBIDList::checkSingleValue(T min, T max, T value, if (isCharType(type)) { // MCOL-641 LBIDList::CasualPartitionDataType() returns false if - // width > 8 for a character type, so T cannot be __int128 here + // width > 8 for a character type, so T cannot be int128_t here uint64_t mmin = order_swap(min); uint64_t mmax = order_swap(max); uint64_t vvalue = order_swap(value); @@ -694,10 +694,10 @@ bool LBIDList::checkSingleValue(T min, T max, T value, } else if (isUnsigned(type)) { - if (typeid(T) == typeid(__int128)) + if (typeid(T) == typeid(int128_t)) { - return (static_cast(value) >= static_cast(min) && - static_cast(value) <= static_cast(max)); + return (static_cast(value) >= static_cast(min) && + static_cast(value) <= static_cast(max)); } else { @@ -718,7 +718,7 @@ bool LBIDList::checkRangeOverlap(T min, T max, T tmin, T tmax, if (isCharType(type)) { // MCOL-641 LBIDList::CasualPartitionDataType() returns false if - // width > 8 for a character type, so T cannot be __int128 here + // width > 8 for a character type, so T cannot be int128_t here uint64_t min2 = order_swap(min); uint64_t max2 = order_swap(max); uint64_t tmin2 = order_swap(tmin); @@ -727,10 +727,10 @@ bool LBIDList::checkRangeOverlap(T min, T max, T tmin, T tmax, } else if (isUnsigned(type)) { - if (typeid(T) == typeid(__int128)) + if (typeid(T) == typeid(int128_t)) { - return (static_cast(tmin) <= static_cast(max) && - static_cast(tmax) >= static_cast(min)); + return (static_cast(tmin) <= static_cast(max) && + static_cast(tmax) >= static_cast(min)); } else { @@ -755,7 +755,7 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, const char* MsgDataPtr = (const char*) bs->buf(); bool scan = true; int64_t value = 0; - __int128 bigValue = 0; + int128_t bigValue = 0; bool bIsUnsigned = datatypes::isUnsigned(ct.colDataType); bool bIsChar = datatypes::isCharType(ct.colDataType); @@ -891,7 +891,7 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, } else { - scan = compareVal(static_cast(cpRange.bigLoVal), static_cast(cpRange.bigHiVal), static_cast(bigValue), op, lcf); + scan = compareVal(static_cast(cpRange.bigLoVal), static_cast(cpRange.bigHiVal), static_cast(bigValue), op, lcf); } } else @@ -959,7 +959,7 @@ void LBIDList::copyLbidList(const LBIDList& rhs) } template -bool LBIDList::GetMinMax<__int128>(__int128& min, __int128& max, int64_t& seq, int64_t lbid, +bool LBIDList::GetMinMax(int128_t& min, int128_t& max, int64_t& seq, int64_t lbid, const std::vector* pEMEntries, execplan::CalpontSystemCatalog::ColDataType colDataType); template @@ -968,7 +968,7 @@ bool LBIDList::GetMinMax(int64_t& min, int64_t& max, int64_t& seq, int6 execplan::CalpontSystemCatalog::ColDataType colDataType); template -bool LBIDList::GetMinMax<__int128>(__int128* min, __int128* max, int64_t* seq, +bool LBIDList::GetMinMax(int128_t* min, int128_t* max, int64_t* seq, int64_t lbid, const tr1::unordered_map& entries, execplan::CalpontSystemCatalog::ColDataType colDataType); @@ -978,7 +978,7 @@ bool LBIDList::GetMinMax(int64_t* min, int64_t* max, int64_t* seq, execplan::CalpontSystemCatalog::ColDataType colDataType); template -void LBIDList::UpdateMinMax<__int128>(__int128 min, __int128 max, int64_t lbid, +void LBIDList::UpdateMinMax(int128_t min, int128_t max, int64_t lbid, execplan::CalpontSystemCatalog::ColDataType type, bool validData = true); template @@ -986,7 +986,7 @@ void LBIDList::UpdateMinMax(int64_t min, int64_t max, int64_t lbid, execplan::CalpontSystemCatalog::ColDataType type, bool validData = true); template -bool LBIDList::checkSingleValue<__int128>(__int128 min, __int128 max, __int128 value, +bool LBIDList::checkSingleValue(int128_t min, int128_t max, int128_t value, execplan::CalpontSystemCatalog::ColDataType type); template @@ -994,7 +994,7 @@ bool LBIDList::checkSingleValue(int64_t min, int64_t max, int64_t value execplan::CalpontSystemCatalog::ColDataType type); template -bool LBIDList::checkRangeOverlap<__int128>(__int128 min, __int128 max, __int128 tmin, __int128 tmax, +bool LBIDList::checkRangeOverlap(int128_t min, int128_t max, int128_t tmin, int128_t tmax, execplan::CalpontSystemCatalog::ColDataType type); template diff --git a/dbcon/joblist/lbidlist.h b/dbcon/joblist/lbidlist.h index 934f42924..0f1925332 100644 --- a/dbcon/joblist/lbidlist.h +++ b/dbcon/joblist/lbidlist.h @@ -55,12 +55,12 @@ struct MinMaxPartition uint32_t blksScanned; union { - __int128 bigMin; + int128_t bigMin; int64_t min; }; union { - __int128 bigMax; + int128_t bigMax; int64_t max; }; }; diff --git a/dbcon/joblist/primitivestep.h b/dbcon/joblist/primitivestep.h index e57443ff3..d594685b1 100644 --- a/dbcon/joblist/primitivestep.h +++ b/dbcon/joblist/primitivestep.h @@ -46,6 +46,7 @@ #include #include +#include "mcs_basic_types.h" #include "calpontsystemcatalog.h" #include "calpontselectexecutionplan.h" #include "brm.h" @@ -1327,7 +1328,7 @@ public: * Note that it is an adder not a setter. For an extent to be scanned, all calls * must have a non-empty intersection. */ - void addCPPredicates(uint32_t OID, const std::vector<__int128>& vals, bool isRange, + void addCPPredicates(uint32_t OID, const std::vector& vals, bool isRange, bool isSmallSideWideDecimal); /* semijoin adds */ diff --git a/dbcon/joblist/tuple-bps.cpp b/dbcon/joblist/tuple-bps.cpp index 44162ba6e..ceee3e8f0 100644 --- a/dbcon/joblist/tuple-bps.cpp +++ b/dbcon/joblist/tuple-bps.cpp @@ -927,7 +927,7 @@ void TupleBPS::prepCasualPartitioning() { uint32_t i; int64_t min, max, seq; - __int128 bigMin, bigMax; + int128_t bigMin, bigMax; boost::mutex::scoped_lock lk(cpMutex); for (i = 0; i < scannedExtents.size(); i++) @@ -1889,15 +1889,15 @@ abort: struct _CPInfo { _CPInfo(int64_t MIN, int64_t MAX, uint64_t l, bool val) : min(MIN), max(MAX), LBID(l), valid(val) { }; - _CPInfo(__int128 BIGMIN, __int128 BIGMAX, uint64_t l, bool val) : bigMin(BIGMIN), bigMax(BIGMAX), LBID(l), valid(val) { }; + _CPInfo(int128_t BIGMIN, int128_t BIGMAX, uint64_t l, bool val) : bigMin(BIGMIN), bigMax(BIGMAX), LBID(l), valid(val) { }; union { - __int128 bigMin; + int128_t bigMin; int64_t min; }; union { - __int128 bigMax; + int128_t bigMax; int64_t max; }; uint64_t LBID; @@ -1918,8 +1918,8 @@ void TupleBPS::receiveMultiPrimitiveMessages(uint32_t threadID) bool validCPData; bool hasBinaryColumn; - __int128 min; - __int128 max; + int128_t min; + int128_t max; uint64_t lbid; vector<_CPInfo> cpv; uint32_t cachedIO; @@ -3212,7 +3212,7 @@ void TupleBPS::setJoinFERG(const RowGroup& rg) fBPP->setJoinFERG(rg); } -void TupleBPS::addCPPredicates(uint32_t OID, const vector<__int128>& vals, bool isRange, +void TupleBPS::addCPPredicates(uint32_t OID, const vector& vals, bool isRange, bool isSmallSideWideDecimal) { @@ -3221,7 +3221,7 @@ void TupleBPS::addCPPredicates(uint32_t OID, const vector<__int128>& vals, bool uint32_t i, j, k; int64_t min, max, seq; - __int128 bigMin, bigMax; + int128_t bigMin, bigMax; bool isValid, intersection; vector colCmdVec = fBPP->getFilterSteps(); ColumnCommandJL* cmd; @@ -3292,7 +3292,7 @@ void TupleBPS::addCPPredicates(uint32_t OID, const vector<__int128>& vals, bool } else { - runtimeCPFlags[j] = ll.checkRangeOverlap((__int128) min, (__int128) max, vals[0], vals[1], + runtimeCPFlags[j] = ll.checkRangeOverlap((int128_t) min, (int128_t) max, vals[0], vals[1], colType.colDataType) && runtimeCPFlags[j]; } } @@ -3310,7 +3310,7 @@ void TupleBPS::addCPPredicates(uint32_t OID, const vector<__int128>& vals, bool else { intersection = intersection || - ll.checkSingleValue((__int128) min, (__int128) max, vals[k], colType.colDataType); + ll.checkSingleValue((int128_t) min, (int128_t) max, vals[k], colType.colDataType); } } diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index e94f19595..d1868f52d 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -56,6 +56,7 @@ using namespace std; #include #include +#include "mcs_basic_types.h" #include "idb_mysql.h" #define NEED_CALPONT_INTERFACE @@ -145,7 +146,6 @@ namespace cal_impl_if extern bool nonConstFunc(Item_func* ifp); } -using int128_t = __int128; namespace { diff --git a/tests/arithmeticoperator-tests.cpp b/tests/arithmeticoperator-tests.cpp index dc192fbd1..5f883eb9d 100644 --- a/tests/arithmeticoperator-tests.cpp +++ b/tests/arithmeticoperator-tests.cpp @@ -16,9 +16,8 @@ MA 02110-1301, USA. */ #include // googletest header file +#include "mcs_basic_types.h" -using int128_t = __int128; -using uint128_t = unsigned __int128; //using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; TEST(SimpleCheck, check1) diff --git a/tests/comparators-tests.cpp b/tests/comparators-tests.cpp index 4d6f8aab2..4b754dacf 100644 --- a/tests/comparators-tests.cpp +++ b/tests/comparators-tests.cpp @@ -42,7 +42,6 @@ #define DEBUG #define MEMORY_LIMIT 14983602176 -using int128_t = __int128; using namespace std; using namespace joblist; using namespace messageqcpp; diff --git a/tests/rowgroup-tests.cpp b/tests/rowgroup-tests.cpp index 7232d4898..cc031606b 100644 --- a/tests/rowgroup-tests.cpp +++ b/tests/rowgroup-tests.cpp @@ -26,8 +26,6 @@ #define WIDE_DEC_PRECISION 38U #define INITIAL_ROW_OFFSET 2 -using int128_t = __int128; -using uint128_t = unsigned __int128; using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; class RowDecimalTest : public ::testing::Test diff --git a/utils/common/any.hpp b/utils/common/any.hpp index 7ae5d553e..f39d5cec2 100755 --- a/utils/common/any.hpp +++ b/utils/common/any.hpp @@ -12,6 +12,7 @@ #include #include #include +#include "mcs_basic_types.h" namespace static_any { @@ -127,8 +128,8 @@ namespace anyimpl #define BIG_POLICY(TYPE) template<> struct \ choose_policy { typedef big_any_policy type; }; - BIG_POLICY(__int128); - BIG_POLICY(unsigned __int128); + BIG_POLICY(int128_t); + BIG_POLICY(uint128_t); /// Specializations for small types. #define SMALL_POLICY(TYPE) template<> struct \ diff --git a/utils/common/hasher.h b/utils/common/hasher.h index 08b6b6ac1..1633986a2 100644 --- a/utils/common/hasher.h +++ b/utils/common/hasher.h @@ -29,9 +29,7 @@ #include #include - -using int128_t = __int128; -using uint128_t = unsigned __int128; +#include "mcs_basic_types.h" namespace utils { diff --git a/utils/common/mcs_basic_types.h b/utils/common/mcs_basic_types.h new file mode 100644 index 000000000..131fcdc68 --- /dev/null +++ b/utils/common/mcs_basic_types.h @@ -0,0 +1,28 @@ +/* + Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. +*/ +#ifndef MCS_BASIC_TYPES_H_INCLUDED +#define MCS_BASIC_TYPES_H_INCLUDED + + +using int128_t = __int128; +using uint128_t = unsigned __int128; + + +#endif // MCS_BASIC_TYPES_H_INCLUDED +// vim:ts=2 sw=2: diff --git a/utils/common/widedecimalutils.h b/utils/common/widedecimalutils.h index b6f1ad5e9..7e931d7fe 100644 --- a/utils/common/widedecimalutils.h +++ b/utils/common/widedecimalutils.h @@ -19,9 +19,7 @@ #define WIDE_DECIMAL_UTILS_H #include - -using int128_t = __int128; -using uint128_t = unsigned __int128; +#include "mcs_basic_types.h" namespace utils { diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 4b1dfa96b..0b918fb8b 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -160,10 +160,6 @@ const int32_t MIN_TIMESTAMP_VALUE = 0; namespace dataconvert { -// WIP MCOL-641 -using int128_t = __int128; -using uint128_t = unsigned __int128; - enum CalpontDateTimeFormat { CALPONTDATE_ENUM = 1, // date format is: "YYYY-MM-DD" @@ -1474,7 +1470,7 @@ inline bool greaterThan128(int128_t a, int128_t b) return a > b; } -// Naive __int128 version of strtoll +// Naive int128_t version of strtoll inline int128_t strtoll128(const char* data, bool& saturate, char** ep) { int128_t res = 0; diff --git a/utils/messageqcpp/bytestream.h b/utils/messageqcpp/bytestream.h index ade220d76..970ff9423 100644 --- a/utils/messageqcpp/bytestream.h +++ b/utils/messageqcpp/bytestream.h @@ -33,6 +33,7 @@ #include #include +#include "mcs_basic_types.h" #include "exceptclasses.h" #include "serializeable.h" #include "any.hpp" @@ -45,8 +46,6 @@ class ByteStreamTestSuite; #define EXPORT #endif -using int128_t = __int128; -using uint128_t = unsigned __int128; namespace messageqcpp { diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index 643b934b5..e9199f8fd 100755 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -238,13 +238,13 @@ const static_any::any& RowAggregation::shortTypeId((short)1); const static_any::any& RowAggregation::intTypeId((int)1); const static_any::any& RowAggregation::longTypeId((long)1); const static_any::any& RowAggregation::llTypeId((long long)1); -const static_any::any& RowAggregation::int128TypeId((__int128)1); +const static_any::any& RowAggregation::int128TypeId((int128_t)1); const static_any::any& RowAggregation::ucharTypeId((unsigned char)1); const static_any::any& RowAggregation::ushortTypeId((unsigned short)1); const static_any::any& RowAggregation::uintTypeId((unsigned int)1); const static_any::any& RowAggregation::ulongTypeId((unsigned long)1); const static_any::any& RowAggregation::ullTypeId((unsigned long long)1); -const static_any::any& RowAggregation::uint128TypeId((unsigned __int128)1); +const static_any::any& RowAggregation::uint128TypeId((uint128_t)1); const static_any::any& RowAggregation::floatTypeId((float)1); const static_any::any& RowAggregation::doubleTypeId((double)1); const static_any::any& RowAggregation::longdoubleTypeId((long double)1); diff --git a/utils/rowgroup/rowgroup-tests.cpp b/utils/rowgroup/rowgroup-tests.cpp index 8fbbe22fc..3ebfb6c0b 100644 --- a/utils/rowgroup/rowgroup-tests.cpp +++ b/utils/rowgroup/rowgroup-tests.cpp @@ -26,8 +26,6 @@ #define WIDE_DEC_PRECISION 38U #define INITIAL_ROW_OFFSET 2 -using int128_t = __int128; -using uint128_t = unsigned __int128; using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; class RowDecimalTest : public ::testing::Test { diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 94f72776a..2d1ba045c 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -65,9 +65,6 @@ typedef const struct charset_info_st CHARSET_INFO; // Workaround for my_global.h #define of isnan(X) causing a std::std namespace -using int128_t = __int128; -using uint128_t = unsigned __int128; - namespace rowgroup { diff --git a/utils/udfsdk/mcsv1_udaf.cpp b/utils/udfsdk/mcsv1_udaf.cpp index dea99c125..cd08e3573 100755 --- a/utils/udfsdk/mcsv1_udaf.cpp +++ b/utils/udfsdk/mcsv1_udaf.cpp @@ -18,6 +18,7 @@ #include #include #include +#include "mcs_basic_types.h" #include "mcsv1_udaf.h" #include "bytestream.h" #include "objectreader.h" @@ -278,13 +279,13 @@ const static_any::any& mcsv1_UDAF::shortTypeId((short)1); const static_any::any& mcsv1_UDAF::intTypeId((int)1); const static_any::any& mcsv1_UDAF::longTypeId((long)1); const static_any::any& mcsv1_UDAF::llTypeId((long long)1); -const static_any::any& mcsv1_UDAF::int128TypeId((__int128)1); +const static_any::any& mcsv1_UDAF::int128TypeId((int128_t)1); const static_any::any& mcsv1_UDAF::ucharTypeId((unsigned char)1); const static_any::any& mcsv1_UDAF::ushortTypeId((unsigned short)1); const static_any::any& mcsv1_UDAF::uintTypeId((unsigned int)1); const static_any::any& mcsv1_UDAF::ulongTypeId((unsigned long)1); const static_any::any& mcsv1_UDAF::ullTypeId((unsigned long long)1); -const static_any::any& mcsv1_UDAF::uint128TypeId((unsigned __int128)1); +const static_any::any& mcsv1_UDAF::uint128TypeId((uint128_t)1); const static_any::any& mcsv1_UDAF::floatTypeId((float)1); const static_any::any& mcsv1_UDAF::doubleTypeId((double)1); const static_any::any& mcsv1_UDAF::strTypeId(typeStr); diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index 1028d3e1d..69344c43e 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -48,9 +48,6 @@ using namespace rowgroup; #include "joblisttypes.h" #include "mcs_decimal.h" - -using int128_t = __int128; - #include "collation.h" // See agg_arg_charsets in sql_type.h to see conversion rules for diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index 75075b778..62d4a2714 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -544,13 +544,13 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, static const static_any::any& intTypeId = (int)1; static const static_any::any& longTypeId = (long)1; static const static_any::any& llTypeId = (long long)1; - static const static_any::any& int128TypeId = (__int128)1; + static const static_any::any& int128TypeId = (int128_t)1; static const static_any::any& ucharTypeId = (unsigned char)1; static const static_any::any& ushortTypeId = (unsigned short)1; static const static_any::any& uintTypeId = (unsigned int)1; static const static_any::any& ulongTypeId = (unsigned long)1; static const static_any::any& ullTypeId = (unsigned long long)1; - static const static_any::any& uint128TypeId = (unsigned __int128)1; + static const static_any::any& uint128TypeId = (uint128_t)1; static const static_any::any& floatTypeId = (float)1; static const static_any::any& doubleTypeId = (double)1; static const std::string typeStr(""); diff --git a/versioning/BRM/brmtypes.h b/versioning/BRM/brmtypes.h index 373b1a2eb..623720263 100644 --- a/versioning/BRM/brmtypes.h +++ b/versioning/BRM/brmtypes.h @@ -31,6 +31,7 @@ #include #include #include +#include "mcs_basic_types.h" #include "logicalpartition.h" #ifndef _MSC_VER @@ -153,12 +154,12 @@ struct CPInfo int32_t seqNum; union { - __int128 bigMax; + int128_t bigMax; int64_t max_; }; union { - __int128 bigMin; + int128_t bigMin; int64_t min_; }; bool isBinaryColumn; @@ -173,12 +174,12 @@ struct CPMaxMin int32_t seqNum; union { - __int128 bigMax; + int128_t bigMax; int64_t max_; }; union { - __int128 bigMin; + int128_t bigMin; int64_t min_; }; bool isBinaryColumn; @@ -198,12 +199,12 @@ struct CPInfoMerge bool newExtent; // is this to be treated as a new extent union { - __int128 bigMax; + int128_t bigMax; int64_t max_; }; union { - __int128 bigMin; + int128_t bigMin; int64_t min_; }; }; @@ -221,12 +222,12 @@ struct CPMaxMinMerge bool newExtent; union { - __int128 bigMax; + int128_t bigMax; int64_t max_; }; union { - __int128 bigMin; + int128_t bigMin; int64_t min_; }; }; diff --git a/versioning/BRM/dbrm.cpp b/versioning/BRM/dbrm.cpp index ad0f96c9f..7c8b3c860 100644 --- a/versioning/BRM/dbrm.cpp +++ b/versioning/BRM/dbrm.cpp @@ -560,7 +560,7 @@ int DBRM::setExtentsMaxMin(const CPInfoList_t& cpInfos) DBRM_THROW { if (it->isBinaryColumn) { - command << (uint8_t)1 << (uint64_t)it->firstLbid << (unsigned __int128)it->bigMax << (unsigned __int128)it->bigMin << (uint32_t)it->seqNum; + command << (uint8_t)1 << (uint64_t)it->firstLbid << (uint128_t)it->bigMax << (uint128_t)it->bigMin << (uint32_t)it->seqNum; } else { @@ -4583,7 +4583,7 @@ void DBRM::invalidateUncommittedExtentLBIDs(execplan::CalpontSystemCatalog::SCN } template -int DBRM::getExtentMaxMin<__int128>(const LBID_t lbid, __int128& max, __int128& min, int32_t& seqNum) throw(); +int DBRM::getExtentMaxMin(const LBID_t lbid, int128_t& max, int128_t& min, int32_t& seqNum) throw(); template int DBRM::getExtentMaxMin(const LBID_t lbid, int64_t& max, int64_t& min, int32_t& seqNum) throw(); diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index 20fde35c9..bb48844b8 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -132,7 +132,7 @@ EMCasualPartition_struct::EMCasualPartition_struct(const int64_t lo, const int64 isValid = CP_INVALID; } -EMCasualPartition_struct::EMCasualPartition_struct(const __int128 bigLo, const __int128 bigHi, const int32_t seqNum) +EMCasualPartition_struct::EMCasualPartition_struct(const int128_t bigLo, const int128_t bigHi, const int32_t seqNum) { bigLoVal = bigLo; bigHiVal = bigHi; @@ -890,15 +890,15 @@ void ExtentMap::mergeExtentsMaxMin(CPMaxMinMergeMap_t& cpMap, bool useLock) } else { - if (static_cast(it->second.bigMin) < - static_cast(fExtentMap[i].partition.cprange.bigLoVal)) + if (static_cast(it->second.bigMin) < + static_cast(fExtentMap[i].partition.cprange.bigLoVal)) { fExtentMap[i].partition.cprange.bigLoVal = it->second.bigMin; } - if (static_cast(it->second.bigMax) > - static_cast(fExtentMap[i].partition.cprange.bigHiVal)) + if (static_cast(it->second.bigMax) > + static_cast(fExtentMap[i].partition.cprange.bigHiVal)) { fExtentMap[i].partition.cprange.bigHiVal = it->second.bigMax; @@ -1033,7 +1033,7 @@ bool ExtentMap::isValidCPRange(const T& max, const T& min, execplan::CalpontSyst { if (isUnsigned(type)) { - if (typeid(T) != typeid(__int128)) + if (typeid(T) != typeid(int128_t)) { if ( (static_cast(min) >= (numeric_limits::max() - 1)) || (static_cast(max) >= (numeric_limits::max() - 1)) ) @@ -1043,11 +1043,11 @@ bool ExtentMap::isValidCPRange(const T& max, const T& min, execplan::CalpontSyst } else { - unsigned __int128 temp; + uint128_t temp; utils::uint128Max(temp); - if ( (static_cast(min) >= (temp - 1)) || - (static_cast(max) >= (temp - 1)) ) + if ( (static_cast(min) >= (temp - 1)) || + (static_cast(max) >= (temp - 1)) ) { return false; } @@ -1055,7 +1055,7 @@ bool ExtentMap::isValidCPRange(const T& max, const T& min, execplan::CalpontSyst } else { - if (typeid(T) != typeid(__int128)) + if (typeid(T) != typeid(int128_t)) { if ( (min <= (numeric_limits::min() + 1)) || (max <= (numeric_limits::min() + 1)) ) @@ -1065,7 +1065,7 @@ bool ExtentMap::isValidCPRange(const T& max, const T& min, execplan::CalpontSyst } else { - __int128 temp; + int128_t temp; utils::int128Min(temp); if ( (min <= (temp + 1)) || @@ -1105,9 +1105,9 @@ int ExtentMap::getMaxMin(const LBID_t lbid, } #endif - if (typeid(T) == typeid(__int128)) + if (typeid(T) == typeid(int128_t)) { - __int128 tmpMax, tmpMin; + int128_t tmpMax, tmpMin; utils::int128Min(tmpMax); utils::int128Max(tmpMin); max = tmpMax; @@ -1143,7 +1143,7 @@ int ExtentMap::getMaxMin(const LBID_t lbid, if (lbid >= fExtentMap[i].range.start && lbid <= lastBlock) { - if (typeid(T) == typeid(__int128)) + if (typeid(T) == typeid(int128_t)) { max = fExtentMap[i].partition.cprange.bigHiVal; min = fExtentMap[i].partition.cprange.bigLoVal; @@ -6059,7 +6059,7 @@ void ExtentMap::dumpTo(ostream& os) */ template -int ExtentMap::getMaxMin<__int128>(const LBID_t lbidRange, __int128& max, __int128& min, int32_t& seqNum); +int ExtentMap::getMaxMin(const LBID_t lbidRange, int128_t& max, int128_t& min, int32_t& seqNum); template int ExtentMap::getMaxMin(const LBID_t lbidRange, int64_t& max, int64_t& min, int32_t& seqNum); diff --git a/versioning/BRM/extentmap.h b/versioning/BRM/extentmap.h index 7a5425935..8f16c3cee 100644 --- a/versioning/BRM/extentmap.h +++ b/versioning/BRM/extentmap.h @@ -137,17 +137,17 @@ struct EMCasualPartition_struct char isValid; //CP_INVALID - No min/max and no DML in progress. CP_UPDATING - Update in progress. CP_VALID- min/max is valid union { - __int128 bigLoVal; // These need to be reinterpreted as unsigned for uint64_t/uint128_t column types. + int128_t bigLoVal; // These need to be reinterpreted as unsigned for uint64_t/uint128_t column types. int64_t loVal; }; union { - __int128 bigHiVal; + int128_t bigHiVal; int64_t hiVal; }; EXPORT EMCasualPartition_struct(); EXPORT EMCasualPartition_struct(const int64_t lo, const int64_t hi, const int32_t seqNum); - EXPORT EMCasualPartition_struct(const __int128 bigLo, const __int128 bigHi, const int32_t seqNum); + EXPORT EMCasualPartition_struct(const int128_t bigLo, const int128_t bigHi, const int32_t seqNum); EXPORT EMCasualPartition_struct(const EMCasualPartition_struct& em); EXPORT EMCasualPartition_struct& operator= (const EMCasualPartition_struct& em); }; diff --git a/versioning/BRM/slavecomm.cpp b/versioning/BRM/slavecomm.cpp index 56b9d5d49..f63ffcadf 100644 --- a/versioning/BRM/slavecomm.cpp +++ b/versioning/BRM/slavecomm.cpp @@ -1330,7 +1330,7 @@ void SlaveComm::do_setExtentsMaxMin(ByteStream& msg) uint64_t tmp64; uint32_t tmp32; uint8_t tmp8; - unsigned __int128 tmp128; + uint128_t tmp128; int err; ByteStream reply; int32_t updateCount; diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index 250356156..e0fadf1f8 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -302,7 +302,7 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, int32_t iDate; char charTmpBuf[MAX_COLUMN_BOUNDARY + 1] = {0}; long long llVal = 0, llDate = 0; - __int128 bigllVal = 0; + int128_t bigllVal = 0; uint64_t tmp64; uint32_t tmp32; uint8_t ubiVal; diff --git a/writeengine/bulk/we_bulkloadbuffer.h b/writeengine/bulk/we_bulkloadbuffer.h index ffccc2aeb..e953c1b82 100644 --- a/writeengine/bulk/we_bulkloadbuffer.h +++ b/writeengine/bulk/we_bulkloadbuffer.h @@ -45,12 +45,12 @@ public: int64_t satCount; union { - __int128 bigMinBufferVal; + int128_t bigMinBufferVal; int64_t minBufferVal_; }; union { - __int128 bigMaxBufferVal; + int128_t bigMaxBufferVal; int64_t maxBufferVal_; }; BLBufferStats(ColDataType colDataType) : satCount(0) diff --git a/writeengine/bulk/we_colextinf.cpp b/writeengine/bulk/we_colextinf.cpp index a4116df9b..c65158292 100644 --- a/writeengine/bulk/we_colextinf.cpp +++ b/writeengine/bulk/we_colextinf.cpp @@ -92,7 +92,7 @@ void ColExtInf::addOrUpdateEntryTemplate( RID lastInputRow, // If all rows had null value for this column, then minVal will be // MAX_INT and maxVal will be MIN_INT (see getCPInfoForBRM()). - __int128 bigMinValInit; + int128_t bigMinValInit; utils::int128Max(bigMinValInit); if ((iter->second.fMinVal == LLONG_MIN && width <= 8) || (iter->second.fbigMinVal == bigMinValInit && width > 8)) // init the range @@ -124,12 +124,12 @@ void ColExtInf::addOrUpdateEntryTemplate( RID lastInputRow, } else { - if (static_cast(minVal) - < static_cast(iter->second.fbigMinVal)) + if (static_cast(minVal) + < static_cast(iter->second.fbigMinVal)) iter->second.fbigMinVal = minVal; - if (static_cast(maxVal) - > static_cast(iter->second.fbigMaxVal)) + if (static_cast(maxVal) + > static_cast(iter->second.fbigMaxVal)) iter->second.fbigMaxVal = maxVal; } } @@ -214,8 +214,8 @@ void ColExtInf::getCPInfoForBRM( JobColumn column, BRMReporter& brmReporter ) // if applicable (indicating an extent with no non-NULL values). int64_t minVal = iter->second.fMinVal; int64_t maxVal = iter->second.fMaxVal; - __int128 bigMinVal = iter->second.fbigMinVal; - __int128 bigMaxVal = iter->second.fbigMaxVal; + int128_t bigMinVal = iter->second.fbigMinVal; + int128_t bigMaxVal = iter->second.fbigMaxVal; if ( bIsChar ) { diff --git a/writeengine/bulk/we_colextinf.h b/writeengine/bulk/we_colextinf.h index 58de88cc2..60930d8ed 100644 --- a/writeengine/bulk/we_colextinf.h +++ b/writeengine/bulk/we_colextinf.h @@ -88,7 +88,7 @@ public: fNewExtent(true) { } // Used to create entry for a new extent, with LBID not yet allocated - ColExtInfEntry(__int128 bigMinVal, __int128 bigMaxVal) : + ColExtInfEntry(int128_t bigMinVal, int128_t bigMaxVal) : fLbid(INVALID_LBID), fNewExtent(true), fbigMinVal(bigMinVal), @@ -102,11 +102,11 @@ public: fNewExtent(true) { } // Used to create entry for a new extent, with LBID not yet allocated - ColExtInfEntry(unsigned __int128 bigMinVal, unsigned __int128 bigMaxVal) : + ColExtInfEntry(uint128_t bigMinVal, uint128_t bigMaxVal) : fLbid(INVALID_LBID), fNewExtent(true), - fbigMinVal(static_cast<__int128>(bigMinVal)), - fbigMaxVal(static_cast<__int128>(bigMaxVal)) { } + fbigMinVal(static_cast(bigMinVal)), + fbigMaxVal(static_cast(bigMaxVal)) { } BRM::LBID_t fLbid; // LBID for an extent; should be the starting LBID int64_t fMinVal; // minimum value for extent associated with LBID @@ -114,12 +114,12 @@ public: bool fNewExtent;// is this a new extent union { - __int128 fbigMinVal; + int128_t fbigMinVal; int64_t fMinVal_; }; union { - __int128 fbigMaxVal; + int128_t fbigMaxVal; int64_t fMaxVal_; }; }; @@ -159,8 +159,8 @@ public: int width ) { } virtual void addOrUpdateEntry( RID lastInputRow, - __int128 minVal, - __int128 maxVal, + int128_t minVal, + int128_t maxVal, ColDataType colDataType, int width ) { } @@ -231,7 +231,7 @@ public: } virtual void addOrUpdateEntry( RID lastInputRow, - __int128 minVal, __int128 maxVal, + int128_t minVal, int128_t maxVal, ColDataType colDataType, int width ) { diff --git a/writeengine/shared/we_type.h b/writeengine/shared/we_type.h index ba77b9409..824058d7f 100644 --- a/writeengine/shared/we_type.h +++ b/writeengine/shared/we_type.h @@ -57,7 +57,6 @@ typedef uint32_t FID; /** @brief File ID */ typedef uint64_t RID; /** @brief Row ID */ typedef uint32_t TxnID; /** @brief Transaction ID (New)*/ typedef uint32_t HWM; /** @brief high water mark */ -typedef unsigned __int128 uint128_t; /************************************************************************ * Type enumerations diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index 3b14b8aae..a7f41e5e5 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -56,8 +56,6 @@ struct RefcolInfo unsigned numExtents; }; -using int128_t = __int128; -using uint128_t = unsigned __int128; /** * Constructor From 3eb26c0d4a220c15dfeb5ca02e22d1ac23c3d6ed Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Fri, 6 Nov 2020 10:52:43 +0000 Subject: [PATCH 69/78] MCOL-4313 Introduced TSInt128 that is a storage class for int128 Removed uint128 from joblist/lbidlist.* Another toString() method for wide-decimal that is EMPTY/NULL aware Unified decimal processing in WF functions Fixed a potential issue in EqualCompData::operator() for wide-decimal processing Fixed some signedness warnings --- datatypes/CMakeLists.txt | 1 + datatypes/mcs_decimal.cpp | 25 +++ datatypes/mcs_decimal.h | 45 +---- datatypes/mcs_int128.cpp | 82 ++++++++ datatypes/mcs_int128.h | 187 +++++++++++++++++++ dbcon/execplan/simplecolumn.cpp | 4 +- dbcon/execplan/treenode.h | 6 + dbcon/execplan/windowfunctioncolumn.cpp | 4 +- dbcon/joblist/batchprimitiveprocessor-jl.cpp | 2 +- dbcon/joblist/lbidlist.cpp | 107 +++-------- dbcon/joblist/lbidlist.h | 2 +- primitives/linux-port/column.cpp | 21 ++- primitives/primproc/CMakeLists.txt | 2 - primitives/primproc/columncommand.cpp | 20 +- utils/common/any.hpp | 1 - utils/funcexp/func_mod.cpp | 4 +- utils/messageqcpp/bytestream.cpp | 44 +---- utils/rowgroup/rowaggregation.cpp | 8 +- utils/rowgroup/rowaggregation.h | 1 - utils/rowgroup/rowgroup.h | 61 +++--- utils/udfsdk/mcsv1_udaf.cpp | 1 - utils/udfsdk/mcsv1_udaf.h | 5 - utils/windowfunction/frameboundrange.cpp | 37 ++-- utils/windowfunction/idborderby.cpp | 5 +- utils/windowfunction/wf_count.cpp | 6 +- utils/windowfunction/wf_lead_lag.cpp | 27 +-- utils/windowfunction/wf_min_max.cpp | 24 +-- utils/windowfunction/wf_nth_value.cpp | 11 +- utils/windowfunction/wf_percentile.cpp | 14 +- utils/windowfunction/wf_stats.cpp | 11 +- utils/windowfunction/wf_sum_avg.cpp | 12 +- utils/windowfunction/wf_udaf.cpp | 11 -- utils/windowfunction/windowfunctiontype.cpp | 57 ++---- utils/windowfunction/windowfunctiontype.h | 11 +- writeengine/server/we_dmlcommandproc.cpp | 10 +- 35 files changed, 505 insertions(+), 364 deletions(-) create mode 100644 datatypes/mcs_int128.cpp create mode 100644 datatypes/mcs_int128.h diff --git a/datatypes/CMakeLists.txt b/datatypes/CMakeLists.txt index b632a9392..c74eaf7fc 100644 --- a/datatypes/CMakeLists.txt +++ b/datatypes/CMakeLists.txt @@ -1,6 +1,7 @@ include_directories( ${ENGINE_COMMON_INCLUDES} ) set(datatypes_LIB_SRCS + mcs_int128.cpp mcs_decimal.cpp) add_library(datatypes SHARED ${datatypes_LIB_SRCS}) diff --git a/datatypes/mcs_decimal.cpp b/datatypes/mcs_decimal.cpp index f66cc68ef..9da2894c5 100644 --- a/datatypes/mcs_decimal.cpp +++ b/datatypes/mcs_decimal.cpp @@ -198,6 +198,14 @@ namespace datatypes std::string Decimal::toString(VDecimal& value) { char buf[Decimal::MAXLENGTH16BYTES]; + if (value.s128Value == Decimal128Null) + { + return std::string("NULL"); + } + else if (value.s128Value == Decimal128Empty) + { + return std::string("EMPTY"); + } dataconvert::DataConvert::decimalToString(&value.s128Value, value.scale, buf, (uint8_t) sizeof(buf), datatypes::SystemCatalog::DECIMAL); @@ -209,6 +217,23 @@ namespace datatypes return toString(const_cast(value)); } + std::string Decimal::toString(const int128_t& value) + { + char buf[Decimal::MAXLENGTH16BYTES]; + if (value == Decimal128Null) + { + return std::string("NULL"); + } + else if (value == Decimal128Empty) + { + return std::string("EMPTY"); + } + int128_t& constLessValue = const_cast(value); + dataconvert::DataConvert::decimalToString(&constLessValue, + 0, buf, sizeof(buf), datatypes::SystemCatalog::DECIMAL); + return std::string(buf); + } + int Decimal::compare(const VDecimal& l, const VDecimal& r) { int128_t divisorL, divisorR; diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index b2fd8a225..3731f1cd6 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -24,11 +24,12 @@ #include "mcs_basic_types.h" #include "exceptclasses.h" #include "widedecimalutils.h" +#include "mcs_int128.h" namespace datatypes { - struct VDecimal; + class VDecimal; } // A class by Fabio Fernandes pulled off of stackoverflow @@ -259,10 +260,11 @@ class Decimal VDecimal& result); /** - @brief Convenience method to put decimal into a std::string. + @brief Convenience methods to put decimal into a std::string. */ static std::string toString(VDecimal& value); static std::string toString(const VDecimal& value); + static std::string toString(const int128_t& value); /** @brief The method detects whether decimal type is wide @@ -339,20 +341,7 @@ class Decimal return static_cast(value); } - /** - @brief The method converts a __float128 value to a double. - */ - static inline double getDoubleFromFloat128(const __float128& value) - { - if (value > static_cast<__float128>(DBL_MAX)) - return DBL_MAX; - else if (value < -static_cast<__float128>(DBL_MAX)) - return -DBL_MAX; - - return static_cast(value); - } - - /** + /** @brief The method converts a wide decimal value to a double. */ static inline double getDoubleFromWideDecimal(const int128_t& value, int8_t scale) @@ -374,19 +363,6 @@ class Decimal return getDoubleFromFloat128(static_cast<__float128>(value)); } - /** - @brief The method converts a __float128 value to a long double. - */ - static inline long double getLongDoubleFromFloat128(const __float128& value) - { - if (value > static_cast<__float128>(LDBL_MAX)) - return LDBL_MAX; - else if (value < -static_cast<__float128>(LDBL_MAX)) - return -LDBL_MAX; - - return static_cast(value); - } - /** @brief The method converts a wide decimal value to a long double. */ @@ -568,19 +544,19 @@ struct NoOverflowCheck { * @brief VDecimal type * */ -struct VDecimal +class VDecimal: public TSInt128 { - VDecimal(): s128Value(0), value(0), scale(0), precision(0) + public: + VDecimal(): value(0), scale(0), precision(0) { } VDecimal(int64_t val, int8_t s, uint8_t p, const int128_t &val128 = 0) : - s128Value(val128), + TSInt128(val128), value(val), scale(s), precision(p) - { - } + { } int decimalComp(const VDecimal& d) const { @@ -843,7 +819,6 @@ struct VDecimal } } - int128_t s128Value; int64_t value; int8_t scale; // 0~38 uint8_t precision; // 1~38 diff --git a/datatypes/mcs_int128.cpp b/datatypes/mcs_int128.cpp new file mode 100644 index 000000000..abaa9a27f --- /dev/null +++ b/datatypes/mcs_int128.cpp @@ -0,0 +1,82 @@ +/* + Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. +*/ + +#include "mcs_int128.h" + +namespace datatypes +{ + // The method converts a wide decimal s128Value to a double. + inline double TSInt128::getDoubleFromWideDecimal() + { + return getDoubleFromFloat128(static_cast<__float128>(s128Value)); + } + + // The method converts a wide decimal s128Value to a long double. + inline long double TSInt128::getLongDoubleFromWideDecimal() + { + return getLongDoubleFromFloat128(static_cast<__float128>(s128Value)); + } + + // The method converts a wide decimal s128Value to an int64_t, + // saturating the s128Value if necessary. + inline int64_t TSInt128::getInt64FromWideDecimal() + { + if (s128Value > static_cast(INT64_MAX)) + return INT64_MAX; + else if (s128Value < static_cast(INT64_MIN)) + return INT64_MIN; + + return static_cast(s128Value); + } + + // The method converts a wide decimal s128Value to an uint32_t. + inline uint32_t TSInt128::getUInt32FromWideDecimal() + { + if (s128Value > static_cast(UINT32_MAX)) + return UINT32_MAX; + else if (s128Value < 0) + return 0; + + return static_cast(s128Value); + } + + // The method converts a wide decimal s128Value to an uint64_t. + inline uint64_t TSInt128::getUInt64FromWideDecimal() + { + if (s128Value > static_cast(UINT64_MAX)) + return UINT64_MAX; + else if (s128Value < 0) + return 0; + + return static_cast(s128Value); + } + + // The method converts a wide decimal s128Value to an int32_t. + inline int32_t TSInt128::getInt32FromWideDecimal() + { + if (s128Value > static_cast(INT32_MAX)) + return INT32_MAX; + else if (s128Value < static_cast(INT32_MIN)) + return INT32_MIN; + + return static_cast(s128Value); + } + +} // end of namespace datatypes +// vim:ts=2 sw=2: diff --git a/datatypes/mcs_int128.h b/datatypes/mcs_int128.h new file mode 100644 index 000000000..ef8a3e262 --- /dev/null +++ b/datatypes/mcs_int128.h @@ -0,0 +1,187 @@ +/* + Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. +*/ +#ifndef MCS_INT128_H_INCLUDED +#define MCS_INT128_H_INCLUDED + +#include +#include +#include +#include + +// Inline asm has three argument lists: output, input and clobber list +#if defined(__GNUC__) && (__GNUC___ > 7) + #define MACRO_VALUE_PTR_128(dst, \ + dst_restrictions, \ + src, \ + src_restrictions, \ + clobb) \ + __asm__ volatile("movups %1,%0" \ + :dst_restrictions ( *(dst) ) \ + :src_restrictions ( (src) ) \ + :clobb \ + ); + #define MACRO_PTR_PTR_128(dst, \ + dst_restrictions, \ + src, \ + src_restrictions, \ + clobb) \ + ::memcpy((dst), (src), sizeof(int128_t)); + +#else + #define MACRO_VALUE_PTR_128(dst, \ + dst_restrictions, \ + src, \ + src_restrictions, \ + clobb) \ + __asm__ volatile("movups %1,%0" \ + :dst_restrictions ( *(dst) ) \ + :src_restrictions ( (src) ) \ + :clobb \ + ); + #define MACRO_PTR_PTR_128(dst, \ + dst_restrictions, \ + src, \ + src_restrictions, \ + clobb) \ + __asm__ volatile("movdqu %1,%%xmm0;" \ + "movups %%xmm0,%0;" \ + :dst_restrictions ( *(dst) ) \ + :src_restrictions ( *(src) ) \ + :"memory", clobb \ + ); +#endif + +namespace datatypes +{ + +using int128_t = __int128; + + +// Type traits +template +struct is_allowed_numeric { + static const bool value = false; +}; + +template <> +struct is_allowed_numeric { + static const bool value = true; +}; + +template <> +struct is_allowed_numeric { + static const bool value = true; +}; + +// The method converts a __float128 s128Value to a double. +static inline double getDoubleFromFloat128(const __float128& value) +{ + if (value > static_cast<__float128>(DBL_MAX)) + return DBL_MAX; + else if (value < -static_cast<__float128>(DBL_MAX)) + return -DBL_MAX; + + return static_cast(value); +} + + +// The method converts a __float128 value to a long double. +static inline long double getLongDoubleFromFloat128(const __float128& value) +{ + if (value > static_cast<__float128>(LDBL_MAX)) + return LDBL_MAX; + else if (value < -static_cast<__float128>(LDBL_MAX)) + return -LDBL_MAX; + + return static_cast(value); +} + + +class TSInt128 +{ + public: + // A variety of ctors for aligned and unaligned arguments + TSInt128(): s128Value(0) { } + + // aligned argument + TSInt128(const int128_t& x) { s128Value = x; } + + // unaligned argument + TSInt128(const int128_t* x) { assignPtrPtr(&s128Value, x); } + + // unaligned argument + TSInt128(const unsigned char* x) { assignPtrPtr(&s128Value, x); } + + // The method copies 16 bytes from one memory cell + // into another using memcpy or SIMD. + // memcpy in gcc >= 7 is replaced with SIMD instructions + template + static inline void assignPtrPtr(D* dst, const S* src) + { + MACRO_PTR_PTR_128(dst, "=m", src, "m", "xmm0") + } + + template + static inline void storeUnaligned(D* dst, const int128_t& src) + { + MACRO_VALUE_PTR_128(dst, "=m", src, "x", "memory") + } + + // operators + template::value> > + inline bool operator<(const T& x) const + { + return s128Value < static_cast(x); + } + + template::value> > + inline bool operator==(const T& x) const + { + return s128Value == static_cast(x); + } + + // The method converts a wide decimal s128Value to a double. + inline double getDoubleFromWideDecimal(); + + // The method converts a wide decimal s128Value to a long double. + inline long double getLongDoubleFromWideDecimal(); + + // The method converts a wide decimal s128Value to an int64_t, + // saturating the s128Value if necessary. + inline int64_t getInt64FromWideDecimal(); + + // The method converts a wide decimal s128Value to an uint32_t. + inline uint32_t getUInt32FromWideDecimal(); + + // The method converts a wide decimal s128Value to an uint64_t. + inline uint64_t getUInt64FromWideDecimal(); + + // The method converts a wide decimal s128Value to an int32_t. + inline int32_t getInt32FromWideDecimal(); + + int128_t s128Value; + }; // end of class + + +} //end of namespace datatypes + +#endif // MCS_TSINT128_H_INCLUDED +// vim:ts=2 sw=2: diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index 7201a366c..64620cb15 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -635,8 +635,8 @@ void SimpleColumn::evaluate(Row& row, bool& isNull) { case 16: { - fResult.decimalVal.s128Value = - *row.getBinaryField_offset(fInputOffset); + datatypes::TSInt128::assignPtrPtr(&fResult.decimalVal.s128Value, + row.getBinaryField_offset(fInputOffset)); break; } case 1: diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index ad35f1768..6b2958bff 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -61,6 +61,12 @@ class IDB_Decimal: public datatypes::VDecimal { public: using datatypes::VDecimal::VDecimal; + + inline void operator=(const datatypes::TSInt128& rhs) + { + value = 0; scale = 0; precision = 0; + datatypes::TSInt128::operator=(rhs); + } }; diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index a6ba127fa..c16c7b44b 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -687,12 +687,12 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull) case 16: { - int128_t dec = row.getInt128Field(fInputIndex); + datatypes::TSInt128 dec(row.getBinaryField(fInputIndex)); if (dec == datatypes::Decimal128Null) isNull = true; else { - fResult.decimalVal.s128Value = dec; + fResult.decimalVal = dec; fResult.decimalVal.scale = (unsigned)fResultType.scale; } diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.cpp b/dbcon/joblist/batchprimitiveprocessor-jl.cpp index aff8f4e8b..1f5707358 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.cpp +++ b/dbcon/joblist/batchprimitiveprocessor-jl.cpp @@ -736,7 +736,7 @@ void BatchPrimitiveProcessorJL::getRowGroupData(ByteStream& in, vector* uint32_t threadID, bool* hasWideColumn, const execplan::CalpontSystemCatalog::ColType& colType) const { uint64_t tmp64; - uint128_t tmp128; + int128_t tmp128; uint8_t tmp8; RGData rgData; uint32_t rowCount; diff --git a/dbcon/joblist/lbidlist.cpp b/dbcon/joblist/lbidlist.cpp index 36725dc8e..60180c8f8 100644 --- a/dbcon/joblist/lbidlist.cpp +++ b/dbcon/joblist/lbidlist.cpp @@ -236,23 +236,15 @@ bool LBIDList::GetMinMax(T& min, T& max, int64_t& seq, int64_t lbid, if (isUnsigned(colDataType)) { - if (typeid(T) == typeid(int128_t)) - { - mmp->bigMax = 0; - mmp->bigMin = -1; - } - else - { - mmp->max = 0; - mmp->min = static_cast(numeric_limits::max()); - } + mmp->max = 0; + mmp->min = static_cast(numeric_limits::max()); } else { if (typeid(T) == typeid(int128_t)) { - utils::int128Min(mmp->bigMax); - utils::int128Max(mmp->bigMin); + mmp->bigMax = datatypes::Decimal::minInt128; + mmp->bigMin = datatypes::Decimal::maxInt128; } else { @@ -296,23 +288,15 @@ bool LBIDList::GetMinMax(T* min, T* max, int64_t* seq, if (isUnsigned(colDataType)) { - if (typeid(T) == typeid(int128_t)) - { - mmp->bigMax = 0; - mmp->bigMin = -1; - } - else - { - mmp->max = 0; - mmp->min = static_cast(numeric_limits::max()); - } + mmp->max = 0; + mmp->min = static_cast(numeric_limits::max()); } else { if (typeid(T) == typeid(int128_t)) { - utils::int128Min(mmp->bigMax); - utils::int128Max(mmp->bigMin); + mmp->bigMax = datatypes::Decimal::minInt128; + mmp->bigMin = datatypes::Decimal::maxInt128; } else { @@ -327,6 +311,7 @@ bool LBIDList::GetMinMax(T* min, T* max, int64_t* seq, return false; } + // *DRRTUY min/max should be 16 aligned here if (typeid(T) == typeid(int128_t)) { *min = entry.partition.cprange.bigLoVal; @@ -356,6 +341,7 @@ int LBIDList::getMinMaxFromEntries(T& min, T& max, int32_t& seq, if (lbid >= EMEntries[i].range.start && lbid <= lastLBID) { + // *DRRTUY min/max should be 16 aligned here if (typeid(T) == typeid(int128_t)) { min = EMEntries[i].partition.cprange.bigLoVal; @@ -426,32 +412,25 @@ void LBIDList::UpdateMinMax(T min, T max, int64_t lbid, CalpontSystemCatalog::Co } else if (datatypes::isUnsigned(type)) { - if (typeid(T) == typeid(int128_t)) - { - if (static_cast(min) < static_cast(mmp->bigMin)) - mmp->bigMin = min; + if (static_cast(min) < static_cast(mmp->min)) + mmp->min = min; - if (static_cast(max) > static_cast(mmp->bigMax)) - mmp->bigMax = max; - } - else - { - if (static_cast(min) < static_cast(mmp->min)) - mmp->min = min; - - if (static_cast(max) > static_cast(mmp->max)) - mmp->max = max; - } + if (static_cast(max) > static_cast(mmp->max)) + mmp->max = max; } else { if (typeid(T) == typeid(int128_t)) { if (min < mmp->bigMin) + { mmp->bigMin = min; + } if (max > mmp->bigMax) + { mmp->bigMax = max; + } } else { @@ -694,16 +673,8 @@ bool LBIDList::checkSingleValue(T min, T max, T value, } else if (isUnsigned(type)) { - if (typeid(T) == typeid(int128_t)) - { - return (static_cast(value) >= static_cast(min) && - static_cast(value) <= static_cast(max)); - } - else - { - return (static_cast(value) >= static_cast(min) && - static_cast(value) <= static_cast(max)); - } + return (static_cast(value) >= static_cast(min) && + static_cast(value) <= static_cast(max)); } else { @@ -727,16 +698,8 @@ bool LBIDList::checkRangeOverlap(T min, T max, T tmin, T tmax, } else if (isUnsigned(type)) { - if (typeid(T) == typeid(int128_t)) - { - return (static_cast(tmin) <= static_cast(max) && - static_cast(tmax) >= static_cast(min)); - } - else - { - return (static_cast(tmin) <= static_cast(max) && - static_cast(tmax) >= static_cast(min)); - } + return (static_cast(tmin) <= static_cast(max) && + static_cast(tmax) >= static_cast(min)); } else { @@ -806,13 +769,6 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, value = static_cast(val); break; } - - case 16: - { - uint128_t val = *(int128_t*)MsgDataPtr; - bigValue = static_cast(val); - break; - } } } else @@ -849,21 +805,17 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, case 16: { - int128_t val = *(int128_t*)MsgDataPtr; - bigValue = val; + datatypes::TSInt128::assignPtrPtr(&bigValue, MsgDataPtr); break; } } } - // Should we also check for empty here? - // TODO MCOL-641 - if (ct.isWideDecimalType()) + if (ct.isWideDecimalType() && execplan::isNull(bigValue, ct)) { - if (isNull(bigValue, ct)) continue; } - else if (isNull(value, ct)) // This will work even if the data column is unsigned. + else if (execplan::isNull(value, ct)) // This will work even if the data column is unsigned. { continue; } @@ -885,14 +837,7 @@ bool LBIDList::CasualPartitionPredicate(const BRM::EMCasualPartition_t& cpRange, } else if (bIsUnsigned) { - if (ct.colWidth != datatypes::MAXDECIMALWIDTH) - { - scan = compareVal(static_cast(cpRange.loVal), static_cast(cpRange.hiVal), static_cast(value), op, lcf); - } - else - { - scan = compareVal(static_cast(cpRange.bigLoVal), static_cast(cpRange.bigHiVal), static_cast(bigValue), op, lcf); - } + scan = compareVal(static_cast(cpRange.loVal), static_cast(cpRange.hiVal), static_cast(value), op, lcf); } else { diff --git a/dbcon/joblist/lbidlist.h b/dbcon/joblist/lbidlist.h index 0f1925332..1d666cccf 100644 --- a/dbcon/joblist/lbidlist.h +++ b/dbcon/joblist/lbidlist.h @@ -91,7 +91,7 @@ public: // Functions to handle min/max values per lbid for casual partitioning; // If pEMEntries is provided, then min/max will be extracted from that // vector, else extents in BRM will be searched. If type is unsigned, caller - // should static cast returned min and max to uint64_t/uint128_t + // should static cast returned min and max to uint64_t/int128_t template bool GetMinMax(T& min, T& max, int64_t& seq, int64_t lbid, const std::vector* pEMEntries, diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 11bbe07ce..a048d984e 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -42,6 +42,7 @@ using namespace boost; #include "primproc.h" #include "dataconvert.h" #include "mcs_decimal.h" + using namespace logging; using namespace dbbc; using namespace primitives; @@ -765,6 +766,8 @@ inline void store(const NewColRequestHeader* in, default: std::cout << __func__ << " WARNING!!! unspecified column width." << std::endl; + [[fallthrough]]; + case 8: ptr2 += (rid << 3); memcpy(ptr1, ptr2, 8); @@ -1723,22 +1726,31 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, // Set the min and max if necessary. Ignore nulls. if (out->ValidMinMax && !isNull && !isEmpty) { + if (in->DataType == CalpontSystemCatalog::CHAR || in->DataType == CalpontSystemCatalog::VARCHAR) { // !!! colCompare is overloaded with int128_t only yet. if (colCompare(out->Min, val, COMPARE_GT, false, in->DataType, W, placeholderRegex)) + { out->Min = val; + } if (colCompare(out->Max, val, COMPARE_LT, false, in->DataType, W, placeholderRegex)) + { out->Max = val; + } } else { if (out->Min > val) + { out->Min = val; + } if (out->Max < val) + { out->Max = val; + } } } @@ -1828,6 +1840,7 @@ void PrimitiveProcessor::p_Col(NewColRequestHeader* in, NewColResultHeader* out, case 32: std::cout << __func__ << " WARNING!!! Not implemented for 32 byte data types." << std::endl; + [[fallthrough]]; default: idbassert(0); @@ -1918,7 +1931,7 @@ boost::shared_ptr parseColumnFilter case 8: ret->prestored_argVals[argIndex] = *reinterpret_cast(args->val); - break; + break; } } else @@ -1956,7 +1969,11 @@ boost::shared_ptr parseColumnFilter break; case 16: - ret->prestored_argVals128[argIndex] = *reinterpret_cast(args->val); + { + datatypes::TSInt128::assignPtrPtr(&(ret->prestored_argVals128[argIndex]), + args->val); + break; + } } } diff --git a/primitives/primproc/CMakeLists.txt b/primitives/primproc/CMakeLists.txt index aab0232ac..b8d2e3741 100644 --- a/primitives/primproc/CMakeLists.txt +++ b/primitives/primproc/CMakeLists.txt @@ -21,8 +21,6 @@ set(PrimProc_SRCS umsocketselector.cpp ../../utils/common/crashtrace.cpp) -#PrimProc_CXXFLAGS = $(march_flags) $(AM_CXXFLAGS) - add_executable(PrimProc ${PrimProc_SRCS}) add_dependencies(PrimProc loggingcpp) diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index fbedc0c6a..ab6fbe223 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -40,6 +40,7 @@ using namespace std; #include "primitiveserver.h" #include "primproc.h" #include "stats.h" +#include "datatypes/mcs_int128.h" using namespace messageqcpp; using namespace rowgroup; @@ -231,11 +232,7 @@ void ColumnCommand::loadData() { ByteStream::hexbyte h; utils::getEmptyRowValue(colType.colDataType, colType.colWidth, (uint8_t*)&h); - __asm__ volatile("movups %1,%0" - :"=m" ( hPtr[idx] ) // output - :"v"( h ) // input - : "memory" // clobbered - ); + datatypes::TSInt128::storeUnaligned(hPtr + idx, h); } } @@ -332,18 +329,7 @@ void ColumnCommand::process_OT_BOTH() bpp->relRids[i] = *((uint16_t*) &bpp->outputMsg[pos]); pos += 2; - int128_t* int128Ptr = reinterpret_cast(&bpp->outputMsg[pos]); - __asm__ volatile("movdqu %0,%%xmm0;" - : - :"m"( *int128Ptr ) // input - :"xmm0" // clobbered - ); - __asm__ volatile("movups %%xmm0,%0;" - : "=m" (wide128Values[i])// output - : // input - : "memory", "xmm0" // clobbered - ); - + datatypes::TSInt128::assignPtrPtr(&wide128Values[i], &bpp->outputMsg[pos]); pos += 16; } diff --git a/utils/common/any.hpp b/utils/common/any.hpp index f39d5cec2..64b6e1bd4 100755 --- a/utils/common/any.hpp +++ b/utils/common/any.hpp @@ -129,7 +129,6 @@ namespace anyimpl choose_policy { typedef big_any_policy type; }; BIG_POLICY(int128_t); - BIG_POLICY(uint128_t); /// Specializations for small types. #define SMALL_POLICY(TYPE) template<> struct \ diff --git a/utils/funcexp/func_mod.cpp b/utils/funcexp/func_mod.cpp index 1afc308f5..01432cea8 100644 --- a/utils/funcexp/func_mod.cpp +++ b/utils/funcexp/func_mod.cpp @@ -242,7 +242,7 @@ double Func_mod::getDoubleVal(Row& row, int128_t value = d.s128Value / scaleDivisor; int128_t lefto = d.s128Value % scaleDivisor; __float128 tmp = (__float128) (value % div) + (__float128) lefto / scaleDivisor; - mod = datatypes::Decimal::getDoubleFromFloat128(tmp); + mod = datatypes::getDoubleFromFloat128(tmp); } } else @@ -365,7 +365,7 @@ long double Func_mod::getLongDoubleVal(Row& row, int128_t value = d.s128Value / scaleDivisor; int128_t lefto = d.s128Value % scaleDivisor; __float128 tmp = (__float128) (value % div) + (__float128) lefto / scaleDivisor; - mod = datatypes::Decimal::getLongDoubleFromFloat128(tmp); + mod = datatypes::getLongDoubleFromFloat128(tmp); } } else diff --git a/utils/messageqcpp/bytestream.cpp b/utils/messageqcpp/bytestream.cpp index d42239eb0..bef473b1c 100644 --- a/utils/messageqcpp/bytestream.cpp +++ b/utils/messageqcpp/bytestream.cpp @@ -37,6 +37,7 @@ using namespace boost; #define BYTESTREAM_DLLEXPORT #include "bytestream.h" #undef BYTESTREAM_DLLEXPORT +#include "datatypes/mcs_int128.h" #define DEBUG_DUMP_STRINGS_LESS_THAN 0 @@ -239,15 +240,8 @@ ByteStream& ByteStream::operator<<(const uint128_t& o) { if (fBuf == 0 || (fCurInPtr - fBuf + 16U > fMaxLen + ISSOverhead)) growBuf(fMaxLen + BlockSize); - - __asm__ volatile("movups %1,%0;" - :"=m" ( *fCurInPtr ) // output - :"v"( o ) // input - : "memory" // clobbered - ); - + datatypes::TSInt128::storeUnaligned(fCurInPtr, o); fCurInPtr += 16; - return *this; } @@ -255,14 +249,8 @@ ByteStream& ByteStream::operator<<(const int128_t& o) { if (fBuf == 0 || (fCurInPtr - fBuf + 16U > fMaxLen + ISSOverhead)) growBuf(fMaxLen + BlockSize); - - __asm__ volatile("movups %1,%0;" - :"=m" ( *fCurInPtr ) // output - :"v"( o ) // input - : "memory" // clobbered - ); + datatypes::TSInt128::storeUnaligned(fCurInPtr, o); fCurInPtr += 16; - return *this; } @@ -446,38 +434,16 @@ void ByteStream::peek(uint64_t& o) const void ByteStream::peek(uint128_t& o) const { - if (length() < 16) throw underflow_error("ByteStream>uint128_t: not enough data in stream to fill datatype"); - - __asm__ volatile("movdqu %0,%%xmm0;" - : - :"m"( *fCurOutPtr ) // input - :"xmm0" // clobbered - ); - __asm__ volatile("movups %%xmm0,%0;" - : "=m" (o)// output - : // input - : "memory", "xmm0" // clobbered - ); + datatypes::TSInt128::assignPtrPtr(&o, fCurOutPtr); } void ByteStream::peek(int128_t& o) const { - if (length() < 16) throw underflow_error("ByteStream>int128_t: not enough data in stream to fill datatype"); - - __asm__ volatile("movdqu %0,%%xmm0;" - : - :"m"( *fCurOutPtr ) // input - :"xmm0" // clobbered - ); - __asm__ volatile("movups %%xmm0,%0;" - : "=m" (o)// output - : // input - : "memory", "xmm0" // clobbered - ); + datatypes::TSInt128::assignPtrPtr(&o, fCurOutPtr); } void ByteStream::peek(string& s) const diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index e9199f8fd..f51fcd022 100755 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -244,7 +244,6 @@ const static_any::any& RowAggregation::ushortTypeId((unsigned short)1); const static_any::any& RowAggregation::uintTypeId((unsigned int)1); const static_any::any& RowAggregation::ulongTypeId((unsigned long)1); const static_any::any& RowAggregation::ullTypeId((unsigned long long)1); -const static_any::any& RowAggregation::uint128TypeId((uint128_t)1); const static_any::any& RowAggregation::floatTypeId((float)1); const static_any::any& RowAggregation::doubleTypeId((double)1); const static_any::any& RowAggregation::longdoubleTypeId((long double)1); @@ -2184,7 +2183,10 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, if (LIKELY(fRowGroupIn.getColumnWidth(colIn) == datatypes::MAXDECIMALWIDTH)) { - datum.columnData = rowIn.getInt128Field(colIn); + // We can't control boost::any asignment + // so let's get an aligned memory + datatypes::TSInt128 val = rowIn.getTSInt128Field(colIn); + datum.columnData = val.s128Value; } else if (fRowGroupIn.getColumnWidth(colIn) <= datatypes::MAXLEGACYWIDTH) { @@ -2753,7 +2755,7 @@ void RowAggregationUM::calculateAvgColumns() bool isWideDecimal = datatypes::Decimal::isWideDecimalTypeByPrecision(precision); - if (LIKELY(!isWideDecimal)) + if (!isWideDecimal) { long double sum = 0.0; long double avg = 0.0; diff --git a/utils/rowgroup/rowaggregation.h b/utils/rowgroup/rowaggregation.h index 7fb4a10dc..34cd6fd03 100644 --- a/utils/rowgroup/rowaggregation.h +++ b/utils/rowgroup/rowaggregation.h @@ -725,7 +725,6 @@ protected: static const static_any::any& ushortTypeId; static const static_any::any& uintTypeId; static const static_any::any& ulongTypeId; - static const static_any::any& uint128TypeId; static const static_any::any& ullTypeId; static const static_any::any& floatTypeId; static const static_any::any& doubleTypeId; diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 2d1ba045c..45f2be327 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -55,6 +55,7 @@ #include "mcsv1_udaf.h" #include "branchpred.h" +#include "datatypes/mcs_int128.h" #include "../winport/winport.h" @@ -386,8 +387,8 @@ public: return 0.0; // TODO: Do something here } inline long double getLongDoubleField(uint32_t colIndex) const; - inline int128_t getInt128Field(uint32_t colIndex) const; - inline uint128_t getUint128Field(uint32_t colIndex) const; + inline void getInt128Field(uint32_t colIndex, int128_t& x) const; + inline datatypes::TSInt128 getTSInt128Field(uint32_t colIndex) const; inline uint64_t getBaseRid() const; inline uint64_t getRid() const; @@ -418,7 +419,6 @@ public: inline void setDecimalField(double val, uint32_t colIndex) { }; // TODO: Do something here inline void setLongDoubleField(const long double& val, uint32_t colIndex); inline void setInt128Field(const int128_t& val, uint32_t colIndex); - inline void setUint128Field(const uint128_t& val, uint32_t colIndex); inline void setRid(uint64_t rid); @@ -429,11 +429,11 @@ public: void setStringField(const std::string& val, uint32_t colIndex); inline void setStringField(const uint8_t*, uint32_t len, uint32_t colIndex); template - inline void setBinaryField(T* strdata, uint32_t width, uint32_t colIndex); + inline void setBinaryField(const T* value, uint32_t width, uint32_t colIndex); template - inline void setBinaryField(T* strdata, uint32_t colIndex); + inline void setBinaryField(const T* value, uint32_t colIndex); template - inline void setBinaryField_offset(T* strdata, uint32_t width, uint32_t colIndex); + inline void setBinaryField_offset(const T* value, uint32_t width, uint32_t colIndex); // support VARBINARY // Add 2-byte length at the CHARSET_INFO*beginning of the field. NULL and zero length field are // treated the same, could use one of the length bit to distinguish these two cases. @@ -821,44 +821,41 @@ inline uint32_t Row::getStringLength(uint32_t colIndex) const } template -inline void Row::setBinaryField(T* value, uint32_t width, uint32_t colIndex) +inline void Row::setBinaryField(const T* value, uint32_t width, uint32_t colIndex) { memcpy(&data[offsets[colIndex]], value, width); } template -inline void Row::setBinaryField(T* value, uint32_t colIndex) +inline void Row::setBinaryField(const T* value, uint32_t colIndex) { *reinterpret_cast(&data[offsets[colIndex]]) = *value; } +template<> +inline void Row::setBinaryField(const int128_t* value, uint32_t colIndex) +{ + datatypes::TSInt128::assignPtrPtr(&data[offsets[colIndex]], value); +} + + // This method !cannot! be applied to uint8_t* buffers. template -inline void Row::setBinaryField_offset(T* value, uint32_t width, uint32_t offset) +inline void Row::setBinaryField_offset(const T* value, uint32_t width, uint32_t offset) { *reinterpret_cast(&data[offset]) = *value; } template<> -inline void Row::setBinaryField_offset(uint8_t* value, uint32_t width, uint32_t offset) +inline void Row::setBinaryField_offset(const uint8_t* value, uint32_t width, uint32_t offset) { memcpy(&data[offset], value, width); } template<> -inline void Row::setBinaryField_offset(int128_t* value, uint32_t width, uint32_t offset) +inline void Row::setBinaryField_offset(const int128_t* value, uint32_t width, uint32_t offset) { - int128_t *dst128Ptr = reinterpret_cast(&data[offset]); - __asm__ volatile("movdqu %0,%%xmm0;" - : - :"m"( *value ) // input - :"xmm0" // clobbered - ); - __asm__ volatile("movups %%xmm0,%0;" - : "=m" (*dst128Ptr)// output - : // input - : "memory", "xmm0" // clobbered - ); + datatypes::TSInt128::assignPtrPtr(&data[offset], value); } inline void Row::setStringField(const uint8_t* strdata, uint32_t length, uint32_t colIndex) @@ -975,18 +972,15 @@ inline long double Row::getLongDoubleField(uint32_t colIndex) const return *((long double*) &data[offsets[colIndex]]); } -// !!! Never ever try to remove inline from this f() b/c it returns -// non-integral 16 byte DT -inline int128_t Row::getInt128Field(uint32_t colIndex) const +inline void Row::getInt128Field(uint32_t colIndex, int128_t& x) const { - return *((int128_t*) &data[offsets[colIndex]]); + datatypes::TSInt128::assignPtrPtr(&x, &data[offsets[colIndex]]); } -// !!! Never ever try to remove inline from this f() b/c it returns -// non-integral 16 byte DT -inline uint128_t Row::getUint128Field(uint32_t colIndex) const +inline datatypes::TSInt128 Row::getTSInt128Field(uint32_t colIndex) const { - return *((uint128_t*) &data[offsets[colIndex]]); + const int128_t* ptr = getBinaryField(colIndex); + return datatypes::TSInt128(ptr); } inline uint64_t Row::getRid() const @@ -1198,12 +1192,7 @@ inline void Row::setLongDoubleField(const long double& val, uint32_t colIndex) inline void Row::setInt128Field(const int128_t& val, uint32_t colIndex) { - *((int128_t*)&data[offsets[colIndex]]) = val; -} - -inline void Row::setUint128Field(const uint128_t& val, uint32_t colIndex) -{ - *((uint128_t*)&data[offsets[colIndex]]) = val; + setBinaryField(&val, colIndex); } inline void Row::setVarBinaryField(const std::string& val, uint32_t colIndex) diff --git a/utils/udfsdk/mcsv1_udaf.cpp b/utils/udfsdk/mcsv1_udaf.cpp index cd08e3573..b9d7cdac1 100755 --- a/utils/udfsdk/mcsv1_udaf.cpp +++ b/utils/udfsdk/mcsv1_udaf.cpp @@ -285,7 +285,6 @@ const static_any::any& mcsv1_UDAF::ushortTypeId((unsigned short)1); const static_any::any& mcsv1_UDAF::uintTypeId((unsigned int)1); const static_any::any& mcsv1_UDAF::ulongTypeId((unsigned long)1); const static_any::any& mcsv1_UDAF::ullTypeId((unsigned long long)1); -const static_any::any& mcsv1_UDAF::uint128TypeId((uint128_t)1); const static_any::any& mcsv1_UDAF::floatTypeId((float)1); const static_any::any& mcsv1_UDAF::doubleTypeId((double)1); const static_any::any& mcsv1_UDAF::strTypeId(typeStr); diff --git a/utils/udfsdk/mcsv1_udaf.h b/utils/udfsdk/mcsv1_udaf.h index 6c0ea7370..ff37457ef 100755 --- a/utils/udfsdk/mcsv1_udaf.h +++ b/utils/udfsdk/mcsv1_udaf.h @@ -644,7 +644,6 @@ protected: static const static_any::any& uintTypeId; static const static_any::any& ulongTypeId; static const static_any::any& ullTypeId; - static const static_any::any& uint128TypeId; static const static_any::any& floatTypeId; static const static_any::any& doubleTypeId; static const static_any::any& strTypeId; @@ -1075,10 +1074,6 @@ inline T mcsv1_UDAF::convertAnyTo(static_any::any& valIn) { val = valIn.cast(); } - else if (valIn.compatible(uint128TypeId)) - { - val = valIn.cast(); - } else { throw std::runtime_error("mcsv1_UDAF::convertAnyTo(): input param has unrecognized type"); diff --git a/utils/windowfunction/frameboundrange.cpp b/utils/windowfunction/frameboundrange.cpp index e10cb6fbd..e2ae172e1 100644 --- a/utils/windowfunction/frameboundrange.cpp +++ b/utils/windowfunction/frameboundrange.cpp @@ -300,7 +300,8 @@ void FrameBoundExpressionRange::validate() case execplan::CalpontSystemCatalog::DECIMAL: { - if (this->fRow.getColumnWidth(this->fIndex[1]) < 16) + if (UNLIKELY(this->fRow.getColumnWidth(this->fIndex[1]) + < datatypes::MAXDECIMALWIDTH)) { int64_t tmp = this->fRow.getIntField(this->fIndex[1]); this->fIsZero = (tmp == 0); @@ -313,7 +314,7 @@ void FrameBoundExpressionRange::validate() } else { - int128_t tmp = this->fRow.getInt128Field(this->fIndex[1]); + datatypes::TSInt128 tmp = this->fRow.getTSInt128Field(this->fIndex[1]); this->fIsZero = (tmp == 0); if (tmp < 0) @@ -325,6 +326,22 @@ void FrameBoundExpressionRange::validate() break; } + case execplan::CalpontSystemCatalog::UDECIMAL: + { + if (UNLIKELY(this->fRow.getColumnWidth(this->fIndex[1]) + < datatypes::MAXDECIMALWIDTH)) + { + uint64_t tmp = this->fRow.getUintField(this->fIndex[1]); + this->fIsZero = (tmp == 0); + } + else + { + datatypes::TSInt128 tmp = this->fRow.getTSInt128Field(this->fIndex[1]); + this->fIsZero = (tmp == 0); + } + break; + } + case execplan::CalpontSystemCatalog::DOUBLE: case execplan::CalpontSystemCatalog::UDOUBLE: { @@ -369,22 +386,6 @@ void FrameBoundExpressionRange::validate() break; } - case execplan::CalpontSystemCatalog::UDECIMAL: - { - if (this->fRow.getColumnWidth(this->fIndex[1]) < 16) - { - uint64_t tmp = this->fRow.getUintField(this->fIndex[1]); - this->fIsZero = (tmp == 0); - break; - } - else - { - uint128_t tmp = this->fRow.getUint128Field(this->fIndex[1]); - this->fIsZero = (tmp == 0); - break; - } - } - case execplan::CalpontSystemCatalog::UTINYINT: case execplan::CalpontSystemCatalog::USMALLINT: case execplan::CalpontSystemCatalog::UMEDINT: diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index 69344c43e..93e2d468e 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -853,13 +853,14 @@ bool EqualCompData::operator()(Row::Pointer a, Row::Pointer b) case CalpontSystemCatalog::UDECIMAL: { // equal compare. ignore sign and null - if (fRow1.getColumnWidth(*i) < datatypes::MAXDECIMALWIDTH) + if (UNLIKELY(fRow1.getColumnWidth(*i) < datatypes::MAXDECIMALWIDTH)) { eq = (fRow1.getUintField(*i) == fRow2.getUintField(*i)); } else if (fRow1.getColumnWidth(*i) == datatypes::MAXDECIMALWIDTH) { - eq = (fRow1.getUint128Field(*i) == fRow2.getUint128Field(*i)); + eq = (*fRow1.getBinaryField(*i) == + *fRow2.getBinaryField(*i)); } break; } diff --git a/utils/windowfunction/wf_count.cpp b/utils/windowfunction/wf_count.cpp index 29987c3b4..67bc82ed4 100644 --- a/utils/windowfunction/wf_count.cpp +++ b/utils/windowfunction/wf_count.cpp @@ -78,11 +78,13 @@ boost::shared_ptr WF_count::makeFunction(int id, const st case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - if (wc->functionParms()[0]->resultType().colWidth < datatypes::MAXDECIMALWIDTH) + decltype(datatypes::MAXDECIMALWIDTH) width = + wc->functionParms()[0]->resultType().colWidth; + if (width < datatypes::MAXDECIMALWIDTH) { func.reset(new WF_count(id, name)); } - else if (wc->functionParms()[0]->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + else if (width == datatypes::MAXDECIMALWIDTH) { func.reset(new WF_count(id, name)); } diff --git a/utils/windowfunction/wf_lead_lag.cpp b/utils/windowfunction/wf_lead_lag.cpp index 1a1b453cd..ff363151b 100644 --- a/utils/windowfunction/wf_lead_lag.cpp +++ b/utils/windowfunction/wf_lead_lag.cpp @@ -84,31 +84,23 @@ boost::shared_ptr WF_lead_lag::makeFunction(int id, const } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { - if (wc->functionParms()[0]->resultType().colWidth < datatypes::MAXDECIMALWIDTH) + decltype(datatypes::MAXDECIMALWIDTH) width = + wc->functionParms()[0]->resultType().colWidth; + if (width < datatypes::MAXDECIMALWIDTH) { - func.reset(new WF_lead_lag(id, name)); + if (ct == CalpontSystemCatalog::UDECIMAL) + func.reset(new WF_lead_lag(id, name)); + else + func.reset(new WF_lead_lag(id, name)); } - else if (wc->functionParms()[0]->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + else if (width == datatypes::MAXDECIMALWIDTH) { func.reset(new WF_lead_lag(id, name)); } break; } - - case CalpontSystemCatalog::UDECIMAL: - { - if (wc->functionParms()[0]->resultType().colWidth < 16) - { - func.reset(new WF_lead_lag(id, name)); - } - else - { - func.reset(new WF_lead_lag(id, name)); - } - break; - } - case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: { @@ -319,7 +311,6 @@ boost::shared_ptr WF_lead_lag::makeFunction(int, co template void WF_lead_lag::parseParms(const std::vector&); template void WF_lead_lag::parseParms(const std::vector&); template void WF_lead_lag::parseParms(const std::vector&); -template void WF_lead_lag::parseParms(const std::vector&); template void WF_lead_lag::parseParms(const std::vector&); template void WF_lead_lag::parseParms(const std::vector&); template void WF_lead_lag::parseParms(const std::vector&); diff --git a/utils/windowfunction/wf_min_max.cpp b/utils/windowfunction/wf_min_max.cpp index f9c410362..036a393fc 100644 --- a/utils/windowfunction/wf_min_max.cpp +++ b/utils/windowfunction/wf_min_max.cpp @@ -83,10 +83,17 @@ boost::shared_ptr WF_min_max::makeFunction(int id, const } case CalpontSystemCatalog::DECIMAL: + case CalpontSystemCatalog::UDECIMAL: { - if (wc->functionParms()[0]->resultType().colWidth < 16) + decltype(datatypes::MAXDECIMALWIDTH) width = + wc->functionParms()[0]->resultType().colWidth; + + if (width < datatypes::MAXDECIMALWIDTH) { - func.reset(new WF_min_max(id, name)); + if (ct == CalpontSystemCatalog::UDECIMAL) + func.reset(new WF_min_max(id, name)); + else + func.reset(new WF_min_max(id, name)); } else { @@ -94,19 +101,6 @@ boost::shared_ptr WF_min_max::makeFunction(int id, const } break; } - - case CalpontSystemCatalog::UDECIMAL: - { - if (wc->functionParms()[0]->resultType().colWidth < 16) - { - func.reset(new WF_min_max(id, name)); - } - else - { - func.reset(new WF_min_max(id, name)); - } - break; - } case CalpontSystemCatalog::DOUBLE: case CalpontSystemCatalog::UDOUBLE: diff --git a/utils/windowfunction/wf_nth_value.cpp b/utils/windowfunction/wf_nth_value.cpp index 41dde88fb..d838c8ec3 100644 --- a/utils/windowfunction/wf_nth_value.cpp +++ b/utils/windowfunction/wf_nth_value.cpp @@ -86,11 +86,16 @@ boost::shared_ptr WF_nth_value::makeFunction(int id, cons case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - if (wc->functionParms()[0]->resultType().colWidth < datatypes::MAXDECIMALWIDTH) + decltype(datatypes::MAXDECIMALWIDTH) width = + wc->functionParms()[0]->resultType().colWidth; + if (width < datatypes::MAXDECIMALWIDTH) { - func.reset(new WF_nth_value(id, name)); + if (ct == CalpontSystemCatalog::UDECIMAL) + func.reset(new WF_nth_value(id, name)); + else + func.reset(new WF_nth_value(id, name)); } - else if (wc->functionParms()[0]->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + else if (width == datatypes::MAXDECIMALWIDTH) { func.reset(new WF_nth_value(id, name)); } diff --git a/utils/windowfunction/wf_percentile.cpp b/utils/windowfunction/wf_percentile.cpp index a407d41e8..d23f9e712 100644 --- a/utils/windowfunction/wf_percentile.cpp +++ b/utils/windowfunction/wf_percentile.cpp @@ -46,8 +46,6 @@ using namespace ordering; #include "constantcolumn.h" using namespace execplan; -#include "mcs_decimal.h" - #include "windowfunctionstep.h" using namespace joblist; @@ -93,11 +91,17 @@ boost::shared_ptr WF_percentile::makeFunction(int id, con case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - if (wc->resultType().colWidth < datatypes::MAXDECIMALWIDTH) + decltype(datatypes::MAXDECIMALWIDTH) width = + wc->resultType().colWidth; + + if (width < datatypes::MAXDECIMALWIDTH) { - func.reset(new WF_percentile(id, name)); + if (ct == CalpontSystemCatalog::UDECIMAL) + func.reset(new WF_percentile(id, name)); + else + func.reset(new WF_percentile(id, name)); } - else if (wc->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + else if (width == datatypes::MAXDECIMALWIDTH) { func.reset(new WF_percentile(id, name)); } diff --git a/utils/windowfunction/wf_stats.cpp b/utils/windowfunction/wf_stats.cpp index 0dee4fa9e..d736704f8 100644 --- a/utils/windowfunction/wf_stats.cpp +++ b/utils/windowfunction/wf_stats.cpp @@ -84,11 +84,16 @@ boost::shared_ptr WF_stats::makeFunction(int id, const st case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - if (wc->functionParms()[0]->resultType().colWidth < datatypes::MAXDECIMALWIDTH) + decltype(datatypes::MAXDECIMALWIDTH) width = + wc->functionParms()[0]->resultType().colWidth; + if (width < datatypes::MAXDECIMALWIDTH) { - func.reset(new WF_stats(id, name)); + if (ct == CalpontSystemCatalog::UDECIMAL) + func.reset(new WF_stats(id, name)); + else + func.reset(new WF_stats(id, name)); } - else if (wc->functionParms()[0]->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + else if (width == datatypes::MAXDECIMALWIDTH) { func.reset(new WF_stats(id, name)); } diff --git a/utils/windowfunction/wf_sum_avg.cpp b/utils/windowfunction/wf_sum_avg.cpp index c008436bf..5a0fbbfc0 100644 --- a/utils/windowfunction/wf_sum_avg.cpp +++ b/utils/windowfunction/wf_sum_avg.cpp @@ -181,11 +181,17 @@ boost::shared_ptr WF_sum_avg::makeFunction(int case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - if (wc->functionParms()[0]->resultType().colWidth < datatypes::MAXDECIMALWIDTH) + decltype(datatypes::MAXDECIMALWIDTH) width = + wc->functionParms()[0]->resultType().colWidth; + + if (width < datatypes::MAXDECIMALWIDTH) { - func.reset(new WF_sum_avg(id, name)); + if (ct == CalpontSystemCatalog::UDECIMAL) + func.reset(new WF_sum_avg(id, name)); + else + func.reset(new WF_sum_avg(id, name)); } - else if (wc->functionParms()[0]->resultType().colWidth == datatypes::MAXDECIMALWIDTH) + else if (width == datatypes::MAXDECIMALWIDTH) { func.reset(new WF_sum_avg(id, name)); } diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index 62d4a2714..b7e7881dd 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -550,7 +550,6 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, static const static_any::any& uintTypeId = (unsigned int)1; static const static_any::any& ulongTypeId = (unsigned long)1; static const static_any::any& ullTypeId = (unsigned long long)1; - static const static_any::any& uint128TypeId = (uint128_t)1; static const static_any::any& floatTypeId = (float)1; static const static_any::any& doubleTypeId = (double)1; static const std::string typeStr(""); @@ -565,7 +564,6 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, int64_t intOut = 0; uint64_t uintOut = 0; int128_t int128Out = 0; - uint128_t uint128Out = 0; float floatOut = 0.0; double doubleOut = 0.0; long double longdoubleOut = 0.0; @@ -664,15 +662,6 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, longdoubleOut = int128Out; oss << longdoubleOut; } - else if (valOut.compatible(uint128TypeId)) - { - uint128Out = valOut.cast(); - uintOut = intOut = uint128Out; // may truncate - floatOut = uint128Out; - doubleOut = uint128Out; - longdoubleOut = uint128Out; - oss << longdoubleOut; - } if (valOut.compatible(strTypeId)) { diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index 738c2bd4e..d93b59683 100644 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -58,7 +58,6 @@ using namespace joblist; #include "wf_stats.h" #include "wf_sum_avg.h" #include "wf_udaf.h" -#include "mcs_decimal.h" namespace windowfunction { @@ -310,17 +309,7 @@ template<> void WindowFunctionType::getValue(uint64_t i, string& t, CDT* template<> void WindowFunctionType::getValue(uint64_t i, int128_t& t, CDT* cdt) { - t = fRow.getInt128Field(i); - - if (cdt) - { - *cdt = execplan::CalpontSystemCatalog::DECIMAL; - } -} - -template<> void WindowFunctionType::getValue(uint64_t i, uint128_t& t, CDT* cdt) -{ - t = fRow.getUint128Field(i); + fRow.getInt128Field(i, t); if (cdt) { @@ -362,11 +351,6 @@ template<> void WindowFunctionType::setValue(uint64_t i, int128_t& t) fRow.setInt128Field(t, i); } -template<> void WindowFunctionType::setValue(uint64_t i, uint128_t& t) -{ - fRow.setUint128Field(t, i); -} - template<> void WindowFunctionType::setValue(uint64_t i, string& t) { fRow.setStringField(t, i); @@ -460,7 +444,7 @@ void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, T* v) } else { - uint128_t iv = *v; + int128_t iv = *v; setValue(i, iv); } break; @@ -525,22 +509,22 @@ void WindowFunctionType::implicit2T(uint64_t i, T& t, int s) } case CalpontSystemCatalog::DECIMAL: - { - uint32_t w = fRow.getColumnWidth(i); - if (w < 16) - t = (T) fRow.getIntField(i); - else - t = (T) fRow.getInt128Field(i); - break; - } - case CalpontSystemCatalog::UDECIMAL: { - uint32_t w = fRow.getColumnWidth(i); - if (w < 16) - t = (T) fRow.getUintField(i); - else - t = (T) fRow.getUint128Field(i); + decltype(datatypes::MAXDECIMALWIDTH) width = + fRow.getColumnWidth(i);; + + if (width < datatypes::MAXDECIMALWIDTH) + { + t = (ct == execplan::CalpontSystemCatalog::DECIMAL) ? + (T) fRow.getIntField(i) : + (T) fRow.getUintField(i); + } + else if (width == datatypes::MAXDECIMALWIDTH) + { + datatypes::TSInt128::assignPtrPtr(&t, + fRow.getBinaryField(i)); + } break; } @@ -615,12 +599,6 @@ void WindowFunctionType::getConstValue(ConstantColumn* cc, int128_t& t t = cc->getDecimalVal(fRow, b).s128Value; } -template<> -void WindowFunctionType::getConstValue(ConstantColumn* cc, uint128_t& t, bool& b) -{ - t = cc->getDecimalVal(fRow, b).s128Value; -} - template<> void WindowFunctionType::getConstValue(ConstantColumn* cc, double& t, bool& b) { @@ -651,15 +629,12 @@ template void WindowFunctionType::implicit2T(uint64_t, float&, int); template void WindowFunctionType::implicit2T(uint64_t, double&, int); template void WindowFunctionType::implicit2T(uint64_t, long double&, int); template void WindowFunctionType::implicit2T(uint64_t, int128_t&, int); -template void WindowFunctionType::implicit2T(uint64_t, uint128_t&, int); - template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, int64_t*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, uint64_t*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, float*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, double*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, long double*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, int128_t*); -template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, uint128_t*); void* WindowFunctionType::getNullValueByType(int ct, int pos) { diff --git a/utils/windowfunction/windowfunctiontype.h b/utils/windowfunction/windowfunctiontype.h index e08bc0611..dfda6e053 100644 --- a/utils/windowfunction/windowfunctiontype.h +++ b/utils/windowfunction/windowfunctiontype.h @@ -33,6 +33,7 @@ #include "rowgroup.h" #include "windowframe.h" #include "constantcolumn.h" +#include "mcs_decimal.h" namespace ordering { @@ -222,14 +223,16 @@ protected: virtual void* getNullValueByType(int, int); + // There are two types of getters for integral types and for + // DTs wider then 8 bytes. + void getInt128Value(uint64_t i, int128_t& x) + { + return fRow.getInt128Field(i, x); + } int64_t getIntValue(uint64_t i) { return fRow.getIntField(i); } - int128_t getInt128Value(uint64_t i) - { - return fRow.getInt128Field(i); - } double getDoubleValue(uint64_t i) { return fRow.getDoubleField(i); diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index f9c577dbd..56f8ebbc5 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -2992,7 +2992,6 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - // WIP MCOL-641 if (fetchColColwidths[fetchColPos] == datatypes::MAXDECIMALWIDTH) { int128_t* dec; @@ -3007,9 +3006,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, break; } - - // else - // fall through to integer cases + [[fallthrough]]; } /* fall through */ @@ -3360,7 +3357,6 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, { if (fetchColColwidths[fetchColPos] == datatypes::MAXDECIMALWIDTH) { - // WIP MCOL-641 int128_t* dec; char buf[datatypes::Decimal::MAXLENGTH16BYTES]; dec = row.getBinaryField(fetchColPos); @@ -3373,9 +3369,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, break; } - - // else - // fall through to integer cases + [[fallthrough]]; } /* fall through */ From 15b1bfa709ca6a2277f77b24bcba1cf65dc45da3 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Sun, 8 Nov 2020 14:57:28 +0000 Subject: [PATCH 70/78] Fix fallthrough compilation warnings --- dbcon/execplan/simplecolumn_decimal.h | 1 + dbcon/execplan/simplecolumn_uint.h | 1 + dbcon/execplan/windowfunctioncolumn.cpp | 1 + dbcon/joblist/batchprimitiveprocessor-jl.cpp | 3 ++- dbcon/joblist/columncommand-jl.cpp | 3 ++- dbcon/joblist/rowestimator.cpp | 4 ++-- primitives/linux-port/column.cpp | 6 +++--- utils/rowgroup/rowgroup.cpp | 4 +++- utils/windowfunction/wf_udaf.cpp | 11 +++++++---- utils/windowfunction/windowfunctiontype.cpp | 3 ++- writeengine/server/we_dmlcommandproc.cpp | 2 -- 11 files changed, 24 insertions(+), 15 deletions(-) diff --git a/dbcon/execplan/simplecolumn_decimal.h b/dbcon/execplan/simplecolumn_decimal.h index 7a5acbb56..a6fdac5e4 100644 --- a/dbcon/execplan/simplecolumn_decimal.h +++ b/dbcon/execplan/simplecolumn_decimal.h @@ -144,6 +144,7 @@ void SimpleColumn_Decimal::setNullVal() break; case 16: std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; + //fallthrough default: fNullVal = joblist::BIGINTNULL; } diff --git a/dbcon/execplan/simplecolumn_uint.h b/dbcon/execplan/simplecolumn_uint.h index f8b8a5c96..edd811cba 100644 --- a/dbcon/execplan/simplecolumn_uint.h +++ b/dbcon/execplan/simplecolumn_uint.h @@ -144,6 +144,7 @@ void SimpleColumn_UINT::setNullVal() break; case 16: std::cout << __FILE__<< ":" <<__LINE__ << " Fix 16 Bytes ?" << std::endl; + //fallthrough default: fNullVal = joblist::UBIGINTNULL; } diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index c16c7b44b..ec6e32f16 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -491,6 +491,7 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull) break; case 16: cout << __FILE__<< ":" <<__LINE__ << " Fix 16 Bytes ?" << endl; + //fallthrough default: if (row.equals(CPNULLSTRMARK, fInputIndex)) isNull = true; diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.cpp b/dbcon/joblist/batchprimitiveprocessor-jl.cpp index 1f5707358..c989a52ed 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.cpp +++ b/dbcon/joblist/batchprimitiveprocessor-jl.cpp @@ -681,7 +681,8 @@ void BatchPrimitiveProcessorJL::getTuples(messageqcpp::ByteStream& in, pos++; break; case 16: - cout << __FILE__<< ":" <<__LINE__ << " Fix 16 Bytes ?" << endl; + cout << __FILE__<< ":" <<__LINE__ << " Fix 16 Bytes ?" << endl; + //fallthrough default: cout << "BPP::getTuples(): bad column width of " << colWidths[j] << endl; diff --git a/dbcon/joblist/columncommand-jl.cpp b/dbcon/joblist/columncommand-jl.cpp index 66b3acb2e..63a57743c 100644 --- a/dbcon/joblist/columncommand-jl.cpp +++ b/dbcon/joblist/columncommand-jl.cpp @@ -270,7 +270,8 @@ uint8_t ColumnCommandJL::getTableColumnType() case 1: return TableColumn::UINT8; case 16: - cout << __FILE__<< ":" <<__LINE__ << " Fix 16 Bytes ?" << endl; + cout << __FILE__<< ":" <<__LINE__ << " Fix 16 Bytes ?" << endl; + //fallthrough default: throw logic_error("ColumnCommandJL: bad column width"); } diff --git a/dbcon/joblist/rowestimator.cpp b/dbcon/joblist/rowestimator.cpp index 95ae57140..78ef9c1bf 100644 --- a/dbcon/joblist/rowestimator.cpp +++ b/dbcon/joblist/rowestimator.cpp @@ -384,8 +384,8 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, bigValue = static_cast(val); break; } - /* fall through */ } + // fallthrough case 8: default: @@ -430,8 +430,8 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, bigValue = static_cast(val); break; } - /* fall through */ } + // fallthrough case 8: default: diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index a048d984e..72a0d747f 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -766,8 +766,8 @@ inline void store(const NewColRequestHeader* in, default: std::cout << __func__ << " WARNING!!! unspecified column width." << std::endl; - [[fallthrough]]; - + // fallthrough + case 8: ptr2 += (rid << 3); memcpy(ptr1, ptr2, 8); @@ -1840,7 +1840,7 @@ void PrimitiveProcessor::p_Col(NewColRequestHeader* in, NewColResultHeader* out, case 32: std::cout << __func__ << " WARNING!!! Not implemented for 32 byte data types." << std::endl; - [[fallthrough]]; + // fallthrough default: idbassert(0); diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 5b460e27d..fbeb69ab7 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -648,7 +648,7 @@ string Row::toString() const os << buf << " "; break; } - // fallthrough if the legacy DECIMAL + //fallthrough default: os << getIntField(i) << " "; break; @@ -712,6 +712,7 @@ string Row::toCSV() const } case CalpontSystemCatalog::BINARY: std::cout << __FILE__<< __LINE__ << ":" << "toCSV"<< std::endl; + //fallthrough default: os << getIntField(i); @@ -1789,6 +1790,7 @@ void RowGroup::addToSysDataList(execplan::CalpontSystemCatalog::NJLSysDataList& case CalpontSystemCatalog::BINARY: std::cout << __FILE__<< __LINE__ << __func__<< std::endl; + //fallthrough default: cr->PutData(row.getIntField<8>(j)); } diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index b7e7881dd..b68173850 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -492,7 +492,8 @@ bool WF_udaf::dropValues(int64_t b, int64_t e) break; } case CalpontSystemCatalog::BINARY: - cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + //fallthrough default: { string errStr = "(" + colType2String[(int)datum.dataType] + ")"; @@ -767,8 +768,9 @@ void WF_udaf::SetUDAFValue(static_any::any& valOut, int64_t colOut, setValue(colDataType, b, e, c, &strOut); } break; - case CalpontSystemCatalog::BINARY: - cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + case CalpontSystemCatalog::BINARY: + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + //fallthrough default: { std::ostringstream errmsg; @@ -1149,7 +1151,8 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) break; } case CalpontSystemCatalog::BINARY: - cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + //fallthrough default: { string errStr = "(" + colType2String[(int)datum.dataType] + ")"; diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index d93b59683..d5bb92047 100644 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -801,7 +801,8 @@ void* WindowFunctionType::getNullValueByType(int ct, int pos) v = &longDoubleNull; break; case CalpontSystemCatalog::BINARY: - cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << endl; + //fallthrough case CalpontSystemCatalog::VARBINARY: default: std::ostringstream oss; diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 56f8ebbc5..3eeed68fc 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -3006,7 +3006,6 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, break; } - [[fallthrough]]; } /* fall through */ @@ -3369,7 +3368,6 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, break; } - [[fallthrough]]; } /* fall through */ From f24fd413105a2fff033a95cf8ddea5be9bbdadc5 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Mon, 9 Nov 2020 07:53:42 +0000 Subject: [PATCH 71/78] Fix constexpr compilation issues in centos 7 --- datatypes/mcs_datatype.h | 4 ++-- datatypes/mcs_decimal.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datatypes/mcs_datatype.h b/datatypes/mcs_datatype.h index abd8a21b9..b699bd172 100644 --- a/datatypes/mcs_datatype.h +++ b/datatypes/mcs_datatype.h @@ -247,7 +247,7 @@ public: @brief The method detects whether decimal type is wide using csc colType. */ - constexpr inline bool isWideDecimalType() const + inline bool isWideDecimalType() const { return (colDataType == DECIMAL || colDataType == UDECIMAL) && @@ -262,7 +262,7 @@ public: @brief The method detects whether decimal type is wide using datatype and width. */ -static constexpr inline bool isWideDecimalType(const datatypes::SystemCatalog::ColDataType &dt, +static inline bool isWideDecimalType(const datatypes::SystemCatalog::ColDataType &dt, const int32_t width) { return width == MAXDECIMALWIDTH && diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index 3731f1cd6..74fd6935a 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -270,7 +270,7 @@ class Decimal @brief The method detects whether decimal type is wide using precision. */ - static constexpr inline bool isWideDecimalTypeByPrecision(const int32_t precision) + static inline bool isWideDecimalTypeByPrecision(const int32_t precision) { return precision > INT64MAXPRECISION && precision <= INT128MAXPRECISION; From c00daa93bd0ac4f12f8e2423f6ff4b512657a4a7 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Fri, 16 Oct 2020 12:13:00 +0000 Subject: [PATCH 72/78] MCOL-4172 MultiDistinctRowAggregation didn't honor multiple UDAF in projection ::doUDAF() doesn't crash anymore trying to access fRGContextColl[] elements that doesn't exist running RowAggregationMultiDistinct::doAggregate() --- utils/rowgroup/rowaggregation.cpp | 112 ++++++++++++++++++++---------- utils/rowgroup/rowaggregation.h | 45 ++++++++---- 2 files changed, 106 insertions(+), 51 deletions(-) diff --git a/utils/rowgroup/rowaggregation.cpp b/utils/rowgroup/rowaggregation.cpp index f51fcd022..05c30f6cb 100755 --- a/utils/rowgroup/rowaggregation.cpp +++ b/utils/rowgroup/rowaggregation.cpp @@ -772,6 +772,9 @@ void RowAggregation::initialize() // save the original output rowgroup data as primary row data fPrimaryRowData = fRowGroupOut->getRGData(); + // Lazy approach w/o a mapping b/w fFunctionCols idx and fRGContextColl idx + fRGContextColl.resize(fFunctionCols.size()); + // Need map only if groupby list is not empty. if (!fGroupByCols.empty()) { @@ -784,8 +787,6 @@ void RowAggregation::initialize() { fRowGroupOut->setRowCount(1); attachGroupConcatAg(); - // Lazy approach w/o a mapping b/w fFunctionCols idx and fRGContextColl idx - fRGContextColl.resize(fFunctionCols.size()); // For UDAF, reset the data for (uint64_t i = 0; i < fFunctionCols.size(); i++) { @@ -867,7 +868,8 @@ void RowAggregationUM::aggReset() } -void RowAggregationUM::aggregateRowWithRemap(Row& row) +void RowAggregationUM::aggregateRowWithRemap(Row& row, + std::vector* rgContextColl) { pair inserted; RowPosition pos(RowPosition::MSB, 0); @@ -925,20 +927,22 @@ void RowAggregationUM::aggregateRowWithRemap(Row& row) fResultDataVec[pos.group]->getRow(pos.row, &fRow); } - updateEntry(row); + updateEntry(row, rgContextColl); } -void RowAggregationUM::aggregateRow(Row& row) +void RowAggregationUM::aggregateRow(Row& row, + std::vector* rgContextColl) { if (UNLIKELY(fKeyOnHeap)) - aggregateRowWithRemap(row); + aggregateRowWithRemap(row, rgContextColl); else - RowAggregation::aggregateRow(row); + RowAggregation::aggregateRow(row, rgContextColl); } -void RowAggregation::aggregateRow(Row& row) +void RowAggregation::aggregateRow(Row& row, + std::vector* rgContextColl) { // groupby column list is not empty, find the entry. if (!fGroupByCols.empty()) @@ -1002,7 +1006,7 @@ void RowAggregation::aggregateRow(Row& row) } } - updateEntry(row); + updateEntry(row, rgContextColl); } @@ -1804,8 +1808,10 @@ void RowAggregation::deserialize(messageqcpp::ByteStream& bs) // NULL values are recognized and ignored for all agg functions except for // COUNT(*), which counts all rows regardless of value. // rowIn(in) - Row to be included in aggregation. +// rgContextColl(in) - ptr to a vector of UDAF contexts //------------------------------------------------------------------------------ -void RowAggregation::updateEntry(const Row& rowIn) +void RowAggregation::updateEntry(const Row& rowIn, + std::vector* rgContextColl) { for (uint64_t i = 0; i < fFunctionCols.size(); i++) { @@ -1862,7 +1868,7 @@ void RowAggregation::updateEntry(const Row& rowIn) case ROWAGG_UDAF: { - doUDAF(rowIn, colIn, colOut, colOut + 1, i); + doUDAF(rowIn, colIn, colOut, colOut + 1, i, rgContextColl); break; } @@ -2087,10 +2093,21 @@ void RowAggregation::doStatistics(const Row& rowIn, int64_t colIn, int64_t colOu fRow.setLongDoubleField(fRow.getLongDoubleField(colAux + 1) + valIn * valIn, colAux + 1); } -void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, - int64_t colAux, uint64_t& funcColsIdx) +void RowAggregation::doUDAF(const Row& rowIn, + int64_t colIn, + int64_t colOut, + int64_t colAux, + uint64_t& funcColsIdx, + std::vector* rgContextColl) { - uint32_t paramCount = fRGContextColl[funcColsIdx].getParameterCount(); + std::vector* udafContextsCollPtr = &fRGContextColl; + if (UNLIKELY(rgContextColl != nullptr)) + { + udafContextsCollPtr = rgContextColl; + } + + std::vector& udafContextsColl = *udafContextsCollPtr; + uint32_t paramCount = udafContextsColl[funcColsIdx].getParameterCount(); // doUDAF changes funcColsIdx to skip UDAF arguments so the real UDAF // column idx is the initial value of the funcColsIdx uint64_t origFuncColsIdx = funcColsIdx; @@ -2119,7 +2136,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, if ((cc && cc->type() == execplan::ConstantColumn::NULLDATA) || (!cc && isNull(&fRowGroupIn, rowIn, colIn) == true)) { - if (fRGContextColl[origFuncColsIdx].getRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS)) + if (udafContextsColl[origFuncColsIdx].getRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS)) { // When Ignore nulls, if there are multiple parameters and any // one of them is NULL, we ignore the entry. We need to increment @@ -2361,7 +2378,7 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, default: { std::ostringstream errmsg; - errmsg << "RowAggregation " << fRGContextColl[origFuncColsIdx].getName() << + errmsg << "RowAggregation " << udafContextsColl[origFuncColsIdx].getName() << ": No logic for data type: " << colDataType; throw logging::QueryDataExcept(errmsg.str(), logging::aggregateFuncErr); break; @@ -2387,19 +2404,19 @@ void RowAggregation::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, } // The intermediate values are stored in userData referenced by colAux. - fRGContextColl[origFuncColsIdx].setDataFlags(dataFlags); - fRGContextColl[origFuncColsIdx].setUserData(fRow.getUserData(colAux)); + udafContextsColl[origFuncColsIdx].setDataFlags(dataFlags); + udafContextsColl[origFuncColsIdx].setUserData(fRow.getUserData(colAux)); mcsv1sdk::mcsv1_UDAF::ReturnCode rc; - rc = fRGContextColl[origFuncColsIdx].getFunction()->nextValue(&fRGContextColl[origFuncColsIdx], + rc = udafContextsColl[origFuncColsIdx].getFunction()->nextValue(&udafContextsColl[origFuncColsIdx], valsIn); - fRGContextColl[origFuncColsIdx].setUserData(NULL); + udafContextsColl[origFuncColsIdx].setUserData(NULL); if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) { RowUDAFFunctionCol* rowUDAF = dynamic_cast(fFunctionCols[origFuncColsIdx].get()); rowUDAF->bInterrupted = true; - throw logging::QueryDataExcept(fRGContextColl[origFuncColsIdx].getErrorMessage(), + throw logging::QueryDataExcept(udafContextsColl[origFuncColsIdx].getErrorMessage(), logging::aggregateFuncErr); } } @@ -2630,8 +2647,10 @@ void RowAggregationUM::attachGroupConcatAg() // Update the aggregation totals in the internal hashmap for the specified row. // NULL values are recognized and ignored for all agg functions except for count // rowIn(in) - Row to be included in aggregation. +// rgContextColl(in) - ptr to a vector of UDAF contexts //------------------------------------------------------------------------------ -void RowAggregationUM::updateEntry(const Row& rowIn) +void RowAggregationUM::updateEntry(const Row& rowIn, + std::vector* rgContextColl) { for (uint64_t i = 0; i < fFunctionCols.size(); i++) { @@ -2699,7 +2718,7 @@ void RowAggregationUM::updateEntry(const Row& rowIn) case ROWAGG_UDAF: { - doUDAF(rowIn, colIn, colOut, colAux, i); + doUDAF(rowIn, colIn, colOut, colAux, i, rgContextColl); break; } @@ -3237,7 +3256,6 @@ void RowAggregationUM::calculateUDAFColumns() RowUDAFFunctionCol* rowUDAF = NULL; static_any::any valOut; - for (uint64_t i = 0; i < fFunctionCols.size(); i++) { if (fFunctionCols[i]->fAggFunction != ROWAGG_UDAF) @@ -4281,8 +4299,10 @@ RowAggregationUMP2::~RowAggregationUMP2() // Update the aggregation totals in the internal hashmap for the specified row. // NULL values are recognized and ignored for all agg functions except for count // rowIn(in) - Row to be included in aggregation. +// rgContextColl(in) - ptr to a vector of UDAF contexts //------------------------------------------------------------------------------ -void RowAggregationUMP2::updateEntry(const Row& rowIn) +void RowAggregationUMP2::updateEntry(const Row& rowIn, + std::vector* rgContextColl) { for (uint64_t i = 0; i < fFunctionCols.size(); i++) { @@ -4348,7 +4368,7 @@ void RowAggregationUMP2::updateEntry(const Row& rowIn) case ROWAGG_UDAF: { - doUDAF(rowIn, colIn, colOut, colAux, i); + doUDAF(rowIn, colIn, colOut, colAux, i, rgContextColl); break; } @@ -4560,11 +4580,23 @@ void RowAggregationUMP2::doBitOp(const Row& rowIn, int64_t colIn, int64_t colOut // colOut(in) - column in the output row group // colAux(in) - Where the UDAF userdata resides // rowUDAF(in) - pointer to the RowUDAFFunctionCol for this UDAF instance +// rgContextColl(in) - ptr to a vector that brings UDAF contextx in //------------------------------------------------------------------------------ -void RowAggregationUMP2::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, - int64_t colAux, uint64_t& funcColsIdx) +void RowAggregationUMP2::doUDAF(const Row& rowIn, + int64_t colIn, + int64_t colOut, + int64_t colAux, + uint64_t& funcColsIdx, + std::vector* rgContextColl) { static_any::any valOut; + std::vector* udafContextsCollPtr = &fRGContextColl; + if (UNLIKELY(rgContextColl != nullptr)) + { + udafContextsCollPtr = rgContextColl; + } + + std::vector& udafContextsColl = *udafContextsCollPtr; // Get the user data boost::shared_ptr userDataIn = rowIn.getUserData(colIn + 1); @@ -4577,7 +4609,7 @@ void RowAggregationUMP2::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, if (!userDataIn) { - if (fRGContextColl[funcColsIdx].getRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS)) + if (udafContextsColl[funcColsIdx].getRunFlag(mcsv1sdk::UDAF_IGNORE_NULLS)) { return; } @@ -4586,14 +4618,14 @@ void RowAggregationUMP2::doUDAF(const Row& rowIn, int64_t colIn, int64_t colOut, flags[0] |= mcsv1sdk::PARAM_IS_NULL; } - fRGContextColl[funcColsIdx].setDataFlags(flags); + udafContextsColl[funcColsIdx].setDataFlags(flags); // The intermediate values are stored in colAux. - fRGContextColl[funcColsIdx].setUserData(fRow.getUserData(colAux)); + udafContextsColl[funcColsIdx].setUserData(fRow.getUserData(colAux)); // Call the UDAF subEvaluate method mcsv1sdk::mcsv1_UDAF::ReturnCode rc; - rc = fRGContextColl[funcColsIdx].getFunction()->subEvaluate(&fRGContextColl[funcColsIdx], userDataIn.get()); + rc = udafContextsColl[funcColsIdx].getFunction()->subEvaluate(&udafContextsColl[funcColsIdx], userDataIn.get()); fRGContext.setUserData(NULL); if (rc == mcsv1sdk::mcsv1_UDAF::ERROR) @@ -4712,8 +4744,10 @@ void RowAggregationDistinct::doDistinctAggregation_rowVec(vector& // Update the aggregation totals in the internal hashmap for the specified row. // for non-DISTINCT columns works partially aggregated results // rowIn(in) - Row to be included in aggregation. +// rgContextColl(in) - ptr to a vector of UDAF contexts //------------------------------------------------------------------------------ -void RowAggregationDistinct::updateEntry(const Row& rowIn) +void RowAggregationDistinct::updateEntry(const Row& rowIn, + std::vector* rgContextColl) { for (uint64_t i = 0; i < fFunctionCols.size(); i++) { @@ -4795,7 +4829,7 @@ void RowAggregationDistinct::updateEntry(const Row& rowIn) case ROWAGG_UDAF: { - doUDAF(rowIn, colIn, colOut, colAux, i); + doUDAF(rowIn, colIn, colOut, colAux, i, rgContextColl); break; } @@ -5083,6 +5117,7 @@ void RowAggregationMultiDistinct::doDistinctAggregation() { fFunctionCols = fSubFunctions[i]; fRowGroupIn = fSubRowGroups[i]; + auto* rgContextColl = fSubAggregators[i]->rgContextColl(); Row rowIn; fRowGroupIn.initRow(&rowIn); @@ -5098,14 +5133,14 @@ void RowAggregationMultiDistinct::doDistinctAggregation() for (uint64_t j = 0; j < fRowGroupIn.getRowCount(); ++j, rowIn.nextRow()) { - aggregateRow(rowIn); + aggregateRow(rowIn, rgContextColl); } } } // restore the function column vector fFunctionCols = origFunctionCols; - fOrigFunctionCols = NULL; + fOrigFunctionCols = nullptr; } @@ -5120,20 +5155,21 @@ void RowAggregationMultiDistinct::doDistinctAggregation_rowVec(vectorrgContextColl(); Row rowIn; fRowGroupIn.initRow(&rowIn); for (uint64_t j = 0; j < inRows[i].size(); ++j) { rowIn.setData(inRows[i][j]); - aggregateRow(rowIn); + aggregateRow(rowIn, rgContextColl); } inRows[i].clear(); } // restore the function column vector fFunctionCols = origFunctionCols; - fOrigFunctionCols = NULL; + fOrigFunctionCols = nullptr; } diff --git a/utils/rowgroup/rowaggregation.h b/utils/rowgroup/rowaggregation.h index 34cd6fd03..d41cd87f8 100644 --- a/utils/rowgroup/rowaggregation.h +++ b/utils/rowgroup/rowaggregation.h @@ -605,7 +605,8 @@ public: return fResultDataVec; } - virtual void aggregateRow(Row& row); + virtual void aggregateRow(Row& row, + std::vector* rgContextColl = nullptr); inline uint32_t aggMapKeyLength() { return fAggMapKeyCount; @@ -619,19 +620,29 @@ public: { return fTimeZone; } + inline std::vector* rgContextColl() + { + return &fRGContextColl; + } protected: virtual void initialize(); virtual void initMapData(const Row& row); virtual void attachGroupConcatAg(); - virtual void updateEntry(const Row& row); + virtual void updateEntry(const Row& row, + std::vector* rgContextColl = nullptr); virtual void doMinMax(const Row&, int64_t, int64_t, int); virtual void doSum(const Row&, int64_t, int64_t, int); virtual void doAvg(const Row&, int64_t, int64_t, int64_t); virtual void doStatistics(const Row&, int64_t, int64_t, int64_t); virtual void doBitOp(const Row&, int64_t, int64_t, int); - virtual void doUDAF(const Row&, int64_t, int64_t, int64_t, uint64_t& funcColsIdx); + virtual void doUDAF(const Row&, + int64_t, + int64_t, + int64_t, + uint64_t& funcColsIdx, + std::vector* rgContextColl = nullptr); virtual bool countSpecial(const RowGroup* pRG) { fRow.setUintField<8>(fRow.getUintField<8>(0) + pRG->getRowCount(), 0); @@ -823,19 +834,21 @@ public: return fGroupConcat; } - void aggregateRow(Row&); - //void initialize(); + void aggregateRow(Row&, + std::vector* rgContextColl = nullptr) override; virtual void aggReset(); void setInputOutput(const RowGroup& pRowGroupIn, RowGroup* pRowGroupOut); protected: - // virtual methods from base - void initialize(); - void aggregateRowWithRemap(Row&); + void initialize() override; + void updateEntry(const Row& row, + std::vector* rgContextColl = nullptr) override; + + void aggregateRowWithRemap(Row&, + std::vector* rgContextColl = nullptr); void attachGroupConcatAg(); - void updateEntry(const Row& row); bool countSpecial(const RowGroup* pRG) { fRow.setIntField<8>( @@ -948,12 +961,18 @@ public: protected: // virtual methods from base - void updateEntry(const Row& row); + void updateEntry(const Row& row, + std::vector* rgContextColl = nullptr) override; void doAvg(const Row&, int64_t, int64_t, int64_t); void doStatistics(const Row&, int64_t, int64_t, int64_t); void doGroupConcat(const Row&, int64_t, int64_t); void doBitOp(const Row&, int64_t, int64_t, int); - void doUDAF(const Row&, int64_t, int64_t, int64_t, uint64_t& funcColsIdx); + void doUDAF(const Row&, + int64_t, + int64_t, + int64_t, + uint64_t& funcColsIdx, + std::vector* rgContextColl = nullptr) override; bool countSpecial(const RowGroup* pRG) { return false; @@ -1021,8 +1040,8 @@ public: } protected: - // virtual methods from base - void updateEntry(const Row& row); + void updateEntry(const Row& row, + std::vector* rgContextColl = nullptr) override; boost::shared_ptr fAggregator; RowGroup fRowGroupDist; From 3d7f5c6fd1dc476f842a9564ca91a522727098a1 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 4 Nov 2020 00:47:55 +0400 Subject: [PATCH 73/78] MCOL-4377 Split DataConvert::convertColumnData() --- datatypes/mcs_datatype.cpp | 370 ++++++++++- datatypes/mcs_datatype.h | 147 +++++ dbcon/execplan/calpontsystemcatalog.cpp | 17 +- dbcon/execplan/calpontsystemcatalog.h | 12 +- utils/dataconvert/dataconvert.cpp | 844 ++++++++++++------------ utils/dataconvert/dataconvert.h | 60 +- 6 files changed, 968 insertions(+), 482 deletions(-) diff --git a/datatypes/mcs_datatype.cpp b/datatypes/mcs_datatype.cpp index 82c87ade9..533a8518d 100644 --- a/datatypes/mcs_datatype.cpp +++ b/datatypes/mcs_datatype.cpp @@ -1045,13 +1045,12 @@ class SimpleConverter: public boost::any bool m_pushWarning; public: SimpleConverter(const SessionParam &sp, - SystemCatalog::ColDataType typeCode, + const TypeHandler *h, const SystemCatalog::TypeAttributesStd &attr, const char *str) - :boost::any(DataConvert::convertColumnData(typeCode, attr, - str, initPushWarning(), - sp.tzname(), - false, true, false)) + :boost::any(h->convertFromString(attr, + ConvertFromStringParam(sp.tzname(), true, false), + str, initPushWarning())) { } round_style_t roundStyle() const { @@ -1084,11 +1083,11 @@ class SimpleConverterSNumeric: public SimpleConverter { public: SimpleConverterSNumeric(const SessionParam &sp, - SystemCatalog::ColDataType typeCode, + const TypeHandler *h, const SystemCatalog::TypeAttributesStd &attr, const char *str, round_style_t &rf) - :SimpleConverter(sp, typeCode, attr, str) + :SimpleConverter(sp, h, attr, str) { rf = roundStyle(str); } @@ -1098,12 +1097,12 @@ public: template SimpleValue toSimpleValueSInt(const SessionParam &sp, - SystemCatalog::ColDataType typeCode, + const TypeHandler *h, const SystemCatalog::TypeAttributesStd &attr, const char *str, round_style_t & rf) { idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); - SimpleConverterSNumeric anyVal(sp, typeCode, attr, str, rf); + SimpleConverterSNumeric anyVal(sp, h, attr, str, rf); return SimpleValueSInt64(static_cast(boost::any_cast(anyVal))); } @@ -1113,7 +1112,7 @@ TypeHandlerSInt8::toSimpleValue(const SessionParam &sp, const SystemCatalog::TypeAttributesStd &attr, const char *str, round_style_t & rf) const { - return toSimpleValueSInt(sp, SystemCatalog::TINYINT, attr, str, rf); + return toSimpleValueSInt(sp, this, attr, str, rf); } @@ -1122,7 +1121,7 @@ TypeHandlerSInt16::toSimpleValue(const SessionParam &sp, const SystemCatalog::TypeAttributesStd &attr, const char *str, round_style_t & rf) const { - return toSimpleValueSInt(sp, SystemCatalog::SMALLINT, attr, str, rf); + return toSimpleValueSInt(sp, this, attr, str, rf); } @@ -1131,7 +1130,7 @@ TypeHandlerSInt24::toSimpleValue(const SessionParam &sp, const SystemCatalog::TypeAttributesStd &attr, const char *str, round_style_t & rf) const { - return toSimpleValueSInt(sp, SystemCatalog::MEDINT, attr, str, rf); + return toSimpleValueSInt(sp, this, attr, str, rf); } @@ -1140,7 +1139,7 @@ TypeHandlerSInt32::toSimpleValue(const SessionParam &sp, const SystemCatalog::TypeAttributesStd &attr, const char *str, round_style_t & rf) const { - return toSimpleValueSInt(sp, SystemCatalog::INT, attr, str, rf); + return toSimpleValueSInt(sp, this, attr, str, rf); } @@ -1149,18 +1148,18 @@ TypeHandlerSInt64::toSimpleValue(const SessionParam &sp, const SystemCatalog::TypeAttributesStd &attr, const char *str, round_style_t & rf) const { - return toSimpleValueSInt(sp, SystemCatalog::BIGINT, attr, str, rf); + return toSimpleValueSInt(sp, this, attr, str, rf); } template SimpleValue toSimpleValueUInt(const SessionParam &sp, - SystemCatalog::ColDataType typeCode, + const TypeHandler *h, const SystemCatalog::TypeAttributesStd &attr, const char *str) { idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); - SimpleConverter anyVal(sp, typeCode, attr, str); + SimpleConverter anyVal(sp, h, attr, str); return SimpleValueSInt64(static_cast(boost::any_cast(anyVal))); } @@ -1170,7 +1169,7 @@ TypeHandlerUInt8::toSimpleValue(const SessionParam &sp, const SystemCatalog::TypeAttributesStd &attr, const char *str, round_style_t & rf) const { - return toSimpleValueUInt(sp, SystemCatalog::UTINYINT, attr, str); + return toSimpleValueUInt(sp, this, attr, str); } @@ -1179,7 +1178,7 @@ TypeHandlerUInt16::toSimpleValue(const SessionParam &sp, const SystemCatalog::TypeAttributesStd &attr, const char *str, round_style_t & rf) const { - return toSimpleValueUInt(sp, SystemCatalog::USMALLINT, attr, str); + return toSimpleValueUInt(sp, this, attr, str); } @@ -1188,7 +1187,7 @@ TypeHandlerUInt24::toSimpleValue(const SessionParam &sp, const SystemCatalog::TypeAttributesStd &attr, const char *str, round_style_t & rf) const { - return toSimpleValueUInt(sp, SystemCatalog::UMEDINT, attr, str); + return toSimpleValueUInt(sp, this, attr, str); } @@ -1197,7 +1196,7 @@ TypeHandlerUInt32::toSimpleValue(const SessionParam &sp, const SystemCatalog::TypeAttributesStd &attr, const char *str, round_style_t & rf) const { - return toSimpleValueUInt(sp, SystemCatalog::UINT, attr, str); + return toSimpleValueUInt(sp, this, attr, str); } @@ -1206,7 +1205,7 @@ TypeHandlerUInt64::toSimpleValue(const SessionParam &sp, const SystemCatalog::TypeAttributesStd &attr, const char *str, round_style_t & rf) const { - return toSimpleValueUInt(sp, SystemCatalog::UBIGINT, attr, str); + return toSimpleValueUInt(sp, this, attr, str); } @@ -1216,7 +1215,7 @@ TypeHandlerDate::toSimpleValue(const SessionParam &sp, const char *str, round_style_t & rf) const { idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); - SimpleConverter anyVal(sp, SystemCatalog::DATE, attr, str); + SimpleConverter anyVal(sp, this, attr, str); return SimpleValueSInt64(static_cast(anyVal.to_uint32())); } @@ -1227,7 +1226,7 @@ TypeHandlerDatetime::toSimpleValue(const SessionParam &sp, const char *str, round_style_t & rf) const { idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); - SimpleConverter anyVal(sp, SystemCatalog::DATETIME, attr, str); + SimpleConverter anyVal(sp, this, attr, str); return SimpleValueSInt64(static_cast(anyVal.to_uint64())); } @@ -1238,7 +1237,7 @@ TypeHandlerTimestamp::toSimpleValue(const SessionParam &sp, const char *str, round_style_t & rf) const { idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); - SimpleConverter anyVal(sp, SystemCatalog::TIMESTAMP, attr, str); + SimpleConverter anyVal(sp, this, attr, str); return SimpleValueTimestamp(anyVal.to_uint64(), sp.tzname()); } @@ -1249,7 +1248,7 @@ TypeHandlerTime::toSimpleValue(const SessionParam &sp, const char *str, round_style_t & rf) const { idbassert(attr.colWidth <= SystemCatalog::EIGHT_BYTE); - SimpleConverter anyVal(sp, SystemCatalog::TIME, attr, str); + SimpleConverter anyVal(sp, this, attr, str); return SimpleValueSInt64(anyVal.to_sint64()); } @@ -1261,7 +1260,7 @@ TypeHandlerXDecimal::toSimpleValue(const SessionParam &sp, { if (attr.colWidth <= SystemCatalog::EIGHT_BYTE) { - SimpleConverterSNumeric anyVal(sp, code(), attr, str, rf); + SimpleConverterSNumeric anyVal(sp, this, attr, str, rf); int64_t v; if (attr.colWidth == SystemCatalog::ONE_BYTE) v = boost::any_cast(anyVal); @@ -1281,7 +1280,7 @@ TypeHandlerXDecimal::toSimpleValue(const SessionParam &sp, else { idbassert(attr.colWidth == datatypes::MAXDECIMALWIDTH); - SimpleConverterSNumeric anyVal(sp, code(), attr, str, rf); + SimpleConverterSNumeric anyVal(sp, this, attr, str, rf); return SimpleValueSInt128(anyVal.to_sint128()); } } @@ -1292,7 +1291,7 @@ TypeHandlerStr::toSimpleValue(const SessionParam &sp, const SystemCatalog::TypeAttributesStd &attr, const char *str, round_style_t & rf) const { - SimpleConverter anyVal(sp, code(), attr, str); + SimpleConverter anyVal(sp, this, attr, str); rf= anyVal.roundStyle(); string i = boost::any_cast(anyVal); // bug 1932, pad nulls up to the size of v @@ -1758,6 +1757,321 @@ TypeHandlerVarbinary::getNullValueForType(const SystemCatalog::TypeAttributesStd return value; } +/****************************************************************************/ + +boost::any +TypeHandlerBit::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToBit(colType, prm, data, pushWarning); +} + + +boost::any +TypeHandlerSInt8::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + int64_t val64; + dataconvert::number_int_value(data, SystemCatalog::TINYINT, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (char) val64; + return value; +} + + +boost::any +TypeHandlerSInt16::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + int64_t val64; + dataconvert::number_int_value(data, SystemCatalog::SMALLINT, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (short) val64; + return value; +} + + +boost::any +TypeHandlerSInt24::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + int64_t val64; + dataconvert::number_int_value(data, SystemCatalog::MEDINT, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (int) val64; + return value; +} + + +boost::any +TypeHandlerSInt32::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + int64_t val64; + dataconvert::number_int_value(data, SystemCatalog::INT, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (int) val64; + return value; +} + + +boost::any +TypeHandlerSInt64::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + int64_t val64; + dataconvert::number_int_value(data, SystemCatalog::BIGINT, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (long long) val64; + return value; +} + + +boost::any +TypeHandlerUInt8::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const + +{ + boost::any value = (uint8_t)dataconvert::number_uint_value(data, SystemCatalog::UTINYINT, colType, pushWarning, prm.noRoundup()); + return value; +} + + +boost::any +TypeHandlerUInt16::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + boost::any value = (uint16_t)dataconvert::number_uint_value(data, SystemCatalog::USMALLINT, colType, pushWarning, prm.noRoundup()); + return value; +} + + +boost::any +TypeHandlerUInt24::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + boost::any value = (uint32_t)dataconvert::number_uint_value(data, SystemCatalog::UMEDINT, colType, pushWarning, prm.noRoundup()); + return value; +} + + +boost::any +TypeHandlerUInt32::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + boost::any value = (uint32_t)dataconvert::number_uint_value(data, SystemCatalog::UINT, colType, pushWarning, prm.noRoundup()); + return value; +} + + +boost::any +TypeHandlerUInt64::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + boost::any value = (uint64_t)dataconvert::number_uint_value(data, SystemCatalog::UBIGINT, colType, pushWarning, prm.noRoundup()); + return value; +} + + +boost::any +TypeHandlerSFloat::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToFloat(SystemCatalog::FLOAT, data, pushWarning); +} + + +boost::any +TypeHandlerUFloat::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToFloat(SystemCatalog::UFLOAT, data, pushWarning); +} + + +boost::any +TypeHandlerSDouble::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToDouble(SystemCatalog::DOUBLE, data, pushWarning); +} + + +boost::any +TypeHandlerUDouble::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToDouble(SystemCatalog::UDOUBLE, data, pushWarning); +} + + +boost::any +TypeHandlerDate::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToDate(data, pushWarning); +} + + +boost::any +TypeHandlerDatetime::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToDatetime(data, pushWarning); +} + + +boost::any +TypeHandlerTime::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToTime(colType, data, pushWarning); +} + +boost::any +TypeHandlerTimestamp::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToTimestamp(prm, data, pushWarning); +} + + +boost::any +TypeHandlerChar::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToString(colType, data, pushWarning); +} + + +boost::any +TypeHandlerVarchar::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToString(colType, data, pushWarning); +} + + +boost::any +TypeHandlerText::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToString(colType, data, pushWarning); +} + + +boost::any +TypeHandlerClob::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + boost::any value = data; + return value; +} + + +boost::any +TypeHandlerBlob::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + boost::any value = data; + return value; +} + + +boost::any +TypeHandlerVarbinary::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + boost::any value = data; + return value; +} + + +boost::any +TypeHandlerSDecimal64::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToSDecimal(colType, prm, data, pushWarning); +} + + +boost::any +TypeHandlerUDecimal64::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToUDecimal(colType, prm, data, pushWarning); +} + + +boost::any +TypeHandlerSDecimal128::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToSDecimal(colType, prm, data, pushWarning); +} + + +boost::any +TypeHandlerUDecimal128::convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& data, + bool& pushWarning) const +{ + return dataconvert::DataConvert::StringToUDecimal(colType, prm, data, pushWarning); +} + + /****************************************************************************/ } // end of namespace datatypes diff --git a/datatypes/mcs_datatype.h b/datatypes/mcs_datatype.h index b699bd172..162fdad8e 100644 --- a/datatypes/mcs_datatype.h +++ b/datatypes/mcs_datatype.h @@ -427,6 +427,24 @@ public: }; +class ConvertFromStringParam +{ + const std::string& m_timeZone; + const bool m_noRoundup; + const bool m_isUpdate; +public: + ConvertFromStringParam(const std::string& timeZone, + bool noRoundup, bool isUpdate) + :m_timeZone(timeZone), + m_noRoundup(noRoundup), + m_isUpdate(isUpdate) + { } + const std::string& timeZone() const { return m_timeZone; } + bool noRoundup() const { return m_noRoundup; } + bool isUpdate() const { return m_isUpdate; } +}; + + class SimpleValue { int64_t m_sint64; @@ -855,6 +873,11 @@ public: } virtual boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const = 0; + + virtual boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const = 0; }; @@ -913,6 +936,10 @@ class TypeHandlerBit: public TypeHandler //TODO: How to communicate with write engine? return boost::any(); } + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -980,6 +1007,10 @@ public: const char *str, round_style_t & rf) const override; boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -1032,6 +1063,10 @@ public: const char *str, round_style_t & rf) const override; boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -1086,6 +1121,10 @@ class TypeHandlerSInt24: public TypeHandlerInt const char *str, round_style_t & rf) const override; boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -1140,6 +1179,10 @@ class TypeHandlerSInt32: public TypeHandlerInt const char *str, round_style_t & rf) const override; boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -1191,6 +1234,10 @@ class TypeHandlerSInt64: public TypeHandlerInt const char *str, round_style_t & rf) const override; boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -1269,6 +1316,10 @@ class TypeHandlerUInt8: public TypeHandlerInt } boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -1346,6 +1397,10 @@ class TypeHandlerUInt16: public TypeHandlerInt } boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -1426,6 +1481,10 @@ class TypeHandlerUInt24: public TypeHandlerInt } boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -1506,6 +1565,10 @@ class TypeHandlerUInt32: public TypeHandlerInt } boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -1583,6 +1646,10 @@ class TypeHandlerUInt64: public TypeHandlerInt } boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -1719,6 +1786,10 @@ public: { return part.isSuitableSInt64(startVal, rfMin, endVal, rfMax); } + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -1795,6 +1866,10 @@ public: { return part.isSuitableSInt64(startVal, rfMin, endVal, rfMax); } + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -1873,6 +1948,10 @@ public: { return part.isSuitableSInt128(startVal, rfMin, endVal, rfMax); } + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -1947,6 +2026,10 @@ public: { return part.isSuitableSInt128(startVal, rfMin, endVal, rfMax); } + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -2002,6 +2085,10 @@ class TypeHandlerSFloat: public TypeHandlerReal } boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -2032,6 +2119,10 @@ class TypeHandlerSDouble: public TypeHandlerReal } boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -2062,6 +2153,10 @@ class TypeHandlerUFloat: public TypeHandlerReal } boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -2092,6 +2187,10 @@ class TypeHandlerUDouble: public TypeHandlerReal } boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -2124,6 +2223,14 @@ class TypeHandlerSLongDouble: public TypeHandlerReal // QQ: DDLPackageProcessor::getNullValueForType() did not handle LONGDOUBLE return boost::any(); } + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override + { + throw logging::QueryDataExcept("convertColumnData: unknown column data type.", logging::dataTypeErr); + return boost::any(); + } }; @@ -2170,6 +2277,10 @@ class TypeHandlerDate: public TypeHandlerTemporal const char *str, round_style_t & rf) const override; boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -2200,6 +2311,10 @@ class TypeHandlerDatetime: public TypeHandlerTemporal const char *str, round_style_t & rf) const override; boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -2230,6 +2345,10 @@ class TypeHandlerTime: public TypeHandlerTemporal const char *str, round_style_t & rf) const override; boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -2260,6 +2379,10 @@ class TypeHandlerTimestamp: public TypeHandlerTemporal const char *str, round_style_t & rf) const override; boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -2334,6 +2457,10 @@ class TypeHandlerChar: public TypeHandlerStr int *state) const override; boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -2380,6 +2507,10 @@ class TypeHandlerVarchar: public TypeHandlerStr { return getNullValueForTypeVarcharText(attr); } + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -2403,6 +2534,10 @@ class TypeHandlerVarbinary: public TypeHandlerStr const SystemCatalog::TypeAttributesStd &attr) const override; boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -2432,6 +2567,10 @@ class TypeHandlerBlob: public TypeHandlerStr } boost::any getNullValueForType(const SystemCatalog::TypeAttributesStd &attr) const override; + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -2464,6 +2603,10 @@ class TypeHandlerText: public TypeHandlerStr { return getNullValueForTypeVarcharText(attr); } + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; @@ -2498,6 +2641,10 @@ class TypeHandlerClob: public TypeHandlerStr { return boost::any(); // QQ } + boost::any convertFromString(const SystemCatalog::TypeAttributesStd& colType, + const ConvertFromStringParam& prm, + const std::string& str, + bool& pushWarning) const override; }; diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index 133b5f76e..7ead90f9d 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -6146,16 +6146,23 @@ const string CalpontSystemCatalog::ColType::toString() const boost::any -CalpontSystemCatalog::ColType::convertColumnData(const std::string& dataOrig, - bool& bSaturate, +CalpontSystemCatalog::ColType::convertColumnData(const std::string& data, + bool& pushWarning, const std::string& timeZone, bool nulFlag, bool noRoundup, bool isUpdate) const { - return dataconvert::DataConvert::convertColumnData(colDataType, *this, - dataOrig, bSaturate, timeZone, - nulFlag, noRoundup, isUpdate); + pushWarning = false; + const datatypes::TypeHandler *h= typeHandler(); + if (!h) + throw QueryDataExcept("convertColumnData: unknown column data type.", dataTypeErr); + + if (nulFlag) + return h->getNullValueForType(*this); + + const datatypes::ConvertFromStringParam prm(timeZone, noRoundup, isUpdate); + return h->convertFromString(*this, prm, data, pushWarning); } diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index b40ca48aa..10ca7ddbe 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -243,7 +243,17 @@ public: b >> charsetNumber; } - boost::any convertColumnData(const std::string& dataOrig, + /** + * @brief convert a columns data, represnted as a string, + * to its native format + * @param data - the string representation + * @param [OUT] bSaturate - the value was truncated/adjusted + * @param timeZone - the time zone name, for TIMESTAMP conversion + * @param nullFlag - SQL NULL flag + * @param nRoundtrip + * @param isUpdate + */ + boost::any convertColumnData(const std::string& data, bool& bSaturate, const std::string& timeZone, bool nulFlag = false, diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 3651e990f..7c416c141 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1372,452 +1372,426 @@ void DataConvert::decimalToString(int128_t* dec, } } + boost::any -DataConvert::convertColumnData(cscDataType typeCode, - const datatypes::SystemCatalog::TypeAttributesStd& colType, - const std::string& dataOrig, bool& pushWarning, - const std::string& timeZone, bool nulFlag, - bool noRoundup, bool isUpdate) +DataConvert::StringToBit(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const datatypes::ConvertFromStringParam &prm, + const std::string& dataOrig, + bool& pushWarning) { - boost::any value; - int64_t val64; - // WIP - std::string data( dataOrig ); - pushWarning = false; + std::string data(dataOrig); + unsigned int x = data.find("("); - //if ( !data.empty() ) - if (!nulFlag) + if (x <= data.length()) { - switch (typeCode) + data.replace ( x, 1, " "); + } + + x = data.find(")"); + + if (x <= data.length()) + { + data.replace (x, 1, " "); + } + + int64_t tmp = 0; + + number_int_value (data, datatypes::SystemCatalog::BIT, colType, pushWarning, prm.noRoundup(), tmp); + + if (tmp) + { + bool bitvalue; + + if (from_string(bitvalue, data, std::dec )) { - case datatypes::SystemCatalog::BIT: - { - unsigned int x = data.find("("); - - if (x <= data.length()) - { - data.replace ( x, 1, " "); - } - - x = data.find(")"); - - if (x <= data.length()) - { - data.replace (x, 1, " "); - } - - int64_t tmp = 0; - - number_int_value (data, typeCode, colType, pushWarning, noRoundup, tmp); - - if (tmp) - { - bool bitvalue; - - if (from_string(bitvalue, data, std::dec )) - { - value = bitvalue; - } - else - { - throw QueryDataExcept("range, valid value or conversion error on BIT type.", formatErr); - } - } - } - break; - - case datatypes::SystemCatalog::TINYINT: - number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); - value = (char) val64; - break; - - case datatypes::SystemCatalog::SMALLINT: - number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); - value = (short) val64; - break; - - case datatypes::SystemCatalog::MEDINT: - case datatypes::SystemCatalog::INT: - number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); - value = (int) val64; - break; - - case datatypes::SystemCatalog::BIGINT: - number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); - value = (long long) val64; - break; - - case datatypes::SystemCatalog::DECIMAL: - if (LIKELY(colType.colWidth == 16)) - { - int128_t val128; - number_int_value(data, typeCode, colType, pushWarning, noRoundup, val128); - value = (int128_t) val128; - } - else if (colType.colWidth == 8) - { - number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); - value = (long long) val64; - } - else if (colType.colWidth == 4) - { - number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); - value = (int) val64; - } - else if (colType.colWidth == 2) - { - number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); - value = (short) val64; - } - else if (colType.colWidth == 1) - { - number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); - value = (char) val64; - } - //else if (colType.colWidth == 32) - // value = data; - - break; - - case datatypes::SystemCatalog::UDECIMAL: - - // UDECIMAL numbers may not be negative - if (LIKELY(colType.colWidth == 16)) - { - int128_t val128; - number_int_value(data, typeCode, colType, pushWarning, noRoundup, val128); - - if (val128 < 0 && - !datatypes::Decimal::isWideDecimalNullValue(val128) && - !datatypes::Decimal::isWideDecimalEmptyValue(val128)) - { - val128 = 0; - pushWarning = true; - } - - value = val128; - } - else if (colType.colWidth == 8) - { - number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); - long long ival = static_cast(val64); - - if (ival < 0 && - ival != static_cast(joblist::BIGINTEMPTYROW) && - ival != static_cast(joblist::BIGINTNULL)) - { - ival = 0; - pushWarning = true; - } - - value = ival; - } - else if (colType.colWidth == 4) - { - number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); - int ival = static_cast(val64); - - if (ival < 0 && - ival != static_cast(joblist::INTEMPTYROW) && - ival != static_cast(joblist::INTNULL)) - { - ival = 0; - pushWarning = true; - } - - value = ival; - } - else if (colType.colWidth == 2) - { - number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); - short ival = (short) val64; - - if (ival < 0 && - ival != static_cast(joblist::SMALLINTEMPTYROW) && - ival != static_cast(joblist::SMALLINTNULL)) - { - ival = 0; - pushWarning = true; - } - - value = ival; - } - else if (colType.colWidth == 1) - { - number_int_value(data, typeCode, colType, pushWarning, noRoundup, val64); - char ival = (char) val64; - - if (ival < 0 && - ival != static_cast(joblist::TINYINTEMPTYROW) && - ival != static_cast(joblist::TINYINTNULL)) - { - ival = 0; - pushWarning = true; - } - - value = ival; - } - - break; - - case datatypes::SystemCatalog::FLOAT: - case datatypes::SystemCatalog::UFLOAT: - { - string::size_type x = data.find('('); - - if (x < string::npos) - data.erase(x, 1); - - x = data.find(')'); - - if (x < string::npos) - data.erase(x, 1); - - if ( number_value ( data ) ) - { - float floatvalue; - errno = 0; -#ifdef _MSC_VER - double dval = strtod(data.c_str(), 0); - - if (dval > MAX_FLOAT) - { - pushWarning = true; - floatvalue = MAX_FLOAT; - } - else if (dval < MIN_FLOAT) - { - pushWarning = true; - floatvalue = MIN_FLOAT; - } - else - { - floatvalue = (float)dval; - } - -#else - floatvalue = strtof(data.c_str(), 0); -#endif - - if (errno == ERANGE) - { - pushWarning = true; -#ifdef _MSC_VER - - if ( abs(floatvalue) == HUGE_VAL ) -#else - if ( abs(floatvalue) == HUGE_VALF ) -#endif - { - if ( floatvalue > 0 ) - floatvalue = MAX_FLOAT; - else - floatvalue = MIN_FLOAT; - } - else - floatvalue = 0; - } - - if (floatvalue < 0.0 && - typeCode == datatypes::SystemCatalog::UFLOAT && - floatvalue != joblist::FLOATEMPTYROW && - floatvalue != joblist::FLOATNULL) - { - value = 0.0; - pushWarning = true; - } - - value = floatvalue; - } - else - throw QueryDataExcept("range, valid value or conversion error on FLOAT type.", formatErr); - } - break; - - case datatypes::SystemCatalog::DOUBLE: - case datatypes::SystemCatalog::UDOUBLE: - { - string::size_type x = data.find('('); - - if (x < string::npos) - data.erase(x, 1); - - x = data.find(')'); - - if (x < string::npos) - data.erase(x, 1); - - if ( number_value ( data ) ) - { - double doublevalue; - errno = 0; - doublevalue = strtod(data.c_str(), 0); - - if (errno == ERANGE) - { - pushWarning = true; -#ifdef _MSC_VER - - if ( abs(doublevalue) == HUGE_VAL ) -#else - if ( abs(doublevalue) == HUGE_VALL ) -#endif - { - if ( doublevalue > 0 ) - value = MAX_DOUBLE; - else - value = MIN_DOUBLE; - } - else - value = 0; - } - else - value = doublevalue; - - if (doublevalue < 0.0 && - typeCode == datatypes::SystemCatalog::UDOUBLE && - doublevalue != joblist::DOUBLEEMPTYROW && - doublevalue != joblist::DOUBLENULL) - { - doublevalue = 0.0; - pushWarning = true; - } - } - else - { - throw QueryDataExcept("range, valid value or conversion error on DOUBLE type.", formatErr); - } - } - break; - - case datatypes::SystemCatalog::UTINYINT: - value = (uint8_t)number_uint_value(data, typeCode, colType, pushWarning, noRoundup); - break; - - case datatypes::SystemCatalog::USMALLINT: - value = (uint16_t)number_uint_value(data, typeCode, colType, pushWarning, noRoundup); - break; - - case datatypes::SystemCatalog::UMEDINT: - case datatypes::SystemCatalog::UINT: - value = (uint32_t)number_uint_value(data, typeCode, colType, pushWarning, noRoundup); - break; - - case datatypes::SystemCatalog::UBIGINT: - value = (uint64_t)number_uint_value(data, typeCode, colType, pushWarning, noRoundup); - break; - - case datatypes::SystemCatalog::CHAR: - case datatypes::SystemCatalog::VARCHAR: - case datatypes::SystemCatalog::TEXT: - { - //check data length - if ( data.length() > (unsigned int)colType.colWidth ) - { - data = data.substr(0, colType.colWidth); - pushWarning = true; - } - else - { - if ( (unsigned int)colType.colWidth > data.length()) - { - //Pad null character to the string - data.resize(colType.colWidth, 0); - } - } - - value = data; - } - break; - - case datatypes::SystemCatalog::DATE: - { - Date aDay; - - if (stringToDateStruct(data, aDay)) - { - value = (*(reinterpret_cast (&aDay))); - } - else - { - value = (uint32_t) 0; - pushWarning = true; - } - } - break; - - case datatypes::SystemCatalog::DATETIME: - { - DateTime aDatetime; - - if (stringToDatetimeStruct(data, aDatetime, 0)) - { - value = *(reinterpret_cast(&aDatetime)); - } - else - { - value = (uint64_t) 0; - pushWarning = true; - } - } - break; - - case datatypes::SystemCatalog::TIME: - { - Time aTime; - - if (!stringToTimeStruct(data, aTime, colType.precision)) - { - pushWarning = true; - } - - value = (int64_t) * (reinterpret_cast(&aTime)); - } - break; - - case datatypes::SystemCatalog::TIMESTAMP: - { - TimeStamp aTimestamp; - - if (!stringToTimestampStruct(data, aTimestamp, timeZone)) - { - pushWarning = true; - } - - value = (uint64_t) *(reinterpret_cast(&aTimestamp)); - } - break; - - case datatypes::SystemCatalog::BLOB: - case datatypes::SystemCatalog::CLOB: - value = data; - break; - - case datatypes::SystemCatalog::VARBINARY: - value = data; - break; - - case datatypes::SystemCatalog::BINARY: - value = data; - break; - - default: - throw QueryDataExcept("convertColumnData: unknown column data type.", dataTypeErr); - break; + boost::any value = bitvalue; + return value; + } + else + { + throw QueryDataExcept("range, valid value or conversion error on BIT type.", formatErr); } } - else //null - { - const datatypes::TypeHandler *h= datatypes::TypeHandler::find(typeCode, colType); - if (!h) - throw QueryDataExcept("convertColumnData: unknown column data type.", dataTypeErr); - else - return h->getNullValueForType(colType); - } + return boost::any(); +} + +boost::any +DataConvert::StringToSDecimal(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const datatypes::ConvertFromStringParam &prm, + const std::string& data, bool& pushWarning) +{ + const cscDataType typeCode= datatypes::SystemCatalog::DECIMAL; + if (LIKELY(colType.colWidth == 16)) + { + int128_t val128; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val128); + boost::any value = (int128_t) val128; + return value; + } + else if (colType.colWidth == 8) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (long long) val64; + return value; + } + else if (colType.colWidth == 4) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (int) val64; + return value; + } + else if (colType.colWidth == 2) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (short) val64; + return value; + } + else if (colType.colWidth == 1) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + boost::any value = (char) val64; + return value; + } + //else if (colType.colWidth == 32) + // value = data; + return boost::any(); +} + + +boost::any +DataConvert::StringToUDecimal(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const datatypes::ConvertFromStringParam &prm, + const std::string& data, bool& pushWarning) +{ + const cscDataType typeCode= datatypes::SystemCatalog::UDECIMAL; + + // UDECIMAL numbers may not be negative + if (LIKELY(colType.colWidth == 16)) + { + int128_t val128; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val128); + + if (val128 < 0 && + !datatypes::Decimal::isWideDecimalNullValue(val128) && + !datatypes::Decimal::isWideDecimalEmptyValue(val128)) + { + val128 = 0; + pushWarning = true; + } + + boost::any value = val128; + return value; + } + else if (colType.colWidth == 8) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + long long ival = static_cast(val64); + + if (ival < 0 && + ival != static_cast(joblist::BIGINTEMPTYROW) && + ival != static_cast(joblist::BIGINTNULL)) + { + ival = 0; + pushWarning = true; + } + + boost::any value = ival; + return value; + } + else if (colType.colWidth == 4) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + int ival = static_cast(val64); + + if (ival < 0 && + ival != static_cast(joblist::INTEMPTYROW) && + ival != static_cast(joblist::INTNULL)) + { + ival = 0; + pushWarning = true; + } + + boost::any value = ival; + return value; + } + else if (colType.colWidth == 2) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + short ival = (short) val64; + + if (ival < 0 && + ival != static_cast(joblist::SMALLINTEMPTYROW) && + ival != static_cast(joblist::SMALLINTNULL)) + { + ival = 0; + pushWarning = true; + } + + boost::any value = ival; + return value; + } + else if (colType.colWidth == 1) + { + int64_t val64; + number_int_value(data, typeCode, colType, pushWarning, prm.noRoundup(), val64); + char ival = (char) val64; + + if (ival < 0 && + ival != static_cast(joblist::TINYINTEMPTYROW) && + ival != static_cast(joblist::TINYINTNULL)) + { + ival = 0; + pushWarning = true; + } + + boost::any value = ival; + return value; + } + return boost::any(); +} + + +boost::any +DataConvert::StringToFloat(cscDataType typeCode, + const std::string& dataOrig, + bool& pushWarning) +{ + boost::any value; + std::string data(dataOrig); + + string::size_type x = data.find('('); + + if (x < string::npos) + data.erase(x, 1); + + x = data.find(')'); + + if (x < string::npos) + data.erase(x, 1); + + if ( number_value ( data ) ) + { + float floatvalue; + errno = 0; +#ifdef _MSC_VER + double dval = strtod(data.c_str(), 0); + + if (dval > MAX_FLOAT) + { + pushWarning = true; + floatvalue = MAX_FLOAT; + } + else if (dval < MIN_FLOAT) + { + pushWarning = true; + floatvalue = MIN_FLOAT; + } + else + { + floatvalue = (float)dval; + } +#else + floatvalue = strtof(data.c_str(), 0); +#endif + + if (errno == ERANGE) + { + pushWarning = true; +#ifdef _MSC_VER + + if ( abs(floatvalue) == HUGE_VAL ) +#else + if ( abs(floatvalue) == HUGE_VALF ) +#endif + { + if ( floatvalue > 0 ) + floatvalue = MAX_FLOAT; + else + floatvalue = MIN_FLOAT; + } + else + floatvalue = 0; + } + + if (floatvalue < 0.0 && + typeCode == datatypes::SystemCatalog::UFLOAT && + floatvalue != joblist::FLOATEMPTYROW && + floatvalue != joblist::FLOATNULL) + { + value = 0.0; // QQ: should it assign floatvalue? + pushWarning = true; + } + + value = floatvalue; + } + else + throw QueryDataExcept("range, valid value or conversion error on FLOAT type.", formatErr); + return value; +} + + + +boost::any +DataConvert::StringToDouble(cscDataType typeCode, + const std::string& dataOrig, + bool& pushWarning) +{ + boost::any value; + std::string data(dataOrig); + + string::size_type x = data.find('('); + + if (x < string::npos) + data.erase(x, 1); + + x = data.find(')'); + + if (x < string::npos) + data.erase(x, 1); + + if ( number_value ( data ) ) + { + double doublevalue; + errno = 0; + doublevalue = strtod(data.c_str(), 0); + + if (errno == ERANGE) + { + pushWarning = true; +#ifdef _MSC_VER + + if ( abs(doublevalue) == HUGE_VAL ) +#else + if ( abs(doublevalue) == HUGE_VALL ) +#endif + { + if ( doublevalue > 0 ) + value = MAX_DOUBLE; + else + value = MIN_DOUBLE; + } + else + value = 0; + } + else + value = doublevalue; + + if (doublevalue < 0.0 && + typeCode == datatypes::SystemCatalog::UDOUBLE && + doublevalue != joblist::DOUBLEEMPTYROW && + doublevalue != joblist::DOUBLENULL) + { + doublevalue = 0.0; // QQ: should it assign "value" ? + pushWarning = true; + } + } + else + { + throw QueryDataExcept("range, valid value or conversion error on DOUBLE type.", formatErr); + } return value; } + +boost::any +DataConvert::StringToString(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const std::string& dataOrig, + bool& pushWarning) + +{ + std::string data(dataOrig); + //check data length + if ( data.length() > (unsigned int)colType.colWidth ) + { + data = data.substr(0, colType.colWidth); + pushWarning = true; + boost::any value = data; + return value; + } + if ( (unsigned int)colType.colWidth > data.length()) + { + //Pad null character to the string + data.resize(colType.colWidth, 0); + } + boost::any value = data; + return value; +} + + +boost::any +DataConvert::StringToDate(const std::string& data, bool& pushWarning) +{ + Date aDay; + + if (stringToDateStruct(data, aDay)) + { + boost::any value = (*(reinterpret_cast (&aDay))); + return value; + } + boost::any value = (uint32_t) 0; + pushWarning = true; + return value; +} + + +boost::any +DataConvert::StringToDatetime(const std::string& data, bool& pushWarning) +{ + DateTime aDatetime; + + if (stringToDatetimeStruct(data, aDatetime, 0)) // QQ: why 0? + { + boost::any value = *(reinterpret_cast(&aDatetime)); + return value; + } + boost::any value = (uint64_t) 0; + pushWarning = true; + return value; +} + + +boost::any +DataConvert::StringToTime(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const std::string& data, + bool& pushWarning) +{ + Time aTime; + + if (!stringToTimeStruct(data, aTime, colType.precision)) + { + pushWarning = true; + } + + boost::any value = (int64_t) * (reinterpret_cast(&aTime)); + return value; +} + + +boost::any +DataConvert::StringToTimestamp(const datatypes::ConvertFromStringParam &prm, + const std::string& data, + bool& pushWarning) +{ + TimeStamp aTimestamp; + + if (!stringToTimestampStruct(data, aTimestamp, prm.timeZone())) + { + pushWarning = true; + } + + boost::any value = (uint64_t) *(reinterpret_cast(&aTimestamp)); + return value; +} + + //------------------------------------------------------------------------------ // Convert date string to binary date. Used by BulkLoad. //------------------------------------------------------------------------------ diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 0b918fb8b..794544b77 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -884,25 +884,18 @@ void number_int_value(const std::string& data, bool noRoundup, T& intVal); +uint64_t number_uint_value(const string& data, + cscDataType typeCode, + const datatypes::SystemCatalog::TypeAttributesStd& ct, + bool& pushwarning, + bool noRoundup); + /** @brief DataConvert is a component for converting string data to Calpont format */ class DataConvert { public: - /** - * @brief convert a columns data, represnted as a string, to it's native - * format - * - * @param type the columns data type - * @param data the columns string representation of it's data - */ - EXPORT static boost::any convertColumnData( - cscDataType typecode, - const datatypes::SystemCatalog::TypeAttributesStd& attr, - const std::string& dataOrig, bool& bSaturate, const std::string& timeZone, - bool nulFlag = false, bool noRoundup = false, bool isUpdate = false); - /** * @brief convert a columns data from native format to a string * @@ -1082,6 +1075,47 @@ public: // bug4388, union type conversion EXPORT static void joinColTypeForUnion(datatypes::SystemCatalog::TypeHolderStd &unionedType, const datatypes::SystemCatalog::TypeHolderStd &type); + + static boost::any StringToBit(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const datatypes::ConvertFromStringParam &prm, + const std::string& dataOrig, + bool& pushWarning); + + static boost::any StringToSDecimal(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const datatypes::ConvertFromStringParam &prm, + const std::string& data, + bool& pushWarning); + + static boost::any StringToUDecimal(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const datatypes::ConvertFromStringParam &prm, + const std::string& data, + bool& pushWarning); + + static boost::any StringToFloat(cscDataType typeCode, + const std::string& dataOrig, + bool& pushWarning); + + static boost::any StringToDouble(cscDataType typeCode, + const std::string& dataOrig, + bool& pushWarning); + + static boost::any StringToString(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const std::string& dataOrig, + bool& pushWarning); + + static boost::any StringToDate(const std::string& data, + bool& pushWarning); + + static boost::any StringToDatetime(const std::string& data, + bool& pushWarning); + + static boost::any StringToTime(const datatypes::SystemCatalog::TypeAttributesStd& colType, + const std::string& data, + bool& pushWarning); + + static boost::any StringToTimestamp(const datatypes::ConvertFromStringParam &prm, + const std::string& data, + bool& pushWarning); }; inline void DataConvert::dateToString( int datevalue, char* buf, unsigned int buflen) From 6fd7916c56f4ee963023e3f8ee75686580ee8264 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Thu, 5 Nov 2020 18:09:10 -0500 Subject: [PATCH 74/78] MCOL-4188 Fix regression in a union subquery involving a numeric field. Since we now perform type promotion to wide decimals for aggregations involving numeric fields, we need to check for wide decimal in in and out ROWs and call the appropriate setter and getter functions. --- dbcon/joblist/tupleunion.cpp | 72 +++++++++++++++++++++++++------ utils/dataconvert/dataconvert.cpp | 28 ++++++++++-- 2 files changed, 83 insertions(+), 17 deletions(-) diff --git a/dbcon/joblist/tupleunion.cpp b/dbcon/joblist/tupleunion.cpp index 914bd850d..4344109f1 100644 --- a/dbcon/joblist/tupleunion.cpp +++ b/dbcon/joblist/tupleunion.cpp @@ -534,7 +534,11 @@ dec1: else val *= (uint64_t) pow((double) 10, (double) diff); - out->setIntField(val, i); + if (out->getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) + out->setInt128Field(val, i); + else + out->setIntField(val, i); + break; } @@ -661,7 +665,11 @@ dec2: else val *= (uint64_t) pow((double) 10, (double) diff); - out->setIntField(val, i); + if (out->getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) + out->setInt128Field(val, i); + else + out->setIntField(val, i); + break; } @@ -979,7 +987,7 @@ dec2: case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { -dec3: /* have to pick a scale to use for the double. using 5... */ +dec3: /* have to pick a scale to use for the double. using 5... */ uint32_t scale = 5; uint64_t ival = (uint64_t) (double) (val * pow((double) 10, (double) scale)); int diff = out->getScale(i) - scale; @@ -989,7 +997,11 @@ dec3: /* have to pick a scale to use for the double. using 5... */ else ival *= (uint64_t) pow((double) 10, (double) diff); - out->setIntField((int64_t) val, i); + if (out->getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) + out->setInt128Field(ival, i); + else + out->setIntField(ival, i); + break; } @@ -1066,7 +1078,11 @@ dec4: /* have to pick a scale to use for the double. using 5... */ else ival *= (uint64_t) pow((double) 10, (double) diff); - out->setIntField((int64_t) val, i); + if (out->getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) + out->setInt128Field(ival, i); + else + out->setIntField(ival, i); + break; } @@ -1083,8 +1099,19 @@ dec4: /* have to pick a scale to use for the double. using 5... */ case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - int64_t val = in.getIntField(i); - uint32_t scale = in.getScale(i); + int64_t val; + int128_t val128; + bool isInputWide = false; + + if (in.getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) + { + in.getInt128Field(i, val128); + isInputWide = true; + } + else + val = in.getIntField(i); + + uint32_t scale = in.getScale(i); switch (out->getColTypes()[i]) { @@ -1101,12 +1128,31 @@ dec4: /* have to pick a scale to use for the double. using 5... */ case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - if (out->getScale(i) == scale) - out->setIntField(val, i); - else if (out->getScale(i) > scale) - out->setIntField(IDB_pow[out->getScale(i) - scale]*val, i); - else // should not happen, the output's scale is the largest - throw logic_error("TupleUnion::normalize(): incorrect scale setting"); + if (out->getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) + { + if (out->getScale(i) == scale) + out->setInt128Field(isInputWide ? val128 : val, i); + else if (out->getScale(i) > scale) + { + int128_t divisor = 1; + datatypes::getScaleDivisor(divisor, out->getScale(i) - scale); + int128_t temp = isInputWide ? divisor*val128 : divisor*val; + out->setInt128Field(temp, i); + } + else // should not happen, the output's scale is the largest + throw logic_error("TupleUnion::normalize(): incorrect scale setting"); + } + // If output type is narrow decimal, input type + // has to be narrow decimal as well. + else + { + if (out->getScale(i) == scale) + out->setIntField(val, i); + else if (out->getScale(i) > scale) + out->setIntField(IDB_pow[out->getScale(i) - scale]*val, i); + else // should not happen, the output's scale is the largest + throw logic_error("TupleUnion::normalize(): incorrect scale setting"); + } break; } diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 7c416c141..12a766976 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -3160,6 +3160,9 @@ DataConvert::joinColTypeForUnion(datatypes::SystemCatalog::TypeHolderStd &unione unionedType.colDataType = datatypes::SystemCatalog::DECIMAL; } + if (type.precision > unionedType.precision) + unionedType.precision = type.precision; + unionedType.scale = (type.scale > unionedType.scale) ? type.scale : unionedType.scale; break; @@ -3390,7 +3393,6 @@ DataConvert::joinColTypeForUnion(datatypes::SystemCatalog::TypeHolderStd &unione case datatypes::SystemCatalog::MEDINT: case datatypes::SystemCatalog::INT: case datatypes::SystemCatalog::BIGINT: - case datatypes::SystemCatalog::DECIMAL: case datatypes::SystemCatalog::FLOAT: case datatypes::SystemCatalog::DOUBLE: case datatypes::SystemCatalog::UTINYINT: @@ -3398,7 +3400,6 @@ DataConvert::joinColTypeForUnion(datatypes::SystemCatalog::TypeHolderStd &unione case datatypes::SystemCatalog::UMEDINT: case datatypes::SystemCatalog::UINT: case datatypes::SystemCatalog::UBIGINT: - case datatypes::SystemCatalog::UDECIMAL: case datatypes::SystemCatalog::UFLOAT: case datatypes::SystemCatalog::UDOUBLE: unionedType.colDataType = datatypes::SystemCatalog::DOUBLE; @@ -3406,6 +3407,16 @@ DataConvert::joinColTypeForUnion(datatypes::SystemCatalog::TypeHolderStd &unione unionedType.colWidth = sizeof(double); break; + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::UDECIMAL: + if (unionedType.colWidth != datatypes::MAXDECIMALWIDTH) + { + unionedType.colDataType = datatypes::SystemCatalog::DOUBLE; + unionedType.scale = 0; + unionedType.colWidth = sizeof(double); + } + break; + default: break; } @@ -3446,7 +3457,6 @@ DataConvert::joinColTypeForUnion(datatypes::SystemCatalog::TypeHolderStd &unione case datatypes::SystemCatalog::MEDINT: case datatypes::SystemCatalog::INT: case datatypes::SystemCatalog::BIGINT: - case datatypes::SystemCatalog::DECIMAL: case datatypes::SystemCatalog::FLOAT: case datatypes::SystemCatalog::DOUBLE: case datatypes::SystemCatalog::UTINYINT: @@ -3454,7 +3464,6 @@ DataConvert::joinColTypeForUnion(datatypes::SystemCatalog::TypeHolderStd &unione case datatypes::SystemCatalog::UMEDINT: case datatypes::SystemCatalog::UINT: case datatypes::SystemCatalog::UBIGINT: - case datatypes::SystemCatalog::UDECIMAL: case datatypes::SystemCatalog::UFLOAT: case datatypes::SystemCatalog::UDOUBLE: case datatypes::SystemCatalog::LONGDOUBLE: @@ -3464,6 +3473,17 @@ DataConvert::joinColTypeForUnion(datatypes::SystemCatalog::TypeHolderStd &unione unionedType.precision = -1; break; + case datatypes::SystemCatalog::DECIMAL: + case datatypes::SystemCatalog::UDECIMAL: + if (unionedType.colWidth != datatypes::MAXDECIMALWIDTH) + { + unionedType.colDataType = datatypes::SystemCatalog::LONGDOUBLE; + unionedType.scale = (type.scale > unionedType.scale) ? type.scale : unionedType.scale; + unionedType.colWidth = sizeof(long double); + unionedType.precision = -1; + } + break; + default: break; } From 007b8a508233da84837da7022a2bf735c6eac797 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Mon, 9 Nov 2020 18:45:22 -0500 Subject: [PATCH 75/78] MCOL-4188 Fix regressions in CEIL()/CHAR() for narrow decimals. In addition, a regression in a WHERE clause with a WF field as the LHS and an addition operation on two WF fields on the RHS is also fixed. The issue was SimpleColumn::getDecimalVal() was setting precision = 19, with the value of one of the operands of the addition operation being set in VDecimal::value instead of VDecimal::s128Value. addSubtractExecute() in mcs_decimal.cpp makes the assumption that if precision > 18 and precision <= 38, we need to fetch the wide s128Value, not the narrow value field. So we are fixing the precision set in SimpleColumn::getDecimalVal(). --- dbcon/execplan/simplecolumn.h | 14 +++++------ dbcon/execplan/treenode.h | 3 +++ dbcon/execplan/windowfunctioncolumn.cpp | 33 ++++++++++++++++--------- utils/funcexp/func_ceil.cpp | 12 +++++++++ utils/funcexp/func_char.cpp | 18 ++------------ 5 files changed, 44 insertions(+), 36 deletions(-) diff --git a/dbcon/execplan/simplecolumn.h b/dbcon/execplan/simplecolumn.h index d6e60929b..ea29444a7 100644 --- a/dbcon/execplan/simplecolumn.h +++ b/dbcon/execplan/simplecolumn.h @@ -335,20 +335,18 @@ public: // the original decimal scale is stored in scale field, which is no use for double. if (fResultType.precision == -1) { - IDB_Decimal rv; if (fResultType.colDataType == CalpontSystemCatalog::DOUBLE) { - rv.scale = fResultType.scale; - rv.precision = 15; - rv.value = (int64_t)(TreeNode::getDoubleVal() * IDB_pow[rv.scale]); + IDB_Decimal rv( + (int64_t)(TreeNode::getDoubleVal() * IDB_pow[fResultType.scale]), + fResultType.scale, 15); return rv; } else if (fResultType.colDataType == CalpontSystemCatalog::LONGDOUBLE) { - IDB_Decimal rv; - rv.scale = fResultType.scale; - rv.precision = 19; - rv.value = (int64_t)(TreeNode::getLongDoubleVal() * IDB_pow[rv.scale]); + IDB_Decimal rv ( + (int64_t)(TreeNode::getLongDoubleVal() * IDB_pow[fResultType.scale]), + fResultType.scale, fResultType.precision); return rv; } } diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 6b2958bff..9eee7e30a 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -62,6 +62,9 @@ class IDB_Decimal: public datatypes::VDecimal public: using datatypes::VDecimal::VDecimal; + IDB_Decimal(int64_t val, int8_t s, uint8_t p, const int128_t &val128 = 0) : + VDecimal(val, s, p, val128) {} + inline void operator=(const datatypes::TSInt128& rhs) { value = 0; scale = 0; precision = 0; diff --git a/dbcon/execplan/windowfunctioncolumn.cpp b/dbcon/execplan/windowfunctioncolumn.cpp index ec6e32f16..f574dbf8b 100644 --- a/dbcon/execplan/windowfunctioncolumn.cpp +++ b/dbcon/execplan/windowfunctioncolumn.cpp @@ -640,8 +640,10 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull) isNull = true; else { - fResult.decimalVal.value = row.getIntField<1>(fInputIndex); - fResult.decimalVal.scale = (unsigned)fResultType.scale; + fResult.decimalVal = IDB_Decimal( + row.getIntField<1>(fInputIndex), + fResultType.scale, + fResultType.precision); } break; @@ -653,8 +655,10 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull) isNull = true; else { - fResult.decimalVal.value = row.getIntField<2>(fInputIndex); - fResult.decimalVal.scale = (unsigned)fResultType.scale; + fResult.decimalVal = IDB_Decimal( + row.getIntField<2>(fInputIndex), + fResultType.scale, + fResultType.precision); } break; @@ -666,8 +670,10 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull) isNull = true; else { - fResult.decimalVal.value = row.getIntField<4>(fInputIndex); - fResult.decimalVal.scale = (unsigned)fResultType.scale; + fResult.decimalVal = IDB_Decimal( + row.getIntField<4>(fInputIndex), + fResultType.scale, + fResultType.precision); } break; @@ -679,8 +685,10 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull) isNull = true; else { - fResult.decimalVal.value = row.getIntField<8>(fInputIndex); - fResult.decimalVal.scale = (unsigned)fResultType.scale; + fResult.decimalVal = IDB_Decimal( + row.getIntField<8>(fInputIndex), + fResultType.scale, + fResultType.precision); } break; @@ -688,13 +696,14 @@ void WindowFunctionColumn::evaluate(Row& row, bool& isNull) case 16: { - datatypes::TSInt128 dec(row.getBinaryField(fInputIndex)); - if (dec == datatypes::Decimal128Null) + int128_t val; + row.getInt128Field(fInputIndex, val); + + if (val == datatypes::Decimal128Null) isNull = true; else { - fResult.decimalVal = dec; - fResult.decimalVal.scale = (unsigned)fResultType.scale; + fResult.decimalVal = IDB_Decimal(0, fResultType.scale, fResultType.precision, val); } break; diff --git a/utils/funcexp/func_ceil.cpp b/utils/funcexp/func_ceil.cpp index 73f8a4624..d78fab53b 100644 --- a/utils/funcexp/func_ceil.cpp +++ b/utils/funcexp/func_ceil.cpp @@ -75,6 +75,12 @@ int64_t Func_ceil::getIntVal(Row& row, case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { + if (op_ct.scale == 0) + { + ret = parm[0]->data()->getIntVal(row, isNull); + break; + } + IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); if (isNull) @@ -222,6 +228,12 @@ uint64_t Func_ceil::getUintVal(Row& row, case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { + if (op_ct.scale == 0) + { + ret = parm[0]->data()->getIntVal(row, isNull); + break; + } + IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); if (isNull) diff --git a/utils/funcexp/func_char.cpp b/utils/funcexp/func_char.cpp index 70320c2b1..f76c36dea 100644 --- a/utils/funcexp/func_char.cpp +++ b/utils/funcexp/func_char.cpp @@ -134,7 +134,7 @@ string Func_char::getStrVal(Row& row, case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: { - IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); + IDB_Decimal d = rc->getDecimalVal(row, isNull); if (ct.colWidth == datatypes::MAXDECIMALWIDTH) { @@ -154,30 +154,16 @@ string Func_char::getStrVal(Row& row, tmpval++; value = datatypes::Decimal::getInt32FromWideDecimal(tmpval); - - // WIP MCOL-641 - /*if ( !getChar((int64_t)tmpval, buf) ) - { - isNull = true; - return ""; - }*/ } else { double dscale = d.scale; // get decimal and round up - int value = d.value / pow(10.0, dscale); + value = d.value / pow(10.0, dscale); int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1); if ( lefto > 4 ) value++; - - // WIP MCOL-641 - /*if ( !getChar((int64_t)value, buf) ) - { - isNull = true; - return ""; - }*/ } } break; From 58495d0d2ff282bd1e3206e5860bb4bfdb543f7c Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 10 Nov 2020 17:27:16 +0000 Subject: [PATCH 76/78] MCOL-4387 Convert dataconvert::decimalToString() into VDecimal and TSInt128 methods --- datatypes/mcs_datatype.cpp | 20 +- datatypes/mcs_datatype.h | 5 +- datatypes/mcs_decimal.cpp | 256 ++++++++++++--- datatypes/mcs_decimal.h | 47 ++- datatypes/mcs_int128.cpp | 96 ++++++ datatypes/mcs_int128.h | 48 ++- dbcon/execplan/simplecolumn_decimal.h | 6 +- dbcon/execplan/treenode.h | 6 +- dbcon/joblist/groupconcat.cpp | 12 +- dbcon/joblist/tupleunion.cpp | 28 +- dbcon/mysql/ha_mcs_datatype.h | 27 +- dbcon/mysql/is_columnstore_extents.cpp | 22 +- tests/dataconvert-tests.cpp | 248 -------------- tests/mcs_decimal-tests.cpp | 397 +++++++++++++++++------ tools/editem/editem.cpp | 27 +- utils/dataconvert/dataconvert.cpp | 169 ---------- utils/dataconvert/dataconvert.h | 99 ------ utils/funcexp/func_cast.cpp | 22 +- utils/funcexp/func_ceil.cpp | 4 +- utils/funcexp/func_floor.cpp | 4 +- utils/funcexp/func_math.cpp | 13 +- utils/funcexp/func_regexp.cpp | 15 +- utils/funcexp/func_round.cpp | 8 +- utils/funcexp/func_truncate.cpp | 8 +- utils/funcexp/funchelpers.h | 4 - utils/rowgroup/rowgroup.cpp | 11 +- writeengine/bulk/we_brmreporter.cpp | 12 +- writeengine/server/we_dmlcommandproc.cpp | 50 +-- writeengine/wrapper/writeengine.cpp | 7 +- 29 files changed, 793 insertions(+), 878 deletions(-) diff --git a/datatypes/mcs_datatype.cpp b/datatypes/mcs_datatype.cpp index 533a8518d..fdbf149f4 100644 --- a/datatypes/mcs_datatype.cpp +++ b/datatypes/mcs_datatype.cpp @@ -632,16 +632,20 @@ int TypeHandlerSLongDouble::storeValueToField(rowgroup::Row &row, int pos, int TypeHandlerXDecimal::storeValueToField64(rowgroup::Row &row, int pos, StoreField *f) const { - int64_t val = row.getIntField(pos); - return f->store_decimal64(val); + return f->store_decimal64(datatypes::VDecimal(row.getIntField(pos), + f->scale(), + f->precision())); } int TypeHandlerXDecimal::storeValueToField128(rowgroup::Row &row, int pos, StoreField *f) const { - int128_t* dec= row.getBinaryField(pos); - return f->store_decimal128(*dec); + int128_t* decPtr = row.getBinaryField(pos); + return f->store_decimal128(datatypes::VDecimal(0, + f->scale(), + f->precision(), + decPtr)); } @@ -780,12 +784,8 @@ TypeHandlerXDecimal::format128(const SimpleValue &v, const { idbassert(isValidXDecimal128(attr)); - ostringstream oss; - char buf[datatypes::Decimal::MAXLENGTH16BYTES]; - int128_t tmp= v.toSInt128(); - DataConvert::decimalToString(&tmp, (unsigned) attr.scale, buf, (uint8_t) sizeof(buf), code()); - oss << buf; - return oss.str(); + datatypes::VDecimal dec(0, attr.scale, attr.precision, v.toSInt128()); + return dec.toString(true); } diff --git a/datatypes/mcs_datatype.h b/datatypes/mcs_datatype.h index 162fdad8e..87c894da3 100644 --- a/datatypes/mcs_datatype.h +++ b/datatypes/mcs_datatype.h @@ -720,6 +720,7 @@ public: virtual ~StoreField() {} virtual int32_t colWidth() const = 0; virtual int32_t precision() const = 0; + virtual int32_t scale() const = 0; virtual int store_date(int64_t val) = 0; virtual int store_datetime(int64_t val) = 0; virtual int store_time(int64_t val) = 0; @@ -730,8 +731,8 @@ public: virtual int store_float(float val) = 0; virtual int store_double(double val) = 0; virtual int store_long_double(long double val) = 0; - virtual int store_decimal64(int64_t val) = 0; - virtual int store_decimal128(const int128_t &val) = 0; + virtual int store_decimal64(const datatypes::VDecimal& dec) = 0; + virtual int store_decimal128(const datatypes::VDecimal& dec) = 0; virtual int store_lob(const char *str, size_t length) = 0; }; diff --git a/datatypes/mcs_decimal.cpp b/datatypes/mcs_decimal.cpp index 9da2894c5..b2563251f 100644 --- a/datatypes/mcs_decimal.cpp +++ b/datatypes/mcs_decimal.cpp @@ -195,45 +195,6 @@ namespace datatypes } } - std::string Decimal::toString(VDecimal& value) - { - char buf[Decimal::MAXLENGTH16BYTES]; - if (value.s128Value == Decimal128Null) - { - return std::string("NULL"); - } - else if (value.s128Value == Decimal128Empty) - { - return std::string("EMPTY"); - } - dataconvert::DataConvert::decimalToString(&value.s128Value, - value.scale, buf, (uint8_t) sizeof(buf), - datatypes::SystemCatalog::DECIMAL); - return std::string(buf); - } - - std::string Decimal::toString(const VDecimal& value) - { - return toString(const_cast(value)); - } - - std::string Decimal::toString(const int128_t& value) - { - char buf[Decimal::MAXLENGTH16BYTES]; - if (value == Decimal128Null) - { - return std::string("NULL"); - } - else if (value == Decimal128Empty) - { - return std::string("EMPTY"); - } - int128_t& constLessValue = const_cast(value); - dataconvert::DataConvert::decimalToString(&constLessValue, - 0, buf, sizeof(buf), datatypes::SystemCatalog::DECIMAL); - return std::string(buf); - } - int Decimal::compare(const VDecimal& l, const VDecimal& r) { int128_t divisorL, divisorR; @@ -559,4 +520,221 @@ namespace datatypes } } + // Writes integer part of a Decimal using int128 argument provided + uint8_t VDecimal::writeIntPart(const int128_t& x, + char* buf, + const uint8_t buflen) const + { + char* p = buf; + int128_t intPart = x; + int128_t high = 0, mid = 0, low = 0; + uint64_t maxUint64divisor = 10000000000000000000ULL; + + // Assuming scale = [0, 56] + switch (scale / datatypes::maxPowOf10) + { + case 2: // scale = [38, 56] + intPart /= datatypes::mcs_pow_10[datatypes::maxPowOf10]; + intPart /= datatypes::mcs_pow_10[datatypes::maxPowOf10]; + low = intPart; + break; + case 1: // scale = [19, 37] + intPart /= datatypes::mcs_pow_10[datatypes::maxPowOf10]; + intPart /= datatypes::mcs_pow_10[scale % datatypes::maxPowOf10]; + low = intPart % maxUint64divisor; + mid = intPart / maxUint64divisor; + break; + case 0: // scale = [0, 18] + intPart /= datatypes::mcs_pow_10[scale % datatypes::maxPowOf10]; + low = intPart % maxUint64divisor; + intPart /= maxUint64divisor; + mid = intPart % maxUint64divisor; + high = intPart / maxUint64divisor; + break; + default: + throw logging::QueryDataExcept("VDecimal::writeIntPart() bad scale", + logging::formatErr); + } + + p += printPodParts(p, high, mid, low); + uint8_t written = p - buf; + if (buflen <= written) + { + throw logging::QueryDataExcept("VDecimal::writeIntPart() char buffer overflow.", + logging::formatErr); + } + + return written; + } + + uint8_t VDecimal::writeFractionalPart(const int128_t& x, + char* buf, + const uint8_t buflen) const + { + int128_t scaleDivisor = 1; + char* p = buf; + + switch (scale / datatypes::maxPowOf10) + { + case 2: + scaleDivisor *= datatypes::mcs_pow_10[datatypes::maxPowOf10]; + scaleDivisor *= datatypes::mcs_pow_10[datatypes::maxPowOf10]; + break; + case 1: + scaleDivisor *= datatypes::mcs_pow_10[datatypes::maxPowOf10]; + //fallthrough + case 0: + scaleDivisor *= datatypes::mcs_pow_10[scale % datatypes::maxPowOf10]; + } + + int128_t fractionalPart = x % scaleDivisor; + + // divide by the base until we have non-zero quotient + scaleDivisor /= 10; + + while (scaleDivisor > 1 && fractionalPart / scaleDivisor == 0) + { + *p++ = '0'; + scaleDivisor /= 10; + } + size_t written = p - buf;; + p += TSInt128::writeIntPart(fractionalPart, p, buflen - written); + return p - buf; + } + + // The method writes Decimal based on TSInt128 with scale provided. + // It first writes sign, then extracts integer part + // prints delimiter and then decimal part. + std::string VDecimal::toStringTSInt128WithScale() const + { + char buf[Decimal::MAXLENGTH16BYTES]; + uint8_t left = sizeof(buf); + char* p = buf; + int128_t tempValue = s128Value; + // sign + if (tempValue < static_cast(0)) + { + *p++ = '-'; + tempValue *= -1; + left--; + } + + // integer part + p += writeIntPart(tempValue, p, left); + + // decimal delimiter + *p++ = '.'; + // decimal part + left = sizeof(buf) - (p - buf); + p += writeFractionalPart(tempValue, p, left); + + *p = '\0'; + + uint8_t written = p - buf; + if (sizeof(buf) <= written) + { + throw logging::QueryDataExcept("VDecimal::toString() char buffer overflow.", + logging::formatErr); + } + return std::string(buf); + } + + std::string VDecimal::toStringTSInt64() const + { + char buf[Decimal::MAXLENGTH8BYTES]; + // Need 19 digits maxium to hold a sum result of 18 digits decimal column. + // We don't make a copy of value b/c we mutate its string + // representation. +#ifndef __LP64__ + snprintf(buf, sizeof(buf), "%lld", value); +#else + snprintf(buf, sizeof(buf), "%ld", value); +#endif + + //we want to move the last dt_scale chars right by one spot + // to insert the dp we want to move the trailing null as well, + // so it's really dt_scale+1 chars + size_t l1 = strlen(buf); + char* ptr = &buf[0]; + + if (value < 0) + { + ptr++; + idbassert(l1 >= 2); + l1--; + } + + //need to make sure we have enough leading zeros for this to work. + //at this point scale is always > 0 + size_t l2 = 1; + + if ((unsigned)scale > l1) + { + const char* zeros = "00000000000000000000"; //20 0's + size_t diff = 0; + + if (value != 0) + diff = scale - l1; //this will always be > 0 + else + diff = scale; + + memmove((ptr + diff), ptr, l1 + 1); //also move null + memcpy(ptr, zeros, diff); + + if (value != 0) + l1 = 0; + else + l1 = 1; + } + else if ((unsigned)scale == l1) + { + l1 = 0; + l2 = 2; + } + else + { + l1 -= scale; + } + + memmove((ptr + l1 + l2), (ptr + l1), scale + 1); //also move null + + if (l2 == 2) + *(ptr + l1++) = '0'; + + *(ptr + l1) = '.'; + return std::string(buf); + } + + // Dispatcher method for toString() implementations + std::string VDecimal::toString(bool hasTSInt128) const + { + // There must be no empty at this point though + if (isNull()) + { + return std::string("NULL"); + } + + if(LIKELY(hasTSInt128 || isTSInt128ByPrecision())) + { + if (scale) + { + return toStringTSInt128WithScale(); + } + return TSInt128::toString(); + } + // TSInt64 Decimal + if (scale) + { + return toStringTSInt64(); + } + return std::to_string(value); + } + + std::ostream& operator<<(std::ostream& os, const VDecimal& dec) + { + os << dec.toString(); + return os; + } + + } // end of namespace diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h index 74fd6935a..387c4f662 100644 --- a/datatypes/mcs_decimal.h +++ b/datatypes/mcs_decimal.h @@ -131,8 +131,8 @@ const int128_t mcs_pow_10_128[20] = }; constexpr uint32_t maxPowOf10 = sizeof(mcs_pow_10)/sizeof(mcs_pow_10[0])-1; -constexpr int128_t Decimal128Null = int128_t(0x8000000000000000LL) << 64; -constexpr int128_t Decimal128Empty = (int128_t(0x8000000000000000LL) << 64) + 1; +constexpr int128_t Decimal128Null = TSInt128::NullValue; +constexpr int128_t Decimal128Empty = TSInt128::EmptyValue; /** @@ -172,7 +172,7 @@ class Decimal Decimal() { }; ~Decimal() { }; - static constexpr uint8_t MAXLENGTH16BYTES = 42; + static constexpr uint8_t MAXLENGTH16BYTES = TSInt128::maxLength(); static constexpr uint8_t MAXLENGTH8BYTES = 23; static inline bool isWideDecimalNullValue(const int128_t& val) @@ -259,13 +259,6 @@ class Decimal const VDecimal& r, VDecimal& result); - /** - @brief Convenience methods to put decimal into a std::string. - */ - static std::string toString(VDecimal& value); - static std::string toString(const VDecimal& value); - static std::string toString(const int128_t& value); - /** @brief The method detects whether decimal type is wide using precision. @@ -558,6 +551,14 @@ class VDecimal: public TSInt128 precision(p) { } + VDecimal(int64_t unused, int8_t s, uint8_t p, const int128_t* val128Ptr) : + TSInt128(val128Ptr), + value(unused), + scale(s), + precision(p) + { } + + int decimalComp(const VDecimal& d) const { lldiv_t d1 = lldiv(value, static_cast(mcs_pow_10[scale])); @@ -819,9 +820,35 @@ class VDecimal: public TSInt128 } } + inline bool isTSInt128ByPrecision() const + { + return precision > INT64MAXPRECISION + && precision <= INT128MAXPRECISION; + } + // hasTSInt128 explicitly tells to print int128 out in cases + // where precision can't detect decimal type properly, e.g. + // DECIMAL(10)/DECIMAL(38) + std::string toString(bool hasTSInt128 = false) const; + friend std::ostream& operator<<(std::ostream& os, const VDecimal& dec); + int64_t value; int8_t scale; // 0~38 uint8_t precision; // 1~38 + + // STRICTLY for unit tests!!! + void setTSInt64Value(const int64_t x) { value = x; } + void setTSInt128Value(const int128_t& x) { s128Value = x; } + +private: + uint8_t writeIntPart(const int128_t& x, + char* buf, + const uint8_t buflen) const; + uint8_t writeFractionalPart(const int128_t& x, + char* buf, + const uint8_t buflen) const; + std::string toStringTSInt128WithScale() const; + std::string toStringTSInt64() const; + }; } //end of namespace diff --git a/datatypes/mcs_int128.cpp b/datatypes/mcs_int128.cpp index abaa9a27f..d94f5f1bb 100644 --- a/datatypes/mcs_int128.cpp +++ b/datatypes/mcs_int128.cpp @@ -17,7 +17,10 @@ MA 02110-1301, USA. */ +#include + #include "mcs_int128.h" +#include "exceptclasses.h" namespace datatypes { @@ -33,6 +36,99 @@ namespace datatypes return getLongDoubleFromFloat128(static_cast<__float128>(s128Value)); } + uint8_t TSInt128::printPodParts(char* buf, + const int128_t& high, + const int128_t& mid, + const int128_t& low) const + { + char* p = buf; + // pod[0] is low 8 bytes, pod[1] is high 8 bytes + const uint64_t* high_pod = reinterpret_cast(&high); + const uint64_t* mid_pod = reinterpret_cast(&mid); + const uint64_t* low_pod = reinterpret_cast(&low); + + if (high_pod[0] != 0) + { + p += sprintf(p, "%lu", high_pod[0]); + p += sprintf(p, "%019lu", mid_pod[0]); + p += sprintf(p, "%019lu", low_pod[0]); + } + else if (mid_pod[0] != 0) + { + p += sprintf(p, "%lu", mid_pod[0]); + p += sprintf(p, "%019lu", low_pod[0]); + } + else + { + p += sprintf(p, "%lu", low_pod[0]); + } + return p - buf; + } + + // This method writes unsigned integer representation of TSInt128 + uint8_t TSInt128::writeIntPart(const int128_t& x, + char* buf, + const uint8_t buflen) const + { + char* p = buf; + int128_t high = 0, mid = 0, low = 0; + uint64_t maxUint64divisor = 10000000000000000000ULL; + + low = x % maxUint64divisor; + int128_t value = x / maxUint64divisor; + mid = value % maxUint64divisor; + high = value / maxUint64divisor; + + p += printPodParts(p, high, mid, low); + uint8_t written = p - buf; + if (buflen <= written) + { + throw logging::QueryDataExcept("TSInt128::writeIntPart() char buffer overflow.", + logging::formatErr); + } + + return written; + } + + // conversion to std::string + std::string TSInt128::toString() const + { + if (isNull()) + { + return std::string("NULL"); + } + + if (isEmpty()) + { + return std::string("EMPTY"); + } + + int128_t tempValue = s128Value; + char buf[TSInt128::MAXLENGTH16BYTES]; + uint8_t left = sizeof(buf); + char* p = buf; + // sign + if (tempValue < static_cast(0)) + { + *p++ = '-'; + tempValue *= -1; + left--; + } + // integer part + // reduce the size by one to account for \0 + left--; + p += writeIntPart(tempValue, p, left); + *p = '\0'; + + return std::string(buf); + } + + std::ostream& operator<<(std::ostream& os, const TSInt128& x) + { + os << x.toString(); + return os; + } + // The method converts a wide decimal s128Value to an int64_t, // saturating the s128Value if necessary. inline int64_t TSInt128::getInt64FromWideDecimal() diff --git a/datatypes/mcs_int128.h b/datatypes/mcs_int128.h index ef8a3e262..1bab9171c 100644 --- a/datatypes/mcs_int128.h +++ b/datatypes/mcs_int128.h @@ -23,6 +23,7 @@ #include #include #include +#include // Inline asm has three argument lists: output, input and clobber list #if defined(__GNUC__) && (__GNUC___ > 7) @@ -116,18 +117,41 @@ static inline long double getLongDoubleFromFloat128(const __float128& value) class TSInt128 { public: - // A variety of ctors for aligned and unaligned arguments + static constexpr uint8_t MAXLENGTH16BYTES = 42; + static constexpr int128_t NullValue = int128_t(0x8000000000000000LL) << 64; + static constexpr int128_t EmptyValue = (int128_t(0x8000000000000000LL) << 64) + 1; + + + // A variety of ctors for aligned and unaligned arguments TSInt128(): s128Value(0) { } - // aligned argument + // aligned argument TSInt128(const int128_t& x) { s128Value = x; } - // unaligned argument + // unaligned argument TSInt128(const int128_t* x) { assignPtrPtr(&s128Value, x); } - // unaligned argument + // unaligned argument TSInt128(const unsigned char* x) { assignPtrPtr(&s128Value, x); } + // Method returns max length of a string representation + static constexpr uint8_t maxLength() + { + return TSInt128::MAXLENGTH16BYTES; + } + + // Checks if the value is NULL + inline bool isNull() const + { + return s128Value == NullValue; + } + + // Checks if the value is Empty + inline bool isEmpty() const + { + return s128Value == EmptyValue; + } + // The method copies 16 bytes from one memory cell // into another using memcpy or SIMD. // memcpy in gcc >= 7 is replaced with SIMD instructions @@ -158,6 +182,22 @@ class TSInt128 return s128Value == static_cast(x); } + // print int128_t parts represented as PODs + uint8_t printPodParts(char* buf, + const int128_t& high, + const int128_t& mid, + const int128_t& low) const; + + // writes integer part of dec into a buffer + uint8_t writeIntPart(const int128_t& x, + char* buf, + const uint8_t buflen) const; + + // string representation of TSInt128 + std::string toString() const; + + friend std::ostream& operator<<(std::ostream& os, const TSInt128& x); + // The method converts a wide decimal s128Value to a double. inline double getDoubleFromWideDecimal(); diff --git a/dbcon/execplan/simplecolumn_decimal.h b/dbcon/execplan/simplecolumn_decimal.h index a6fdac5e4..4e78607cf 100644 --- a/dbcon/execplan/simplecolumn_decimal.h +++ b/dbcon/execplan/simplecolumn_decimal.h @@ -153,8 +153,10 @@ void SimpleColumn_Decimal::setNullVal() template inline const std::string& SimpleColumn_Decimal:: getStrVal(rowgroup::Row& row, bool& isNull) { - dataconvert::DataConvert::decimalToString((int64_t)row.getIntField(fInputIndex), fResultType.scale, tmp, 22, fResultType.colDataType); - fResult.strVal = std::string(tmp); + datatypes::VDecimal dec((int64_t)row.getIntField(fInputIndex), + fResultType.scale, + fResultType.precision); + fResult.strVal = dec.toString(); return fResult.strVal; } diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 9eee7e30a..1b7fd316c 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -632,10 +632,10 @@ inline const std::string& TreeNode::getStrVal(const std::string& timeZone) case CalpontSystemCatalog::UDECIMAL: { if (fResultType.colWidth == datatypes::MAXDECIMALWIDTH) - dataconvert::DataConvert::decimalToString(&fResult.decimalVal.s128Value, fResult.decimalVal.scale, tmp, datatypes::Decimal::MAXLENGTH16BYTES, fResultType.colDataType); + // Explicit path for TSInt128 decimals with low precision + fResult.strVal = fResult.decimalVal.toString(true); else - dataconvert::DataConvert::decimalToString(fResult.decimalVal.value, fResult.decimalVal.scale, tmp, 22, fResultType.colDataType); - fResult.strVal = std::string(tmp); + fResult.strVal = fResult.decimalVal.toString(); break; } diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index b5ba3390f..a1d0a28ae 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -460,13 +460,11 @@ void GroupConcator::outputRow(std::ostringstream& oss, const rowgroup::Row& row) if (LIKELY(row.getColumnWidth(*i) == datatypes::MAXDECIMALWIDTH)) { - char buf[datatypes::Decimal::MAXLENGTH16BYTES]; - - int128_t* dec = row.getBinaryField(*i); - dataconvert::DataConvert::decimalToString(dec, - static_cast(scale), buf, - (uint8_t) sizeof(buf), types[*i]); - oss << fixed << buf; + datatypes::VDecimal dec(0, + scale, + row.getPrecision(*i), + row.getBinaryField(*i)); + oss << fixed << dec; } else { diff --git a/dbcon/joblist/tupleunion.cpp b/dbcon/joblist/tupleunion.cpp index 4344109f1..438dc8ef2 100644 --- a/dbcon/joblist/tupleunion.cpp +++ b/dbcon/joblist/tupleunion.cpp @@ -1099,8 +1099,8 @@ dec4: /* have to pick a scale to use for the double. using 5... */ case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - int64_t val; - int128_t val128; + int64_t val = 0; + int128_t val128 = 0; bool isInputWide = false; if (in.getColumnWidth(i) == datatypes::MAXDECIMALWIDTH) @@ -1185,15 +1185,21 @@ dec4: /* have to pick a scale to use for the double. using 5... */ case CalpontSystemCatalog::VARCHAR: default: { - char buf[50]; - dataconvert::DataConvert::decimalToString(val, scale, buf, 50, out->getColTypes()[i]); - /* ostringstream oss; - if (scale == 0) - oss << val; - else - oss << (val / IDB_pow[scale]) << "." - << setw(scale) << setfill('0') << (val % IDB_pow[scale]); */ - out->setStringField(string(buf), i); + if (LIKELY(isInputWide)) + { + datatypes::VDecimal dec(0, + in.getScale(i), + in.getPrecision(i), + val128); + out->setStringField(dec.toString(), i); + } + else + { + datatypes::VDecimal dec(val, + in.getScale(i), + in.getPrecision(i)); + out->setStringField(dec.toString(), i); + } break; } } diff --git a/dbcon/mysql/ha_mcs_datatype.h b/dbcon/mysql/ha_mcs_datatype.h index 56ffd7cba..0014d65a9 100644 --- a/dbcon/mysql/ha_mcs_datatype.h +++ b/dbcon/mysql/ha_mcs_datatype.h @@ -38,6 +38,7 @@ public: const CalpontSystemCatalog::ColType &type() const { return m_type; } int32_t colWidth() const override { return m_type.colWidth; } int32_t precision() const override { return m_type.precision; } + int32_t scale() const override { return m_type.scale; } int store_date(int64_t val) override { @@ -160,30 +161,16 @@ public: return m_field->store(static_cast(dl)); } - int store_decimal64(int64_t val) override + int store_decimal64(const datatypes::VDecimal& dec) override { - // @bug4388 stick to InfiniDB's scale in case mysql gives wrong scale due - // to create vtable limitation. - //if (f2->dec < m_type.scale) - // f2->dec = m_type.scale; - - // WIP MCOL-641 - // This is too much - char buf[256]; - dataconvert::DataConvert::decimalToString(val, (unsigned)m_type.scale, - buf, sizeof(buf), m_type.colDataType); - return m_field->store(buf, strlen(buf), m_field->charset()); + std::string decAsAStr = dec.toString(); + return m_field->store(decAsAStr.c_str(), decAsAStr.length(), m_field->charset()); } - int store_decimal128(const int128_t &val) override + int store_decimal128(const datatypes::VDecimal& dec) override { - // We won't have more than [+-][0][.] + up to 38 digits - char buf[datatypes::Decimal::MAXLENGTH16BYTES]; - dataconvert::DataConvert::decimalToString((int128_t*) &val, - (unsigned) m_type.scale, - buf, (uint8_t) sizeof(buf), - m_type.colDataType); - return m_field->store(buf, strlen(buf), m_field->charset()); + std::string decAsAStr = dec.toString(true); + return m_field->store(decAsAStr.c_str(), decAsAStr.length(), m_field->charset()); } int store_lob(const char *str, size_t length) override diff --git a/dbcon/mysql/is_columnstore_extents.cpp b/dbcon/mysql/is_columnstore_extents.cpp index 7e6f01165..f1ab1f3a7 100644 --- a/dbcon/mysql/is_columnstore_extents.cpp +++ b/dbcon/mysql/is_columnstore_extents.cpp @@ -112,12 +112,11 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM* emp, TABLE* table, THD* th else { table->field[4]->set_notnull(); - - char buf[datatypes::Decimal::MAXLENGTH16BYTES]; - dataconvert::DataConvert::decimalToString( - &iter->partition.cprange.bigLoVal, - 0, buf, (uint8_t) sizeof(buf), datatypes::SystemCatalog::DECIMAL); - table->field[4]->store(buf, strlen(buf), table->field[4]->charset()); + std::string decAsAStr = datatypes::TSInt128(iter->partition.cprange.bigLoVal) + .toString(); + table->field[4]->store(decAsAStr.c_str(), + decAsAStr.length(), + table->field[4]->charset()); } if (iter->partition.cprange.bigHiVal <= (utils::minInt128 + 1)) @@ -127,12 +126,11 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM* emp, TABLE* table, THD* th else { table->field[5]->set_notnull(); - - char buf[datatypes::Decimal::MAXLENGTH16BYTES]; - dataconvert::DataConvert::decimalToString( - &iter->partition.cprange.bigHiVal, - 0, buf, (uint8_t) sizeof(buf), datatypes::SystemCatalog::DECIMAL); - table->field[5]->store(buf, strlen(buf), table->field[5]->charset()); + std::string decAsAStr = datatypes::TSInt128(iter->partition.cprange.bigHiVal) + .toString(); + table->field[5]->store(decAsAStr.c_str(), + decAsAStr.length(), + table->field[5]->charset()); } } diff --git a/tests/dataconvert-tests.cpp b/tests/dataconvert-tests.cpp index 17c816c98..a6c7179b5 100644 --- a/tests/dataconvert-tests.cpp +++ b/tests/dataconvert-tests.cpp @@ -25,8 +25,6 @@ using namespace dataconvert; #include "joblisttypes.h" #include "columnwidth.h" -using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; - TEST(DataConvertTest, Strtoll128) { char *ep = NULL; @@ -543,252 +541,6 @@ TEST(DataConvertTest, NumberIntValue) EXPECT_EQ(b2, b4); EXPECT_TRUE(pushWarning); } - -TEST(DataConvertTest, DecimalToStringCheckScale0) -{ - datatypes::SystemCatalog::TypeHolderStd ct; - ct.colDataType = datatypes::SystemCatalog::DECIMAL; - char buf[42]; - string input, expected; - ct.precision = 38; - ct.scale = 0; - int128_t res; - - // test simple values - res = 0; - expected = "0"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = 2; - expected = "2"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = -2; - expected = "-2"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = 123; - expected = "123"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = -123; - expected = "-123"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - - // test max/min decimal (i.e. 38 9's) - res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - expected = "99999999999999999999999999999999999999"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = -res; - expected = "-99999999999999999999999999999999999999"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - - // test trailing zeros - res = 123000; - expected = "123000"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = -res; - expected = "-123000"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); -} -TEST(DataConvertTest, DecimalToStringCheckScale10) -{ - datatypes::SystemCatalog::TypeHolderStd ct; - ct.colDataType = datatypes::SystemCatalog::DECIMAL; - char buf[42]; - string input, expected; - ct.precision = 38; - ct.scale = 10; - int128_t res; - - // test simple values - res = 0; - expected = "0.0000000000"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = 2; - expected = "0.0000000002"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = -2; - expected = "-0.0000000002"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = 123; - expected = "0.0000000123"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = -123; - expected = "-0.0000000123"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = 12345678901; - expected = "1.2345678901"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = -12345678901; - expected = "-1.2345678901"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - - // test max/min decimal (i.e. 38 9's) - res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - expected = "9999999999999999999999999999.9999999999"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = -res; - expected = "-9999999999999999999999999999.9999999999"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - - // test trailing zeros - res = 123000; - expected = "0.0000123000"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = -res; - expected = "-0.0000123000"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - - // test leading zeros - res = 10000000009; - expected = "1.0000000009"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = -res; - expected = "-1.0000000009"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); -} - -TEST(DataConvertTest, DecimalToStringCheckScale38) -{ - datatypes::SystemCatalog::TypeHolderStd ct; - ct.colDataType = datatypes::SystemCatalog::DECIMAL; - char buf[42]; - string input, expected; - ct.precision = 38; - ct.scale = 38; - int128_t res; - - // test simple values - res = 0; - expected = "0.00000000000000000000000000000000000000"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = 2; - expected = "0.00000000000000000000000000000000000002"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = -2; - expected = "-0.00000000000000000000000000000000000002"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = 123; - expected = "0.00000000000000000000000000000000000123"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = -123; - expected = "-0.00000000000000000000000000000000000123"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = ((((((int128_t)1234567890 * 10000000000) + 1234567890) * 10000000000) + 1234567890) * 100000000 ) + 12345678; - expected = "0.12345678901234567890123456789012345678"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = -res; - expected = "-0.12345678901234567890123456789012345678"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - - // test max/min decimal (i.e. 38 9's) - res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - expected = "0.99999999999999999999999999999999999999"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = -res; - expected = "-0.99999999999999999999999999999999999999"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - - // test trailing zeros - res = 123000; - expected = "0.00000000000000000000000000000000123000"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = -res; - expected = "-0.00000000000000000000000000000000123000"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); -} - -TEST(DataConvertTest, DecimalToStringCheckScale37) -{ - datatypes::SystemCatalog::TypeHolderStd ct; - ct.colDataType = datatypes::SystemCatalog::DECIMAL; - char buf[42]; - string expected; - ct.precision = 38; - ct.scale = 37; - int128_t res; - - // test simple values - res = 0; - expected = "0.0000000000000000000000000000000000000"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = 2; - expected = "0.0000000000000000000000000000000000002"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = -2; - expected = "-0.0000000000000000000000000000000000002"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = 123; - expected = "0.0000000000000000000000000000000000123"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = -123; - expected = "-0.0000000000000000000000000000000000123"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = ((((((int128_t)1234567890 * 10000000000) + 1234567890) * 10000000000) + 1234567890) * 100000000 ) + 12345678; - expected = "1.2345678901234567890123456789012345678"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = -res; - expected = "-1.2345678901234567890123456789012345678"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - - // test max/min decimal (i.e. 38 9's) - res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; - expected = "9.9999999999999999999999999999999999999"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = -res; - expected = "-9.9999999999999999999999999999999999999"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - - // test trailing zeros - res = 123000; - expected = "0.0000000000000000000000000000000123000"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); - res = -res; - expected = "-0.0000000000000000000000000000000123000"; - DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); - EXPECT_EQ(string(buf), expected); -} - TEST(DataConvertTest, ConvertColumnData) { } diff --git a/tests/mcs_decimal-tests.cpp b/tests/mcs_decimal-tests.cpp index 762db7365..5d5b29189 100644 --- a/tests/mcs_decimal-tests.cpp +++ b/tests/mcs_decimal-tests.cpp @@ -84,8 +84,6 @@ TEST(Decimal, compareCheck) TEST(Decimal, additionNoOverflowCheck) { - datatypes::SystemCatalog::ColDataType colDataType = datatypes::SystemCatalog::DECIMAL; - char buf[42]; // Addition w/o overflow check execplan::IDB_Decimal l, r, result; // same precision, same scale, both positive values @@ -143,8 +141,7 @@ TEST(Decimal, additionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("0.00000000000042000000000000000000000042", std::string(buf)); + EXPECT_EQ("0.00000000000042000000000000000000000042", result.toString()); // same precision, L scale > R scale, both negative values l.s128Value = -42; @@ -152,8 +149,7 @@ TEST(Decimal, additionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("-0.00000000000042000000000000000000000042", std::string(buf)); + EXPECT_EQ("-0.00000000000042000000000000000000000042", result.toString()); // same precision, L scale > R scale, +- values l.s128Value = 42; @@ -161,8 +157,7 @@ TEST(Decimal, additionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("-0.00000000000041999999999999999999999958", std::string(buf)); + EXPECT_EQ("-0.00000000000041999999999999999999999958", result.toString()); // same precision, L scale > R scale, both 0 l.s128Value = 0; @@ -186,8 +181,7 @@ TEST(Decimal, additionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("0.00000000000004200000000000000000000420", std::string(buf)); + EXPECT_EQ("0.00000000000004200000000000000000000420", result.toString()); // same precision, L scale < R scale, both negative values l.s128Value = -42; @@ -195,8 +189,7 @@ TEST(Decimal, additionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("-0.00000000000004200000000000000000000420", std::string(buf)); + EXPECT_EQ("-0.00000000000004200000000000000000000420", result.toString()); // same precision, L scale < R scale, +- values l.s128Value = 42; @@ -204,8 +197,7 @@ TEST(Decimal, additionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::addition(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("0.00000000000004199999999999999999999580", std::string(buf)); + EXPECT_EQ("0.00000000000004199999999999999999999580", result.toString()); // same precision, L scale < R scale, both 0 l.s128Value = 0; @@ -220,8 +212,6 @@ TEST(Decimal, divisionNoOverflowCheck) { // DIVISION // same precision, same scale, both positive values - datatypes::SystemCatalog::ColDataType colDataType = datatypes::SystemCatalog::DECIMAL; - char buf[42]; execplan::IDB_Decimal l, r, result; l.scale = 38; @@ -237,8 +227,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("9.7674418605", std::string(buf)); + EXPECT_EQ("9.7674418605", result.toString()); // same precision, same scale, both negative values l.s128Value = -43; @@ -246,8 +235,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("9.7674418605", std::string(buf)); + EXPECT_EQ("9.7674418605", result.toString()); // same precision, same scale, +- values l.s128Value = 2200000; @@ -255,8 +243,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("-1157.8947368421", std::string(buf)); + EXPECT_EQ("-1157.8947368421", result.toString()); // same precision, same scale, l = 0 l.s128Value = 0; @@ -278,8 +265,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("115789473684210526315789.4736842105", std::string(buf)); + EXPECT_EQ("115789473684210526315789.4736842105", result.toString()); // same precision, L scale > R scale, both negative values l.s128Value = -22; @@ -287,8 +273,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("86363636363636363636363.6363636364", std::string(buf)); + EXPECT_EQ("86363636363636363636363.6363636364", result.toString()); // same precision, L scale > R scale, +- values l.s128Value = 19; @@ -296,8 +281,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("-115789473684210526315789.4736842105", std::string(buf)); + EXPECT_EQ("-115789473684210526315789.4736842105", result.toString()); // same precision, L scale > R scale, R = 0 l.s128Value = 424242; @@ -313,8 +297,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("100000000000000000000000.0000000000", std::string(buf)); + EXPECT_EQ("100000000000000000000000.0000000000", result.toString()); // same precision, L scale > R scale, both MIN negative values utils::int128Min(l.s128Value); @@ -322,8 +305,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("100000000000000000000000.0000000000", std::string(buf)); + EXPECT_EQ("100000000000000000000000.0000000000", result.toString()); // same precision, L scale < R scale, both positive values l.scale = 15; @@ -339,8 +321,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("0.00000000000000000000001000000000000000", std::string(buf)); + EXPECT_EQ("0.00000000000000000000001000000000000000", result.toString()); // same precision, L scale < R scale, both negative values l.s128Value = -22; @@ -348,8 +329,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("0.00000000000000000000000863636363636364", std::string(buf)); + EXPECT_EQ("0.00000000000000000000000863636363636364", result.toString()); // same precision, L scale < R scale, +- values l.s128Value = 22; @@ -357,8 +337,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("-0.00000000000000000000000863636363636364", std::string(buf)); + EXPECT_EQ("-0.00000000000000000000000863636363636364", result.toString()); // same precision, L scale < R scale, R = 0 l.s128Value = 42; @@ -375,8 +354,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("0.00000000000000000000001000000000000000", std::string(buf)); + EXPECT_EQ("0.00000000000000000000001000000000000000", result.toString()); // same precision, L scale < R scale, both MIN negative values utils::int128Min(l.s128Value); @@ -384,8 +362,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("0.00000000000000000000001000000000000000", std::string(buf)); + EXPECT_EQ("0.00000000000000000000001000000000000000", result.toString()); // same precision, L scale < R scale, result.scale < (r.scale-l.scale) // both positive values @@ -402,8 +379,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("1", std::string(buf)); + EXPECT_EQ("1", result.toString()); // same precision, L scale < R scale, result.scale < (r.scale-l.scale) // both negative values @@ -412,8 +388,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("9", std::string(buf)); + EXPECT_EQ("9", result.toString()); // same precision, L scale < R scale, result.scale < (r.scale-l.scale) // +- values @@ -422,8 +397,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("-9", std::string(buf)); + EXPECT_EQ("-9", result.toString()); // same precision, L scale < R scale, result.scale < (r.scale-l.scale) // R = 0 @@ -441,8 +415,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("0", std::string(buf)); + EXPECT_EQ("0", result.toString()); // same precision, L scale < R scale, result.scale < (r.scale-l.scale) // both MIN negative values @@ -451,8 +424,7 @@ TEST(Decimal, divisionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::division(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("0", std::string(buf)); + EXPECT_EQ("0", result.toString()); } void doDiv(const execplan::IDB_Decimal& l, @@ -464,8 +436,6 @@ void doDiv(const execplan::IDB_Decimal& l, TEST(Decimal, divisionWithOverflowCheck) { - datatypes::SystemCatalog::ColDataType colDataType = datatypes::SystemCatalog::DECIMAL; - char buf[42]; // Divide min int128 by -1 execplan::IDB_Decimal l, r, result; l.scale = 0; @@ -503,8 +473,7 @@ TEST(Decimal, divisionWithOverflowCheck) result.s128Value = 0; EXPECT_NO_THROW(doDiv(l, r, result)); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("9223372036854775809", std::string(buf)); + EXPECT_EQ("9223372036854775809", result.toString()); } void doAdd(const execplan::IDB_Decimal& l, @@ -516,8 +485,6 @@ void doAdd(const execplan::IDB_Decimal& l, TEST(Decimal, additionWithOverflowCheck) { - datatypes::SystemCatalog::ColDataType colDataType = datatypes::SystemCatalog::DECIMAL; - char buf[42]; // Add two max ints execplan::IDB_Decimal l, r, result; l.scale = 0; @@ -555,14 +522,11 @@ TEST(Decimal, additionWithOverflowCheck) result.s128Value = 0; EXPECT_NO_THROW(doAdd(l, r, result)); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("-170141183460469231713240559642174554113", std::string(buf)); + EXPECT_EQ("-170141183460469231713240559642174554113", result.toString()); } TEST(Decimal, subtractionNoOverflowCheck) { - datatypes::SystemCatalog::ColDataType colDataType = datatypes::SystemCatalog::DECIMAL; - char buf[42]; // Subtractio w/o overflow check execplan::IDB_Decimal l, r, result; // same precision, same scale, both positive values @@ -579,8 +543,7 @@ TEST(Decimal, subtractionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::subtraction(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("-0.00000000000000000000000000000000000378", std::string(buf)); + EXPECT_EQ("-0.00000000000000000000000000000000000378", result.toString()); // same precision, same scale, both negative values l.s128Value = -42; @@ -588,8 +551,7 @@ TEST(Decimal, subtractionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::subtraction(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("0.00000000000000000000000000000000000378", std::string(buf)); + EXPECT_EQ("0.00000000000000000000000000000000000378", result.toString()); // same precision, same scale, +- values l.s128Value = 42; @@ -597,8 +559,7 @@ TEST(Decimal, subtractionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::subtraction(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("0.00000000000000000000000000000000000462", std::string(buf)); + EXPECT_EQ("0.00000000000000000000000000000000000462", result.toString()); // same precision, same scale, both 0 l.s128Value = 0; @@ -623,8 +584,7 @@ TEST(Decimal, subtractionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::subtraction(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("-0.00000000000041999999999999999999999958", std::string(buf)); + EXPECT_EQ("-0.00000000000041999999999999999999999958", result.toString()); // same precision, L scale > R scale, both negative values l.s128Value = -42; @@ -632,8 +592,7 @@ TEST(Decimal, subtractionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::subtraction(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("0.00000000000041999999999999999999999958", std::string(buf)); + EXPECT_EQ("0.00000000000041999999999999999999999958", result.toString()); // same precision, L scale > R scale, +- values l.s128Value = 42; @@ -641,8 +600,7 @@ TEST(Decimal, subtractionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::subtraction(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("0.00000000000042000000000000000000000042", std::string(buf)); + EXPECT_EQ("0.00000000000042000000000000000000000042", result.toString()); // same precision, L scale > R scale, both 0 l.s128Value = 0; @@ -666,8 +624,7 @@ TEST(Decimal, subtractionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::subtraction(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("0.00000000000004199999999999999999999580", std::string(buf)); + EXPECT_EQ("0.00000000000004199999999999999999999580", result.toString()); // same precision, L scale < R scale, both negative values l.s128Value = -42; @@ -675,8 +632,7 @@ TEST(Decimal, subtractionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::subtraction(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("-0.00000000000004199999999999999999999580", std::string(buf)); + EXPECT_EQ("-0.00000000000004199999999999999999999580", result.toString()); // same precision, L scale < R scale, +- values l.s128Value = 42; @@ -684,8 +640,7 @@ TEST(Decimal, subtractionNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::subtraction(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("0.00000000000004200000000000000000000420", std::string(buf)); + EXPECT_EQ("0.00000000000004200000000000000000000420", result.toString()); // same precision, L scale < R scale, both 0 l.s128Value = 0; @@ -705,8 +660,6 @@ void doSubtract(const execplan::IDB_Decimal& l, TEST(Decimal, subtractionWithOverflowCheck) { - datatypes::SystemCatalog::ColDataType colDataType = datatypes::SystemCatalog::DECIMAL; - char buf[42]; // Subtract a max int from a min int execplan::IDB_Decimal l, r, result; l.scale = 0; @@ -744,16 +697,13 @@ TEST(Decimal, subtractionWithOverflowCheck) result.s128Value = 0; EXPECT_NO_THROW(doSubtract(l, r, result)); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("170141183460469231713240559642174554112", std::string(buf)); + EXPECT_EQ("170141183460469231713240559642174554112", result.toString()); } TEST(Decimal, multiplicationNoOverflowCheck) { // Multiplication // same precision, l.scale + r.scale = result.scale, both positive values - datatypes::SystemCatalog::ColDataType colDataType = datatypes::SystemCatalog::DECIMAL; - char buf[42]; execplan::IDB_Decimal l, r, result; l.scale = 19; @@ -769,8 +719,7 @@ TEST(Decimal, multiplicationNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::multiplication(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("0.85070591730234615861231965839514664960", std::string(buf)); + EXPECT_EQ("0.85070591730234615861231965839514664960", result.toString()); // same precision, l.scale + r.scale = result.scale, both negative values l.s128Value = -4611686018427387904; @@ -779,8 +728,7 @@ TEST(Decimal, multiplicationNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::multiplication(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("0.85070591730234615861231965839514664960", std::string(buf)); + EXPECT_EQ("0.85070591730234615861231965839514664960", result.toString()); // same precision, l.scale + r.scale = result.scale, +- values l.s128Value = -4611686018427387904; @@ -788,8 +736,7 @@ TEST(Decimal, multiplicationNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::multiplication(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("-0.85070591730234615861231965839514664960", std::string(buf)); + EXPECT_EQ("-0.85070591730234615861231965839514664960", result.toString()); // same precision, l.scale + r.scale = result.scale, l = 0 l.s128Value = 0; @@ -813,8 +760,7 @@ TEST(Decimal, multiplicationNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::multiplication(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("0.66461399789245793645190353014017228800", std::string(buf)); + EXPECT_EQ("0.66461399789245793645190353014017228800", result.toString()); // same precision, l.scale + r.scale < result.scale, both negative values l.s128Value = -72057594037927936; @@ -823,8 +769,7 @@ TEST(Decimal, multiplicationNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::multiplication(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("0.66461399789245793645190353014017228800", std::string(buf)); + EXPECT_EQ("0.66461399789245793645190353014017228800", result.toString()); // same precision, l.scale + r.scale < result.scale, +- values l.s128Value = -72057594037927936; @@ -832,8 +777,7 @@ TEST(Decimal, multiplicationNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::multiplication(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("-0.66461399789245793645190353014017228800", std::string(buf)); + EXPECT_EQ("-0.66461399789245793645190353014017228800", result.toString()); // same precision, l.scale + r.scale < result.scale, l = 0 l.s128Value = 0; @@ -859,8 +803,7 @@ TEST(Decimal, multiplicationNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::multiplication(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("0.01524157875323883675019051998750190521", std::string(buf)); + EXPECT_EQ("0.01524157875323883675019051998750190521", result.toString()); // same precision, l.scale + r.scale > result.scale, both negative values l.s128Value = -l.s128Value; @@ -868,16 +811,14 @@ TEST(Decimal, multiplicationNoOverflowCheck) result.s128Value = 0; datatypes::Decimal::multiplication(r, l, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("0.01524157875323883675019051998750190521", std::string(buf)); + EXPECT_EQ("0.01524157875323883675019051998750190521", result.toString()); // same precision, l.scale + r.scale > result.scale, +- values r.s128Value = -r.s128Value; result.s128Value = 0; datatypes::Decimal::multiplication(l, r, result); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("-0.01524157875323883675019051998750190521", std::string(buf)); + EXPECT_EQ("-0.01524157875323883675019051998750190521", result.toString()); // same precision, l.scale + r.scale > result.scale, l = 0 r.s128Value = 0; @@ -896,8 +837,6 @@ void doMultiply(const execplan::IDB_Decimal& l, TEST(Decimal, multiplicationWithOverflowCheck) { - datatypes::SystemCatalog::ColDataType colDataType = datatypes::SystemCatalog::DECIMAL; - char buf[42]; execplan::IDB_Decimal l, r, result; // result.scale >= l.scale + r.scale l.scale = 0; @@ -940,6 +879,248 @@ TEST(Decimal, multiplicationWithOverflowCheck) result.s128Value = 0; EXPECT_NO_THROW(doMultiply(l, r, result)); - dataconvert::DataConvert::decimalToString(&result.s128Value, result.scale, buf, 42, colDataType); - EXPECT_EQ("21267647932558653966460912964485513216", std::string(buf)); + EXPECT_EQ("21267647932558653966460912964485513216", result.toString()); } + +TEST(Decimal, DecimalToStringCheckScale0) +{ + string input, expected; + int128_t res; + int precision = 38; + int scale = 0; + res = 0; + datatypes::VDecimal dec(0, scale, precision, res); + + // test simple values + expected = "0"; + EXPECT_EQ(dec.toString(), expected); + res = 2; + expected = "2"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -2; + expected = "-2"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = 123; + expected = "123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -123; + expected = "-123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + // test max/min decimal (i.e. 38 9's) + res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + expected = "99999999999999999999999999999999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-99999999999999999999999999999999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + // test trailing zeros + res = 123000; + expected = "123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); +} +TEST(Decimal, DecimalToStringCheckScale10) +{ + string input, expected; + int128_t res; + int precision = 38; + int scale = 10; + res = 0; + datatypes::VDecimal dec(0, scale, precision, res); + + // test simple values + expected = "0.0000000000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + res = 2; + expected = "0.0000000002"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + res = -2; + expected = "-0.0000000002"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + res = 123; + expected = "0.0000000123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -123; + expected = "-0.0000000123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = 12345678901; + expected = "1.2345678901"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -12345678901; + expected = "-1.2345678901"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + // test max/min decimal (i.e. 38 9's) + res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + expected = "9999999999999999999999999999.9999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-9999999999999999999999999999.9999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + // test trailing zeros + res = 123000; + expected = "0.0000123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-0.0000123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + // test leading zeros + res = 10000000009; + expected = "1.0000000009"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-1.0000000009"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); +} + +TEST(Decimal, DecimalToStringCheckScale38) +{ + string input, expected; + int128_t res; + int precision = 38; + int scale = 38; + res = 0; + datatypes::VDecimal dec(0, scale, precision, res); + + // test simple values + res = 0; + expected = "0.00000000000000000000000000000000000000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = 2; + expected = "0.00000000000000000000000000000000000002"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -2; + expected = "-0.00000000000000000000000000000000000002"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = 123; + expected = "0.00000000000000000000000000000000000123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -123; + expected = "-0.00000000000000000000000000000000000123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = ((((((int128_t)1234567890 * 10000000000) + 1234567890) * 10000000000) + 1234567890) * 100000000 ) + 12345678; + expected = "0.12345678901234567890123456789012345678"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-0.12345678901234567890123456789012345678"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + // test max/min decimal (i.e. 38 9's) + res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + expected = "0.99999999999999999999999999999999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-0.99999999999999999999999999999999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + // test trailing zeros + res = 123000; + expected = "0.00000000000000000000000000000000123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-0.00000000000000000000000000000000123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); +} + +TEST(Decimal, DecimalToStringCheckScale37) +{ + string input, expected; + int128_t res; + int precision = 38; + int scale = 37; + res = 0; + datatypes::VDecimal dec(0, scale, precision, res); + + // test simple values + res = 0; + expected = "0.0000000000000000000000000000000000000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = 2; + expected = "0.0000000000000000000000000000000000002"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -2; + expected = "-0.0000000000000000000000000000000000002"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = 123; + expected = "0.0000000000000000000000000000000000123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -123; + expected = "-0.0000000000000000000000000000000000123"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = ((((((int128_t)1234567890 * 10000000000) + 1234567890) * 10000000000) + 1234567890) * 100000000 ) + 12345678; + expected = "1.2345678901234567890123456789012345678"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-1.2345678901234567890123456789012345678"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + // test max/min decimal (i.e. 38 9's) + res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + expected = "9.9999999999999999999999999999999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-9.9999999999999999999999999999999999999"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + + // test trailing zeros + res = 123000; + expected = "0.0000000000000000000000000000000123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); + res = -res; + expected = "-0.0000000000000000000000000000000123000"; + dec.setTSInt128Value(res); + EXPECT_EQ(dec.toString(), expected); +} + + diff --git a/tools/editem/editem.cpp b/tools/editem/editem.cpp index dd39875df..e106db90d 100644 --- a/tools/editem/editem.cpp +++ b/tools/editem/editem.cpp @@ -191,14 +191,7 @@ const string fmt(T v) } else { - char buf[datatypes::Decimal::MAXLENGTH16BYTES]; - - int128_t tmp = v; - - dataconvert::DataConvert::decimalToString( - &tmp, 0, buf, (uint8_t) sizeof(buf), datatypes::SystemCatalog::DECIMAL); - - oss << buf; + oss << datatypes::TSInt128(v); } } else if (uflg) @@ -218,14 +211,7 @@ const string fmt(T v) } else { - char buf[datatypes::Decimal::MAXLENGTH16BYTES]; - - int128_t tmp = static_cast(v); - - dataconvert::DataConvert::decimalToString( - &tmp, 0, buf, (uint8_t) sizeof(buf), datatypes::SystemCatalog::DECIMAL); - - oss << buf; + oss << datatypes::TSInt128(v); } } } @@ -247,14 +233,7 @@ const string fmt(T v) } else { - char buf[datatypes::Decimal::MAXLENGTH16BYTES]; - - int128_t tmp = v; - - dataconvert::DataConvert::decimalToString( - &tmp, 0, buf, (uint8_t) sizeof(buf), datatypes::SystemCatalog::DECIMAL); - - oss << buf; + oss << datatypes::TSInt128(v); } } } diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 12a766976..11c40c221 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1204,175 +1204,6 @@ bool stringToTimestampStruct(const string& data, TimeStamp& timeStamp, const str } -size_t DataConvert::writeIntPart(int128_t* dec, - char* p, - const unsigned int buflen, - const uint8_t scale) -{ - int128_t intPart = *dec; - int128_t high = 0, mid = 0, low = 0; - uint64_t maxUint64divisor = 10000000000000000000ULL; - - if (scale) - { - // Assuming scale = [0, 56] - switch (scale / datatypes::maxPowOf10) - { - case 2: // scale = [38, 56] - intPart /= datatypes::mcs_pow_10[datatypes::maxPowOf10]; - intPart /= datatypes::mcs_pow_10[datatypes::maxPowOf10]; - low = intPart; - break; - case 1: // scale = [19, 37] - intPart /= datatypes::mcs_pow_10[datatypes::maxPowOf10]; - intPart /= datatypes::mcs_pow_10[scale % datatypes::maxPowOf10]; - low = intPart % maxUint64divisor; - mid = intPart / maxUint64divisor; - break; - case 0: // scale = [0, 18] - intPart /= datatypes::mcs_pow_10[scale % datatypes::maxPowOf10]; - low = intPart % maxUint64divisor; - intPart /= maxUint64divisor; - mid = intPart % maxUint64divisor; - high = intPart / maxUint64divisor; - break; - default: - throw QueryDataExcept("writeIntPart() bad scale", formatErr); - } - } - else - { - low = intPart % maxUint64divisor; - intPart /= maxUint64divisor; - mid = intPart % maxUint64divisor; - high = intPart / maxUint64divisor; - } - - // pod[0] is low 8 bytes, pod[1] is high 8 bytes - uint64_t* high_pod = reinterpret_cast(&high); - uint64_t* mid_pod = reinterpret_cast(&mid); - uint64_t* low_pod = reinterpret_cast(&low); - char* original_p = p; - - // WIP replace sprintf with streams - if (high_pod[0] != 0) - { - p += sprintf(p, "%lu", high_pod[0]); - p += sprintf(p, "%019lu", mid_pod[0]); - p += sprintf(p, "%019lu", low_pod[0]); - } - else if (mid_pod[0] != 0) - { - p += sprintf(p, "%lu", mid_pod[0]); - p += sprintf(p, "%019lu", low_pod[0]); - } - else - { - p += sprintf(p, "%lu", low_pod[0]); - } - - size_t written = p - original_p; - - if (buflen <= written) - { - throw QueryDataExcept("writeIntPart() char buffer overflow.", formatErr); - } - - return written; -} - -size_t DataConvert::writeFractionalPart(int128_t* dec, - char* p, - const unsigned int buflen, - const uint8_t scale) -{ - int128_t scaleDivisor = 1; - - switch (scale / datatypes::maxPowOf10) - { - case 2: - scaleDivisor *= datatypes::mcs_pow_10[datatypes::maxPowOf10]; - scaleDivisor *= datatypes::mcs_pow_10[datatypes::maxPowOf10]; - break; - case 1: - scaleDivisor *= datatypes::mcs_pow_10[datatypes::maxPowOf10]; - case 0: - scaleDivisor *= datatypes::mcs_pow_10[scale % datatypes::maxPowOf10]; - } - - int128_t fractionalPart = *dec % scaleDivisor; - - // divide by the base until we have non-zero quotient - size_t written = 0; - scaleDivisor /= 10; - - char* original_p = p; - - while (scaleDivisor > 1 && fractionalPart / scaleDivisor == 0) - { - *p++ = '0'; - written++; - scaleDivisor /= 10; - } - - p += writeIntPart(&fractionalPart, p, buflen - written, 0); - - written = p - original_p; - - // this should never be true - if (written < scale) - { - for (size_t left = written; left < scale; left++) - { - *p++ = '0'; - } - } - - return written; -} - -void DataConvert::decimalToString(int128_t* dec, - const uint8_t scale, - char *p, - const unsigned int buflen, - cscDataType colDataType) // colDataType is redundant -{ - char* original_p = p; - size_t written = 0; - // Raise exception on NULL and EMPTY value - if (datatypes::Decimal::isWideDecimalNullValue(*dec) || - datatypes::Decimal::isWideDecimalEmptyValue(*dec)) - { - throw QueryDataExcept("toString() char buffer overflow.", formatErr); - } - - if (*dec < static_cast(0)) - { - *p++ = '-'; - *dec *= -1; - } - - written = writeIntPart(dec, p, buflen, scale); - p += written; - - if (scale) - { - *p++ = '.'; - written = p - original_p; - p += writeFractionalPart(dec, p, buflen - written, scale); - } - - *p = '\0'; - - written = p - original_p; - - if (buflen <= written) - { - throw QueryDataExcept("toString() char buffer overflow.", formatErr); - } -} - - boost::any DataConvert::StringToBit(const datatypes::SystemCatalog::TypeAttributesStd& colType, const datatypes::ConvertFromStringParam &prm, diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 794544b77..5cbd28929 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -1037,15 +1037,6 @@ public: EXPORT static bool isColumnTimeValid( int64_t time ); EXPORT static bool isColumnTimeStampValid( int64_t timeStamp ); - static inline std::string decimalToString(int64_t value, uint8_t scale, cscDataType colDataType); - static inline void decimalToString(int64_t value, uint8_t scale, char* buf, unsigned int buflen, cscDataType colDataType); - - static void decimalToString(int128_t* dec, const uint8_t scale, char* buf, const unsigned int buflen, cscDataType colDataType); - static size_t writeIntPart(int128_t* dec, char* p, const unsigned int buflen, - const uint8_t scale); - static size_t writeFractionalPart(int128_t* dec, char* p, const unsigned int buflen, - const uint8_t scale); - static inline std::string constructRegexp(const std::string& str); static inline void trimWhitespace(int64_t& charData); static inline bool isEscapedChar(char c) @@ -1298,96 +1289,6 @@ inline void DataConvert::timeToString1( long long timevalue, char* buf, unsigned #endif } -inline std::string DataConvert::decimalToString(int64_t value, uint8_t scale, cscDataType colDataType) -{ - // This is too much - char buf[80]; - DataConvert::decimalToString(value, scale, buf, 80, colDataType); - return std::string(buf); -} - -inline void DataConvert::decimalToString(int64_t int_val, uint8_t scale, - char* buf, unsigned int buflen, cscDataType colDataType) -{ - // Need to convert a string with a binary unsigned number in it to a 64-bit signed int - - // MySQL seems to round off values unless we use the string store method. Groan. - // Taken from ha_mcs_impl.cpp - - //biggest Calpont supports is DECIMAL(18,x), or 18 total digits+dp+sign for column - // Need 19 digits maxium to hold a sum result of 18 digits decimal column. - if (isUnsigned(colDataType)) - { -#ifndef __LP64__ - snprintf(buf, buflen, "%llu", static_cast(int_val)); -#else - snprintf(buf, buflen, "%lu", static_cast(int_val)); -#endif - } - else - { -#ifndef __LP64__ - snprintf(buf, buflen, "%lld", int_val); -#else - snprintf(buf, buflen, "%ld", int_val); -#endif - } - - if (scale == 0) - return; - - //we want to move the last dt_scale chars right by one spot to insert the dp - //we want to move the trailing null as well, so it's really dt_scale+1 chars - size_t l1 = strlen(buf); - char* ptr = &buf[0]; - - if (int_val < 0) - { - ptr++; - idbassert(l1 >= 2); - l1--; - } - - //need to make sure we have enough leading zeros for this to work... - //at this point scale is always > 0 - size_t l2 = 1; - - if ((unsigned)scale > l1) - { - const char* zeros = "00000000000000000000"; //20 0's - size_t diff = 0; - - if (int_val != 0) - diff = scale - l1; //this will always be > 0 - else - diff = scale; - - memmove((ptr + diff), ptr, l1 + 1); //also move null - memcpy(ptr, zeros, diff); - - if (int_val != 0) - l1 = 0; - else - l1 = 1; - } - else if ((unsigned)scale == l1) - { - l1 = 0; - l2 = 2; - } - else - { - l1 -= scale; - } - - memmove((ptr + l1 + l2), (ptr + l1), scale + 1); //also move null - - if (l2 == 2) - *(ptr + l1++) = '0'; - - *(ptr + l1) = '.'; -} - inline void DataConvert::trimWhitespace(int64_t& charData) { // Trims whitespace characters off non-dict character data diff --git a/utils/funcexp/func_cast.cpp b/utils/funcexp/func_cast.cpp index a4c9fbe49..aa0212a48 100644 --- a/utils/funcexp/func_cast.cpp +++ b/utils/funcexp/func_cast.cpp @@ -542,15 +542,10 @@ string Func_cast_char::getStrVal(Row& row, { IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull); - char buf[80]; - if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) - dataconvert::DataConvert::decimalToString( &d.s128Value, d.scale, buf, 80, parm[0]->data()->resultType().colDataType); + return d.toString(true).substr(0, length); else - dataconvert::DataConvert::decimalToString( d.value, d.scale, buf, 80, parm[0]->data()->resultType().colDataType); - - string sbuf = buf; - return sbuf.substr(0, length); + return d.toString().substr(0, length); } break; @@ -1231,17 +1226,10 @@ string Func_cast_decimal::getStrVal(Row& row, parm, isNull, operationColType); - - char buf[80]; - - if (decimal.precision > datatypes::INT64MAXPRECISION) - dataconvert::DataConvert::decimalToString( &decimal.s128Value, decimal.scale, buf, 80, operationColType.colDataType); + if (operationColType.colWidth == datatypes::MAXDECIMALWIDTH) + return decimal.toString(true); else - dataconvert::DataConvert::decimalToString( decimal.value, decimal.scale, buf, 80, operationColType.colDataType); - - string value = buf; - return value; - + return decimal.toString(); } diff --git a/utils/funcexp/func_ceil.cpp b/utils/funcexp/func_ceil.cpp index d78fab53b..de374af85 100644 --- a/utils/funcexp/func_ceil.cpp +++ b/utils/funcexp/func_ceil.cpp @@ -515,11 +515,11 @@ string Func_ceil::getStrVal(Row& row, if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) { - dataconvert::DataConvert::decimalToString(&d.s128Value, d.scale, tmp, 511, op_ct.colDataType); + return d.toString(true); } else { - dataconvert::DataConvert::decimalToString(d.value, d.scale, tmp, 511, op_ct.colDataType); + return d.toString(); } } else if (isUnsigned(op_ct.colDataType)) diff --git a/utils/funcexp/func_floor.cpp b/utils/funcexp/func_floor.cpp index a8afa77f1..3252fa031 100644 --- a/utils/funcexp/func_floor.cpp +++ b/utils/funcexp/func_floor.cpp @@ -436,11 +436,11 @@ string Func_floor::getStrVal(Row& row, if (op_ct.colWidth == datatypes::MAXDECIMALWIDTH) { - dataconvert::DataConvert::decimalToString(&d.s128Value, d.scale, tmp, 511, op_ct.colDataType); + return d.toString(true); } else { - dataconvert::DataConvert::decimalToString(d.value, d.scale, tmp, 511, op_ct.colDataType); + return d.toString(); } } else if (isUnsigned(op_ct.colDataType)) diff --git a/utils/funcexp/func_math.cpp b/utils/funcexp/func_math.cpp index fd0c0753b..a64e8fd75 100644 --- a/utils/funcexp/func_math.cpp +++ b/utils/funcexp/func_math.cpp @@ -1849,8 +1849,7 @@ string Func_format::getStrVal(Row& row, { IDB_Decimal decimal = parm[0]->data()->getDecimalVal(row, isNull); - char buf[80]; - + // This is an unacceptable way of doing rounding //perform rouding if needed if ( scale < 0 ) scale = 0; @@ -1912,9 +1911,7 @@ string Func_format::getStrVal(Row& row, decimal.s128Value = x; } - - dataconvert::DataConvert::decimalToString(&decimal.s128Value, - decimal.scale, buf, 80, parm[0]->data()->resultType().colDataType); + value = decimal.toString(true); } else { @@ -1973,12 +1970,8 @@ string Func_format::getStrVal(Row& row, decimal.value = x; } - - dataconvert::DataConvert::decimalToString( decimal.value, - decimal.scale, buf, 80, parm[0]->data()->resultType().colDataType); + value = decimal.toString(); } - - value = buf; } break; diff --git a/utils/funcexp/func_regexp.cpp b/utils/funcexp/func_regexp.cpp index 4a484c8e3..0fa61b0a0 100644 --- a/utils/funcexp/func_regexp.cpp +++ b/utils/funcexp/func_regexp.cpp @@ -116,18 +116,15 @@ inline bool getBool(rowgroup::Row& row, { IDB_Decimal d = pm[0]->data()->getDecimalVal(row, isNull); - char buf[80]; - if (pm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) { - dataconvert::DataConvert::decimalToString(&d.s128Value, d.scale, buf, 80, pm[0]->data()->resultType().colDataType); + expr = d.toString(true); } else { - dataconvert::DataConvert::decimalToString(d.value, d.scale, buf, 80, pm[0]->data()->resultType().colDataType); + expr = d.toString(); } - expr = buf; break; } @@ -198,18 +195,14 @@ inline bool getBool(rowgroup::Row& row, { IDB_Decimal d = pm[1]->data()->getDecimalVal(row, isNull); - char buf[80]; - if (pm[1]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH) { - dataconvert::DataConvert::decimalToString(&d.s128Value, d.scale, buf, 80, pm[1]->data()->resultType().colDataType); + pattern = d.toString(true); } else { - dataconvert::DataConvert::decimalToString(d.value, d.scale, buf, 80, pm[1]->data()->resultType().colDataType); + pattern = d.toString(); } - - pattern = buf; break; } diff --git a/utils/funcexp/func_round.cpp b/utils/funcexp/func_round.cpp index 4e3ad260c..b034cf3b3 100644 --- a/utils/funcexp/func_round.cpp +++ b/utils/funcexp/func_round.cpp @@ -721,13 +721,9 @@ string Func_round::getStrVal(Row& row, } if (!op_ct.isWideDecimalType()) - return dataconvert::DataConvert::decimalToString(x.value, x.scale, op_ct.colDataType); + return x.toString(); else - { - char buf[datatypes::Decimal::MAXLENGTH16BYTES]; - dataconvert::DataConvert::decimalToString( &x.s128Value, x.scale, buf, (uint8_t) sizeof(buf), op_ct.colDataType); - return string(buf); - } + return x.toString(true); } diff --git a/utils/funcexp/func_truncate.cpp b/utils/funcexp/func_truncate.cpp index 9ab383774..02455de61 100644 --- a/utils/funcexp/func_truncate.cpp +++ b/utils/funcexp/func_truncate.cpp @@ -740,13 +740,9 @@ string Func_truncate::getStrVal(Row& row, } if (!op_ct.isWideDecimalType()) - return dataconvert::DataConvert::decimalToString(x.value, x.scale, op_ct.colDataType); + return x.toString(); else - { - char buf[datatypes::Decimal::MAXLENGTH16BYTES]; - dataconvert::DataConvert::decimalToString( &x.s128Value, x.scale, buf, (uint8_t) sizeof(buf), op_ct.colDataType); - return string(buf); - } + return x.toString(true); } diff --git a/utils/funcexp/funchelpers.h b/utils/funcexp/funchelpers.h index 709d441bc..3317a5f69 100644 --- a/utils/funcexp/funchelpers.h +++ b/utils/funcexp/funchelpers.h @@ -785,10 +785,6 @@ string longDoubleToString(long double ld) return buf; } -//@bug6146, remove duplicate function with incorrect impl. Use the DataConvert::decimalToString() -//string decimalToString( execplan::IDB_Decimal x, int p ) - - uint64_t dateAdd( uint64_t time, const std::string& expr, execplan::IntervalColumn::interval_type unit, bool dateType, execplan::OpType funcType ); const std::string IDB_date_format(const dataconvert::DateTime&, const std::string&); const std::string timediff(int64_t, int64_t, bool isDateTime = true); diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index fbeb69ab7..d477bafc3 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -640,12 +640,11 @@ string Row::toString() const case CalpontSystemCatalog::UDECIMAL: if (colWidths[i] == datatypes::MAXDECIMALWIDTH) { - unsigned int buflen = datatypes::Decimal::MAXLENGTH16BYTES; - char *buf = (char*)alloca(buflen); - // empty the buffer - dataconvert::DataConvert::decimalToString(getBinaryField(i), - scale[i], buf, buflen, types[i]); - os << buf << " "; + datatypes::VDecimal dec(0, + scale[i], + precision[i], + getBinaryField(i)); + os << dec << " "; break; } //fallthrough diff --git a/writeengine/bulk/we_brmreporter.cpp b/writeengine/bulk/we_brmreporter.cpp index 229a8c333..5517325e1 100644 --- a/writeengine/bulk/we_brmreporter.cpp +++ b/writeengine/bulk/we_brmreporter.cpp @@ -300,7 +300,6 @@ void BRMReporter::sendCPToFile( ) { if (fCPInfo.size() > 0) { - char buf[datatypes::Decimal::MAXLENGTH16BYTES]; std::ostringstream oss; oss << "Writing " << fCPInfo.size() << " CP updates for table " << fTableName << " to report file " << fRptFileName; @@ -319,14 +318,9 @@ void BRMReporter::sendCPToFile( ) } else { - std::string bigMin, bigMax; - - dataconvert::DataConvert::decimalToString(&fCPInfo[i].bigMin, 0, buf, (uint8_t) sizeof(buf), fCPInfo[i].type); - bigMin = buf; - - dataconvert::DataConvert::decimalToString(&fCPInfo[i].bigMax, 0, buf, (uint8_t) sizeof(buf), fCPInfo[i].type); - bigMax = buf; - + datatypes::TSInt128 bigMin(&fCPInfo[i].bigMin); + datatypes::TSInt128 bigMax(&fCPInfo[i].bigMax); + fRptFile << "CP: " << fCPInfo[i].startLbid << ' ' << bigMax << ' ' << bigMin << ' ' << diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 3eeed68fc..0753ac7b8 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -2994,16 +2994,11 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, { if (fetchColColwidths[fetchColPos] == datatypes::MAXDECIMALWIDTH) { - int128_t* dec; - char buf[datatypes::Decimal::MAXLENGTH16BYTES]; - dec = row.getBinaryField(fetchColPos); - - dataconvert::DataConvert::decimalToString(dec, - (unsigned)fetchColScales[fetchColPos], buf, - (uint8_t) sizeof(buf), fetchColTypes[fetchColPos]); - - value.assign(buf); - + datatypes::VDecimal dec(0, + fetchColScales[fetchColPos], + rowGroups[txnId]->getPrecision()[fetchColPos], + row.getBinaryField(fetchColPos)); + value = dec.toString(true); break; } } @@ -3042,12 +3037,10 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, } else { - const int ctmp_size = 65 + 1 + 1 + 1; - char ctmp[ctmp_size] = {0}; - DataConvert::decimalToString( - intColVal, fetchColScales[fetchColPos], - ctmp, ctmp_size, fetchColTypes[fetchColPos]); - value = ctmp; // null termination by decimalToString + datatypes::VDecimal dec(intColVal, + fetchColScales[fetchColPos], + rowGroups[txnId]->getPrecision()[fetchColPos]); + value = dec.toString(); } } break; @@ -3356,16 +3349,11 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, { if (fetchColColwidths[fetchColPos] == datatypes::MAXDECIMALWIDTH) { - int128_t* dec; - char buf[datatypes::Decimal::MAXLENGTH16BYTES]; - dec = row.getBinaryField(fetchColPos); - - dataconvert::DataConvert::decimalToString(dec, - (unsigned)fetchColScales[fetchColPos], buf, - (uint8_t) sizeof(buf), fetchColTypes[fetchColPos]); - - value = buf; - + datatypes::VDecimal dec(0, + fetchColScales[fetchColPos], + rowGroups[txnId]->getPrecision()[fetchColPos], + row.getBinaryField(fetchColPos)); + value = dec.toString(true); break; } } @@ -3405,12 +3393,10 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, } else { - const int ctmp_size = 65 + 1 + 1 + 1; - char ctmp[ctmp_size] = {0}; - DataConvert::decimalToString( - intColVal, fetchColScales[fetchColPos], - ctmp, ctmp_size, fetchColTypes[fetchColPos]); - value = ctmp; // null termination by decimalToString + datatypes::VDecimal dec(intColVal, + fetchColScales[fetchColPos], + rowGroups[txnId]->getPrecision()[fetchColPos]); + value = dec.toString(); } } break; diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 2b5948a92..de9853f5d 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -3896,11 +3896,8 @@ void WriteEngineWrapper::printInputValue(const ColStructList& colStructList, curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); else if (curTuple.data.type() == typeid(int128_t)) { - // WIP replace with a single call - char buf[datatypes::Decimal::MAXLENGTH16BYTES]; - int128_t val = boost::any_cast(curTuple.data); - dataconvert::DataConvert::decimalToString(&val, 0, buf, (uint8_t) sizeof(buf), curColStruct.colDataType); - curStr.assign(buf); + datatypes::TSInt128 val(boost::any_cast(curTuple.data)); + curStr = val.toString(); } else if (curTuple.data.type() == typeid(double)) curStr = boost::lexical_cast(boost::any_cast(curTuple.data)); From 31e0909552af7155a2fe0be05763aae3ef862291 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 17 Nov 2020 11:38:25 +0000 Subject: [PATCH 77/78] Merge for 13264feb7 that is the fix for multiple LDI issues described in MCOL-4320/4364/4370 --- datatypes/mcs_datatype.h | 8 +- dbcon/mysql/ha_mcs_datatype.h | 199 ++++++++++------------------------ dbcon/mysql/ha_mcs_dml.cpp | 12 +- 3 files changed, 70 insertions(+), 149 deletions(-) diff --git a/datatypes/mcs_datatype.h b/datatypes/mcs_datatype.h index 87c894da3..a84bcce05 100644 --- a/datatypes/mcs_datatype.h +++ b/datatypes/mcs_datatype.h @@ -662,21 +662,17 @@ class ColBatchWriter FILE *m_filePtr; char m_delimiter; char m_enclosed_by; - bool m_utf8; public: ColBatchWriter(FILE *f, char delimiter, - char enclosed_by, - bool utf8) + char enclosed_by) :m_filePtr(f), m_delimiter(delimiter), - m_enclosed_by(enclosed_by), - m_utf8(utf8) + m_enclosed_by(enclosed_by) { } FILE *filePtr() const { return m_filePtr; } char delimiter() const { return m_delimiter; } char enclosed_by() const { return m_enclosed_by; } - bool utf8() const { return m_utf8; } }; diff --git a/dbcon/mysql/ha_mcs_datatype.h b/dbcon/mysql/ha_mcs_datatype.h index 0014d65a9..d3bae935f 100644 --- a/dbcon/mysql/ha_mcs_datatype.h +++ b/dbcon/mysql/ha_mcs_datatype.h @@ -191,8 +191,11 @@ class WriteBatchFieldMariaDB: public WriteBatchField public: Field *m_field; const CalpontSystemCatalog::ColType &m_type; - WriteBatchFieldMariaDB(Field *field, const CalpontSystemCatalog::ColType type) - :m_field(field), m_type(type) + uint32_t m_mbmaxlen; + WriteBatchFieldMariaDB(Field *field, + const CalpontSystemCatalog::ColType type, + uint32_t mbmaxlen) + :m_field(field), m_type(type), m_mbmaxlen(mbmaxlen) { } size_t ColWriteBatchDate(const uchar *buf, bool nullVal, ColBatchWriter &ci) override { @@ -328,6 +331,7 @@ public: size_t ColWriteBatchChar(const uchar *buf, bool nullVal, ColBatchWriter &ci) override { + uint32_t colWidthInBytes = m_type.colWidth * m_mbmaxlen; if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) { fprintf(ci.filePtr(), "%c", ci.delimiter()); @@ -338,10 +342,7 @@ public: { std::string escape; // Pad to the full length of the field - if (ci.utf8()) - escape.assign((char*)buf, m_type.colWidth * 3); - else - escape.assign((char*)buf, m_type.colWidth); + escape.assign((char*)buf, colWidthInBytes); boost::replace_all(escape, "\\", "\\\\"); @@ -368,93 +369,48 @@ public: } } - if (ci.utf8()) - return m_type.colWidth * 3; - else - return m_type.colWidth; + return colWidthInBytes; } size_t ColWriteBatchVarchar(const uchar *buf, bool nullVal, ColBatchWriter &ci) override { const uchar *buf0= buf; + uint32_t colWidthInBytes = m_type.colWidth * m_mbmaxlen; if (nullVal && (m_type.constraintType != CalpontSystemCatalog::NOTNULL_CONSTRAINT)) { fprintf(ci.filePtr(), "%c", ci.delimiter()); - - if (!ci.utf8()) + if (colWidthInBytes < 256) { - if (m_type.colWidth < 256) - { - buf++; - } - else - { - buf = buf + 2 ; - } + buf++; } - else //utf8 + else { - if (m_type.colWidth < 86) - { - buf++; - } - else - { - buf = buf + 2 ; - } + buf = buf + 2 ; } } else { int dataLength = 0; - if (!ci.utf8()) + if (colWidthInBytes < 256) { - if (m_type.colWidth < 256) - { dataLength = *(uint8_t*) buf; buf++; - } - else - { - dataLength = *(uint16_t*) buf; - buf = buf + 2 ; - } - std::string escape; - escape.assign((char*)buf, dataLength); - boost::replace_all(escape, "\\", "\\\\"); - fprintf(ci.filePtr(), "%c%.*s%c%c", - ci.enclosed_by(), - (int)escape.length(), escape.c_str(), - ci.enclosed_by(), ci.delimiter()); } - else //utf8 + else { - if (m_type.colWidth < 86) - { - dataLength = *(uint8_t*) buf; - buf++; - } - else - { dataLength = *(uint16_t*) buf; buf = buf + 2 ; - } - - std::string escape; - escape.assign((char*)buf, dataLength); - boost::replace_all(escape, "\\", "\\\\"); - - fprintf(ci.filePtr(), "%c%.*s%c%c", - ci.enclosed_by(), - (int)escape.length(), escape.c_str(), - ci.enclosed_by(), ci.delimiter()); } + std::string escape; + escape.assign((char*)buf, dataLength); + boost::replace_all(escape, "\\", "\\\\"); + fprintf(ci.filePtr(), "%c%.*s%c%c", + ci.enclosed_by(), + (int)escape.length(), escape.c_str(), + ci.enclosed_by(), ci.delimiter()); } - if (ci.utf8()) - buf += (m_type.colWidth * 3); - else - buf += m_type.colWidth; + buf += colWidthInBytes; return buf - buf0; } @@ -1125,89 +1081,42 @@ public: { fprintf(ci.filePtr(), "%c", ci.delimiter()); - if (!ci.utf8()) + if (m_type.colWidth < 256) { - if (m_type.colWidth < 256) - { - buf++; - } - else - { - buf = buf + 2; - } + buf++; } - else //utf8 + else { - if (m_type.colWidth < 86) - { - buf++; - } - else - { - buf = buf + 2 ; - } + buf = buf + 2; } } else { - int dataLength = 0; + uint16_t dataLength = 0; - if (!ci.utf8()) + if (m_type.colWidth < 256) { - if (m_type.colWidth < 256) - { - dataLength = *(int8_t*) buf; - buf++; - } - else - { - dataLength = *(int16_t*) buf; - buf = buf + 2 ; - } - - const uchar* tmpBuf = buf; - - for (int32_t i = 0; i < dataLength; i++) - { - fprintf(ci.filePtr(), "%02x", *(uint8_t*)tmpBuf); - tmpBuf++; - } - - fprintf(ci.filePtr(), "%c", ci.delimiter()); + dataLength = *(int8_t*) buf; + buf++; } - else //utf8 + else { - if (m_type.colWidth < 86) - { - dataLength = *(int8_t*) buf; - buf++; - } - else - { - dataLength = *(uint16_t*) buf; - buf = buf + 2 ; - } - - if ( dataLength > m_type.colWidth) - dataLength = m_type.colWidth; - - const uchar* tmpBuf = buf; - - for (int32_t i = 0; i < dataLength; i++) - { - fprintf(ci.filePtr(), "%02x", *(uint8_t*)tmpBuf); - tmpBuf++; - } - - fprintf(ci.filePtr(), "%c", ci.delimiter()); + dataLength = *(int16_t*) buf; + buf = buf + 2 ; } + + const uchar* tmpBuf = buf; + + for (int32_t i = 0; i < dataLength; i++) + { + fprintf(ci.filePtr(), "%02x", *(uint8_t*)tmpBuf); + tmpBuf++; + } + + fprintf(ci.filePtr(), "%c", ci.delimiter()); + } - if (ci.utf8()) - buf += (m_type.colWidth * 3); // QQ: why? It is varbinary! - else - buf += m_type.colWidth; - return buf - buf0; } @@ -1221,10 +1130,20 @@ public: uint32_t dataLength = 0; uintptr_t* dataptr; uchar* ucharptr; - uint colWidthInBytes = (ci.utf8() ? - m_type.colWidth * 3: m_type.colWidth); + bool isBlob = m_type.colDataType == CalpontSystemCatalog::BLOB; + uint colWidthInBytes = isBlob ? m_type.colWidth : m_type.colWidth * m_mbmaxlen; - if (colWidthInBytes < 256) + if (!isBlob && m_field->char_length() == 65535) + { + // Special case for TEXT field without default length, + // such as: + // CREATE TABLE mcol4364 (a TEXT); + // Here, char_length() represents the number of bytes, + // not number of characters. + dataLength = *(uint16_t*) buf; + buf += 2; + } + else if (colWidthInBytes < 256) { dataLength = *(uint8_t*) buf; buf++; @@ -1254,7 +1173,7 @@ public: ucharptr = (uchar*)*dataptr; buf += sizeof(uintptr_t); - if (m_type.colDataType == CalpontSystemCatalog::BLOB) + if (isBlob) { for (uint32_t i = 0; i < dataLength; i++) { diff --git a/dbcon/mysql/ha_mcs_dml.cpp b/dbcon/mysql/ha_mcs_dml.cpp index 88d9997f9..68ccff713 100644 --- a/dbcon/mysql/ha_mcs_dml.cpp +++ b/dbcon/mysql/ha_mcs_dml.cpp @@ -736,9 +736,15 @@ int ha_mcs_impl_write_batch_row_(const uchar* buf, TABLE* table, cal_impl_if::ca const datatypes::TypeHandler *h= colType.typeHandler(); if (h) // QQ: error reporting { - datatypes::ColBatchWriter writer(ci.filePtr, ci.delimiter, - ci.enclosed_by, ci.utf8); - datatypes::WriteBatchFieldMariaDB field(table->field[colpos], colType); + datatypes::ColBatchWriter writer(ci.filePtr, + ci.delimiter, + ci.enclosed_by); + Field* fieldPtr= table->field[colpos]; + uint32_t mbmaxlen = (fieldPtr->charset() && fieldPtr->charset()->mbmaxlen) + ? fieldPtr->charset()->mbmaxlen : 0; + datatypes::WriteBatchFieldMariaDB field(fieldPtr, + colType, + mbmaxlen); idbassert(table == table->field[colpos]->table); buf+= h->ColWriteBatch(&field, buf, nullVal, writer); } From aa44bca473fc239fbd41a5f961e4c230032ac049 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Wed, 18 Nov 2020 13:44:27 +0000 Subject: [PATCH 78/78] A pack of fixes for compilation errors and warnings for all platforms Add libdatatypes.so into debian packaging --- .drone.jsonnet | 6 +++--- datatypes/mcs_datatype.cpp | 6 +++--- datatypes/mcs_datatype.h | 1 + datatypes/mcs_int128.h | 21 ++++++++++++++++++ dbcon/execplan/arithmeticoperator.h | 2 +- dbcon/execplan/calpontsystemcatalog.cpp | 20 +++++++++++++++++ dbcon/execplan/treenode.h | 3 +-- dbcon/joblist/rowestimator.cpp | 2 +- dbcon/joblist/tuple-bps.cpp | 3 ++- debian/mariadb-plugin-columnstore.install | 1 + utils/common/checks.h | 26 +++++++++++++++-------- utils/funcexp/func_cast.cpp | 4 +++- utils/funcexp/func_elt.cpp | 4 ++-- 13 files changed, 76 insertions(+), 23 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index 882c3d70a..399ba1549 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -25,9 +25,9 @@ local platformMap(branch, platform) = }; local platform_map = { - 'opensuse/leap:15': 'zypper ' + rpm_build_deps + ' cmake libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel libquadmath0 && cmake ' + cmakeflags + branch_cmakeflags_map[branch] + ' -DRPM=sles15 && make -j$(nproc) package', - 'centos:7': 'yum install -y epel-release && yum install -y cmake3 && ln -s /usr/bin/cmake3 /usr/bin/cmake && yum ' + rpm_build_deps + ' libquadmath && cmake ' + cmakeflags + branch_cmakeflags_map[branch] + ' -DRPM=centos7 && make -j$(nproc) package', - 'centos:8': "yum install -y libgcc && sed -i 's/enabled=0/enabled=1/' /etc/yum.repos.d/CentOS-PowerTools.repo && yum " + rpm_build_deps + ' cmake libquadmath && cmake ' + cmakeflags + branch_cmakeflags_map[branch] + ' -DRPM=centos8 && make -j$(nproc) package', + 'opensuse/leap:15': 'zypper ' + rpm_build_deps + ' cmake libboost_system-devel libboost_filesystem-devel libboost_thread-devel libboost_regex-devel libboost_date_time-devel libboost_chrono-devel libboost_atomic-devel gcc-fortran && cmake ' + cmakeflags + branch_cmakeflags_map[branch] + ' -DRPM=sles15 && make -j$(nproc) package', + 'centos:7': 'yum install -y epel-release && yum install -y cmake3 && ln -s /usr/bin/cmake3 /usr/bin/cmake && yum ' + rpm_build_deps + ' libquadmath libquadmath-devel && cmake ' + cmakeflags + branch_cmakeflags_map[branch] + ' -DRPM=centos7 && make -j$(nproc) package', + 'centos:8': "yum install -y libgcc && sed -i 's/enabled=0/enabled=1/' /etc/yum.repos.d/CentOS-PowerTools.repo && yum " + rpm_build_deps + ' cmake libquadmath libquadmath-devel && cmake ' + cmakeflags + branch_cmakeflags_map[branch] + ' -DRPM=centos8 && make -j$(nproc) package', 'debian:9': deb_build_deps + " && CMAKEFLAGS='" + cmakeflags + branch_cmakeflags_map[branch] + " -DDEB=stretch' debian/autobake-deb.sh", 'debian:10': deb_build_deps + " && CMAKEFLAGS='" + cmakeflags + branch_cmakeflags_map[branch] + " -DDEB=buster' debian/autobake-deb.sh", 'ubuntu:16.04': deb_build_deps + " && CMAKEFLAGS='" + cmakeflags + branch_cmakeflags_map[branch] + " -DDEB=xenial' debian/autobake-deb.sh", diff --git a/datatypes/mcs_datatype.cpp b/datatypes/mcs_datatype.cpp index fdbf149f4..493d64fd9 100644 --- a/datatypes/mcs_datatype.cpp +++ b/datatypes/mcs_datatype.cpp @@ -346,13 +346,13 @@ TypeHandler::find(SystemCatalog::ColDataType typeCode, case SystemCatalog::UDOUBLE: return &mcs_type_handler_udouble; case SystemCatalog::DECIMAL: - if (ct.colWidth < datatypes::MAXDECIMALWIDTH) + if (static_cast(ct.colWidth) < datatypes::MAXDECIMALWIDTH) return &mcs_type_handler_sdecimal64; else return &mcs_type_handler_sdecimal128; case SystemCatalog::UDECIMAL: - if (ct.colWidth < datatypes::MAXDECIMALWIDTH) + if (static_cast(ct.colWidth) < datatypes::MAXDECIMALWIDTH) return &mcs_type_handler_udecimal64; else return &mcs_type_handler_udecimal128; @@ -1303,7 +1303,7 @@ TypeHandlerStr::toSimpleValue(const SessionParam &sp, /****************************************************************************/ MinMaxPartitionInfo::MinMaxPartitionInfo(const BRM::EMEntry &entry) - :m_status(entry.status == BRM::EXTENTOUTOFSERVICE ? ET_DISABLED : 0) + :m_status(entry.status == BRM::EXTENTOUTOFSERVICE ? ET_DISABLED : EXPL_NULL) { } diff --git a/datatypes/mcs_datatype.h b/datatypes/mcs_datatype.h index a84bcce05..470950960 100644 --- a/datatypes/mcs_datatype.h +++ b/datatypes/mcs_datatype.h @@ -582,6 +582,7 @@ class MinMaxPartitionInfo: public MinMaxInfo { enum status_flag_t : uint64_t { + EXPL_NULL = 0x0000, ET_DISABLED = 0x0002, CPINVALID = 0x0004 }; diff --git a/datatypes/mcs_int128.h b/datatypes/mcs_int128.h index 1bab9171c..3d508420a 100644 --- a/datatypes/mcs_int128.h +++ b/datatypes/mcs_int128.h @@ -72,6 +72,7 @@ namespace datatypes { using int128_t = __int128; +using uint128_t = unsigned __int128; // Type traits @@ -90,6 +91,26 @@ struct is_allowed_numeric { static const bool value = true; }; +template +struct is_int128_t { + static const bool value = false; +}; + +template<> +struct is_int128_t { + static const bool value = true; +}; + +template +struct is_uint128_t { + static const bool value = false; +}; + +template<> +struct is_uint128_t { + static const bool value = true; +}; + // The method converts a __float128 s128Value to a double. static inline double getDoubleFromFloat128(const __float128& value) { diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index 96c7ad75c..042d47cd1 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -196,7 +196,7 @@ public: return TreeNode::getBoolVal(); } void adjustResultType(const CalpontSystemCatalog::ColType& m); - const inline bool getOverflowCheck() const + inline bool getOverflowCheck() const { return fDecimalOverflowCheck; } diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index 7ead90f9d..18b0fb7c5 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -6119,6 +6119,26 @@ CalpontSystemCatalog::ColType::ColType(const ColType& rhs) cs = rhs.cs; } +CalpontSystemCatalog::ColType& CalpontSystemCatalog::ColType::operator=(const ColType& rhs) +{ + TypeHolderStd::operator=(rhs); + constraintType = rhs.constraintType; + ddn = rhs.ddn; + defaultValue = rhs.defaultValue; + colPosition = rhs.colPosition; + compressionType = rhs.compressionType; + columnOID = rhs.columnOID; + autoincrement = rhs.autoincrement; + nextvalue = rhs.nextvalue; + charsetNumber = rhs.charsetNumber; + cs = rhs.cs; + + return *this; +} + + + + CHARSET_INFO* CalpontSystemCatalog::ColType::getCharset() { if (!cs) diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 1b7fd316c..3d94a6e09 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -60,8 +60,7 @@ typedef execplan::CalpontSystemCatalog::ColType Type; class IDB_Decimal: public datatypes::VDecimal { public: - using datatypes::VDecimal::VDecimal; - + IDB_Decimal() = default; IDB_Decimal(int64_t val, int8_t s, uint8_t p, const int128_t &val128 = 0) : VDecimal(val, s, p, val128) {} diff --git a/dbcon/joblist/rowestimator.cpp b/dbcon/joblist/rowestimator.cpp index 78ef9c1bf..d40d15248 100644 --- a/dbcon/joblist/rowestimator.cpp +++ b/dbcon/joblist/rowestimator.cpp @@ -303,7 +303,7 @@ float RowEstimator::estimateRowReturnFactor(const BRM::EMEntry& emEntry, float factor = 1.0; float tempFactor = 1.0; - uint64_t adjustedMin, adjustedMax; + uint64_t adjustedMin = 0, adjustedMax = 0; uint128_t adjustedBigMin, adjustedBigMax; uint32_t distinctValuesEstimate; diff --git a/dbcon/joblist/tuple-bps.cpp b/dbcon/joblist/tuple-bps.cpp index ceee3e8f0..2d91f87de 100644 --- a/dbcon/joblist/tuple-bps.cpp +++ b/dbcon/joblist/tuple-bps.cpp @@ -1584,7 +1584,8 @@ bool TupleBPS::processSingleFilterString(int8_t BOP, int8_t colWidth, T val, con throw logic_error("invalid column width"); } - if (colWidth < datatypes::MAXDECIMALWIDTH) + // Assumption is that colWidth > 0 + if (static_cast(colWidth) < datatypes::MAXDECIMALWIDTH) thisPredicate = compareSingleValue(COP, (int64_t) val, val2); else thisPredicate = compareSingleValue(COP, (int128_t) val, bigVal2); diff --git a/debian/mariadb-plugin-columnstore.install b/debian/mariadb-plugin-columnstore.install index 3f4cc792b..b74b47b8f 100644 --- a/debian/mariadb-plugin-columnstore.install +++ b/debian/mariadb-plugin-columnstore.install @@ -121,6 +121,7 @@ usr/lib/*/libwindowfunction.so usr/lib/*/libwriteengine.so usr/lib/*/libwriteengineclient.so usr/lib/*/libwriteengineredistribute.so +usr/lib/*/libdatatypes.so usr/lib/mysql/plugin/ha_columnstore.so usr/lib/mysql/plugin/libregr_mysql.so usr/lib/mysql/plugin/libudf_mysql.so diff --git a/utils/common/checks.h b/utils/common/checks.h index 36b87312e..fb4a94e94 100644 --- a/utils/common/checks.h +++ b/utils/common/checks.h @@ -18,25 +18,33 @@ #define UTILS_COMMON_CHECKS_H #include +#include "mcs_int128.h" namespace utils { template -typename std::enable_if::value, bool>::type is_nonnegative(T) -{ return true; }; +typename std::enable_if::value || + datatypes::is_uint128_t::value, + bool>::type +is_nonnegative(T) { return true; }; template -typename std::enable_if::value, bool>::type is_nonnegative(T v) -{ return v >= 0; }; - +typename std::enable_if::value || + datatypes::is_int128_t::value, + bool>::type +is_nonnegative(T v) { return v >= 0; }; template -typename std::enable_if::value, bool>::type is_negative(T) -{ return false; }; +typename std::enable_if::value || + datatypes::is_uint128_t::value, + bool>::type +is_negative(T) { return false; }; template -typename std::enable_if::value, bool>::type is_negative(T v) -{ return v < 0; }; +typename std::enable_if::value || + datatypes::is_int128_t::value, + bool>::type +is_negative(T v) { return v < 0; }; } // namespace utils diff --git a/utils/funcexp/func_cast.cpp b/utils/funcexp/func_cast.cpp index aa0212a48..ef36bb320 100644 --- a/utils/funcexp/func_cast.cpp +++ b/utils/funcexp/func_cast.cpp @@ -383,7 +383,7 @@ uint64_t Func_cast_unsigned::getUintVal(Row& row, uint128_t tmpval = d.s128Value / scaleDivisor; int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; - if (tmpval >= 0 && lefto > 4) + if (utils::is_nonnegative(tmpval) && lefto > 4) tmpval++; if (tmpval > static_cast(UINT64_MAX)) @@ -404,7 +404,9 @@ uint64_t Func_cast_unsigned::getUintVal(Row& row, int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1); if ( utils::is_nonnegative(value) && lefto > 4 ) + { value++; + } return value; } diff --git a/utils/funcexp/func_elt.cpp b/utils/funcexp/func_elt.cpp index b3bf6a3b3..c9c811d6c 100644 --- a/utils/funcexp/func_elt.cpp +++ b/utils/funcexp/func_elt.cpp @@ -86,10 +86,10 @@ string Func_elt::getStrVal(rowgroup::Row& row, int128_t tmpval = d.s128Value / scaleDivisor; int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2; - if (utils::is_nonnegative(tmpval) >= 0 && lefto > 4) + if (utils::is_nonnegative(tmpval) && lefto > 4) tmpval++; - if (utils::is_negative(tmpval) < 0 && lefto < -4) + if (utils::is_negative(tmpval) && lefto < -4) tmpval--; number = datatypes::Decimal::getInt64FromWideDecimal(tmpval);