From 98213c0094d10de184e8f5422b96f98b48872b64 Mon Sep 17 00:00:00 2001 From: drrtuy Date: Sun, 26 Jan 2020 11:07:45 +0300 Subject: [PATCH] MCOL-641 Addition now works for DECIMAL columns with precision > 18. --- dbcon/execplan/arithmeticoperator.h | 19 +++- dbcon/execplan/returnedcolumn.cpp | 6 ++ dbcon/execplan/returnedcolumn.h | 1 + dbcon/execplan/simplecolumn.cpp | 19 +++- dbcon/execplan/treenode.h | 17 +++- .../primproc/batchprimitiveprocessor.cpp | 2 +- primitives/primproc/columncommand.cpp | 2 +- utils/dataconvert/dataconvert.cpp | 2 +- utils/funcexp/funcexp.cpp | 22 ++++- utils/rowgroup/rowgroup.cpp | 96 +++++++++++++++++++ utils/rowgroup/rowgroup.h | 3 + 11 files changed, 175 insertions(+), 14 deletions(-) diff --git a/dbcon/execplan/arithmeticoperator.h b/dbcon/execplan/arithmeticoperator.h index ddf0b7ead..4660387fe 100644 --- a/dbcon/execplan/arithmeticoperator.h +++ b/dbcon/execplan/arithmeticoperator.h @@ -43,6 +43,7 @@ namespace execplan class ArithmeticOperator : public Operator { +using cscType = execplan::CalpontSystemCatalog::ColType; public: ArithmeticOperator(); @@ -200,6 +201,7 @@ private: template inline result_t execute(result_t op1, result_t op2, bool& isNull); inline void execute(IDB_Decimal& result, IDB_Decimal op1, IDB_Decimal op2, bool& isNull); + inline void execute(IDB_Decimal& result, IDB_Decimal op1, IDB_Decimal op2, bool& isNull, cscType& resultCscType); std::string fTimeZone; }; @@ -236,12 +238,11 @@ inline void ArithmeticOperator::evaluate(rowgroup::Row& row, bool& isNull, Parse case execplan::CalpontSystemCatalog::LONGDOUBLE: fResult.longDoubleVal = execute(lop->getLongDoubleVal(row, isNull), rop->getLongDoubleVal(row, isNull), isNull); break; - + // WIP MCOL-641 case execplan::CalpontSystemCatalog::DECIMAL: case execplan::CalpontSystemCatalog::UDECIMAL: - execute (fResult.decimalVal, lop->getDecimalVal(row, isNull), rop->getDecimalVal(row, isNull), isNull); + execute(fResult.decimalVal, lop->getDecimalVal(row, isNull), rop->getDecimalVal(row, isNull), isNull, fOperationType); break; - default: { std::ostringstream oss; @@ -282,11 +283,21 @@ inline result_t ArithmeticOperator::execute(result_t op1, result_t op2, bool& is } } -inline void ArithmeticOperator::execute(IDB_Decimal& result, IDB_Decimal op1, IDB_Decimal op2, bool& isNull) +inline void ArithmeticOperator::execute(IDB_Decimal& result, IDB_Decimal op1, IDB_Decimal op2, bool& isNull, cscType& resultCscType) { switch (fOp) { case OP_ADD: + if (resultCscType.precision > 18) + { + // WIP make this a separate function w and w/o overflow check + if (resultCscType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) + result.__v.__s128 = op1.__v.__s128 + op2.__v.__s128; + else + result.__v.__u128 = op1.__v.__u128 + op2.__v.__u128; + break; + } + if (result.scale == op1.scale && result.scale == op2.scale) { result.value = op1.value + op2.value; diff --git a/dbcon/execplan/returnedcolumn.cpp b/dbcon/execplan/returnedcolumn.cpp index 44278424f..5900f077f 100644 --- a/dbcon/execplan/returnedcolumn.cpp +++ b/dbcon/execplan/returnedcolumn.cpp @@ -49,6 +49,7 @@ ReturnedColumn::ReturnedColumn(): fReturnAll (false), fColPosition(-1), fHasAggregate(false), fInputIndex(-1), + fInputOffset(-1), fOutputIndex(-1), fExpressionId ((uint32_t) - 1) { @@ -69,6 +70,7 @@ ReturnedColumn::ReturnedColumn(const string& sql): fHasAggregate(false), fData(sql), fInputIndex(-1), + fInputOffset(-1), fOutputIndex(-1), fExpressionId ((uint32_t) - 1) { @@ -88,6 +90,7 @@ ReturnedColumn::ReturnedColumn(const uint32_t sessionID, const bool returnAll): fColPosition(-1), fHasAggregate(false), fInputIndex(-1), + fInputOffset(-1), fOutputIndex(-1), fExpressionId ((uint32_t) - 1) { @@ -109,6 +112,7 @@ ReturnedColumn::ReturnedColumn(const ReturnedColumn& rhs, const uint32_t session fHasAggregate(rhs.fHasAggregate), fData(rhs.fData), fInputIndex(rhs.fInputIndex), + fInputOffset(rhs.fInputOffset), fOutputIndex(rhs.fOutputIndex), fExpressionId (rhs.fExpressionId) { @@ -141,6 +145,7 @@ void ReturnedColumn::serialize(messageqcpp::ByteStream& b) const b << (uint64_t)fColSource; b << (int64_t)fColPosition; b << (uint32_t)fInputIndex; + b << (uint32_t)fInputOffset; b << (uint32_t)fOutputIndex; b << (int32_t)fSequence; b << (uint8_t)fReturnAll; @@ -163,6 +168,7 @@ void ReturnedColumn::unserialize(messageqcpp::ByteStream& b) b >> (uint64_t&)fColSource; b >> (int64_t&)fColPosition; b >> (uint32_t&)fInputIndex; + b >> (uint32_t&)fInputOffset; b >> (uint32_t&)fOutputIndex; b >> (int32_t&)fSequence; b >> (uint8_t&)fReturnAll; diff --git a/dbcon/execplan/returnedcolumn.h b/dbcon/execplan/returnedcolumn.h index 45f23c047..6c07f5694 100644 --- a/dbcon/execplan/returnedcolumn.h +++ b/dbcon/execplan/returnedcolumn.h @@ -374,6 +374,7 @@ public: protected: std::string fErrMsg; /// error occured in evaluation uint32_t fInputIndex; /// index to the input rowgroup + uint32_t fInputOffset; /// index to the input rowgroup uint32_t fOutputIndex; /// index to the output rowgroup uint32_t fExpressionId; /// unique id for this expression }; diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index 921d4d180..96466403a 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -51,6 +51,7 @@ using namespace joblist; #include "aggregatecolumn.h" #include "constantfilter.h" #include "../../utils/windowfunction/windowfunction.h" +#include "utils/common/branchpred.h" namespace execplan { @@ -482,6 +483,7 @@ void SimpleColumn::setDerivedTable() // @todo make aggregate filter to having clause. not optimize it for now if (fDerivedRefCol && + // WIP replace with typeid() (dynamic_cast(fDerivedRefCol) || dynamic_cast(fDerivedRefCol))) fDerivedTable = ""; @@ -500,6 +502,12 @@ bool SimpleColumn::singleTable(CalpontSystemCatalog::TableAliasName& tan) // @todo move to inline void SimpleColumn::evaluate(Row& row, bool& isNull) { + // WIP Move this block into an appropriate place + if (UNLIKELY(fInputOffset == -1)) + { + fInputOffset = row.getOffset(fInputIndex); + } + bool isNull2 = row.isNullValue(fInputIndex); if (isNull2) @@ -627,7 +635,16 @@ void SimpleColumn::evaluate(Row& row, bool& isNull) { case 16: { - fResult.decimalVal.value = row.getIntField<16>(fInputIndex); + if (fResultType.colDataType == CalpontSystemCatalog::UDECIMAL) + { + fResult.decimalVal.__v.__u128 = + *row.getBinaryField_offset(fInputOffset); + } + else + { + fResult.decimalVal.__v.__s128 = + *row.getBinaryField_offset(fInputOffset); + } fResult.decimalVal.scale = (unsigned)fResultType.scale; break; } diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index fcc92018d..760760a9d 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -36,6 +36,7 @@ #include "exceptclasses.h" #include "dataconvert.h" +using int128_t = __int128; using uint128_t = unsigned __int128; namespace messageqcpp @@ -63,11 +64,17 @@ typedef execplan::CalpontSystemCatalog::ColType Type; */ struct IDB_Decimal { - IDB_Decimal(): val(0), value(0), scale(0), precision(0) {} + IDB_Decimal(): value(0), scale(0), precision(0) + { + __v.__s128 = 0; + } IDB_Decimal(int64_t val, int8_t s, uint8_t p) : value (val), scale(s), - precision(p) {} + precision(p) + { + __v.__s128 = 0; + } int decimalComp(const IDB_Decimal& d) const { @@ -151,7 +158,11 @@ struct IDB_Decimal return (decimalComp(rhs) != 0); } - uint128_t val; + union { + uint128_t __u128; + int128_t __s128; + int64_t __s64[2]; + } __v; int64_t value; int8_t scale; // 0~38 uint8_t precision; // 1~38 diff --git a/primitives/primproc/batchprimitiveprocessor.cpp b/primitives/primproc/batchprimitiveprocessor.cpp index 3750c0cde..54fdcf482 100644 --- a/primitives/primproc/batchprimitiveprocessor.cpp +++ b/primitives/primproc/batchprimitiveprocessor.cpp @@ -1726,7 +1726,7 @@ void BatchPrimitiveProcessor::execute() if (fe2->evaluate(&fe2In)) { applyMapping(fe2Mapping, fe2In, &fe2Out); - //cerr << " passed. output row: " << fe2Out.toString() << endl; + cerr << " passed. output row: " << fe2Out.toString() << endl; fe2Out.setRid (fe2In.getRelRid()); fe2Output.incRowCount(); fe2Out.nextRow(); diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index 30d76d0f2..3b35f7c77 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -593,7 +593,7 @@ void ColumnCommand::prep(int8_t outputType, bool absRids) break; case 16: // WIP MCOL-641 - cout << __FILE__<< ":" <<__LINE__ << " Fix shift and mask for 16 Bytes ?"<< endl; + cout << __FILE__<< ":" <<__LINE__ << " Set shift and mask for 16 Bytes"<< endl; shift = 1; mask = 0x01; break; diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index f9510c311..96146a71a 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1198,7 +1198,7 @@ void DataConvert::toString(T* dec, char *p, size_t buflen) char* original_p = p; int printed_chars = 0; - // WIP replace snprintf with streams + // WIP replace snprintf with streams if (high_pod->lo != 0) { printed_chars = snprintf(p, div_log+1, "%lu", high_pod->lo); p += printed_chars; diff --git a/utils/funcexp/funcexp.cpp b/utils/funcexp/funcexp.cpp index 79039c923..cce824b47 100644 --- a/utils/funcexp/funcexp.cpp +++ b/utils/funcexp/funcexp.cpp @@ -472,10 +472,26 @@ void FuncExp::evaluate(rowgroup::Row& row, std::vector& expressi { IDB_Decimal val = expression[i]->getDecimalVal(row, isNull); - if (isNull) - row.setIntField<8>(BIGINTNULL, expression[i]->outputIndex()); + // WIP + if (expression[i]->resultType().precision > 18) + { + // WIP make this a separate function w and w/o overflow check + if (expression[i]->resultType().colDataType == execplan::CalpontSystemCatalog::DECIMAL) + row.setBinaryField(reinterpret_cast(&val.__v.__s128), + expression[i]->resultType().colWidth, + row.getOffset(expression[i]->outputIndex())); + else + row.setBinaryField(reinterpret_cast(&val.__v.__u128), + expression[i]->resultType().colWidth, + row.getOffset(expression[i]->outputIndex())); + } else - row.setIntField<8>(val.value, expression[i]->outputIndex()); + { + if (isNull) + row.setIntField<8>(BIGINTNULL, expression[i]->outputIndex()); + else + row.setIntField<8>(val.value, expression[i]->outputIndex()); + } break; } diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 9e55810d6..e756e1c5e 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -42,12 +42,15 @@ using namespace execplan; #include "nullvaluemanip.h" #include "rowgroup.h" +#include "dataconvert.h" #include "collation.h" namespace rowgroup { +using cscType = execplan::CalpontSystemCatalog::ColDataType; + StringStore::StringStore() : empty(true), fUseStoreStringMutex(false) { } StringStore::StringStore(const StringStore&) @@ -630,6 +633,19 @@ string Row::toString() const } case CalpontSystemCatalog::BINARY: std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; + break; + // WIP + case CalpontSystemCatalog::DECIMAL: + if (precision[i] > 18) + { + char *buf = (char*)alloca(precision[i] + 3); + // empty the buffer + dataconvert::DataConvert::toString(getBinaryField(i), + buf, precision[i]+3); + os << buf << " "; + break; + } + // fallback if the precision < 18 default: os << getIntField(i) << " "; break; @@ -866,6 +882,79 @@ void Row::initToNull() } } +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + ostringstream os; + os << "Row::isNullValue(): got bad column type at offset("; + os << offset; + os << "). Width="; + os << width << endl; + throw logic_error(os.str()); +} +/* +// WIP how to make if that enables explicit template for two cscTypes? +// Method template resolution could impose some perf degradation +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) == static_cast(joblist::BIGINTNULL)); +} + +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) == static_cast(joblist::BIGINTNULL)); +} + +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) == static_cast(joblist::BIGINTNULL)); +} + +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) == static_cast(joblist::BIGINTNULL)); +} + +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (data[offset] == joblist::TINYINTNULL); +} + +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (data[offset] == joblist::TINYINTNULL); +} + +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) == static_cast(joblist::SMALLINTNULL)); +} + +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) == static_cast(joblist::SMALLINTNULL)); +} + +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) == static_cast(joblist::INTNULL)); +} + +template +inline bool Row::isNullValue_offset(uint32_t offset) const +{ + return (*reinterpret_cast(&data[offset]) == static_cast(joblist::INTNULL)); +} +*/ bool Row::isNullValue(uint32_t colIndex) const { switch (types[colIndex]) @@ -1533,6 +1622,13 @@ void applyMapping(const int* mapping, const Row& in, Row* out) out->setUintField(in.getUintField(i), mapping[i]); else if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::LONGDOUBLE)) out->setLongDoubleField(in.getLongDoubleField(i), mapping[i]); + // WIP this doesn't look right b/c we can pushdown colType + // Migrate to offset based methods here + // code precision 2 width convertor + else if (UNLIKELY(in.getColTypes()[i] == execplan::CalpontSystemCatalog::DECIMAL && + in.getColumnWidth(i) == 16)) + out->setBinaryField(in.getBinaryField(i), 16, + out->getOffset(mapping[i])); else if (in.isUnsigned(i)) out->setUintField(in.getUintField(i), mapping[i]); else diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 534b9b67a..ed02c6ee7 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -69,6 +69,7 @@ typedef const struct charset_info_st CHARSET_INFO; namespace rowgroup { +//using cscType = execplan::CalpontSystemCatalog::ColDataType; const int16_t rgCommonSize = 8192; /* @@ -447,6 +448,8 @@ public: uint64_t getNullValue(uint32_t colIndex) const; bool isNullValue(uint32_t colIndex) const; + template + inline bool isNullValue_offset(uint32_t offset) const; // when NULLs are pulled out via getIntField(), they come out with these values. // Ex: the 1-byte int null value is 0x80. When it gets cast to an int64_t