diff --git a/datatypes/mcs_decimal.cpp b/datatypes/mcs_decimal.cpp index 2f9272491..007ac9c81 100644 --- a/datatypes/mcs_decimal.cpp +++ b/datatypes/mcs_decimal.cpp @@ -24,6 +24,27 @@ namespace datatypes { + + struct lldiv_t_128 + { + int128_t quot; + int128_t rem; + lldiv_t_128() : quot(0), rem(0) {} + }; + + inline lldiv_t_128 lldiv128(const int128_t& dividend, const int128_t& divisor) + { + lldiv_t_128 res; + + if (UNLIKELY(divisor == 0) || UNLIKELY(dividend == 0)) + return res; + + res.quot = dividend / divisor; + res.rem = dividend % divisor; + + return res; + } + template @@ -94,42 +115,42 @@ namespace datatypes int Decimal::compare(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r) { - int128_t divL, divR; - getScaleDivisor(divL, l.scale); - getScaleDivisor(divR, r.scale); - int128_t quotinentL, quotinentR, remainderL, remainderR; - quotinentL = l.s128Value/divL; - remainderL = l.s128Value%divL; - quotinentR = r.s128Value/divR; - remainderR = r.s128Value%divR; + int128_t divisorL, divisorR; + getScaleDivisor(divisorL, l.scale); + getScaleDivisor(divisorR, r.scale); + + lldiv_t_128 d1 = lldiv128(l.s128Value, divisorL); + lldiv_t_128 d2 = lldiv128(r.s128Value, divisorR); int ret = 0; - if (quotinentL > quotinentR) + if (d1.quot > d2.quot) { ret = 1; } - else if (quotinentL < quotinentR) + else if (d1.quot < d2.quot) { ret = -1; } else { // rem carries the value's sign, but needs to be normalized. - int32_t s = l.scale - r.scale; + int64_t s = l.scale - r.scale; + int128_t divisor; + getScaleDivisor(divisor, abs(s)); if (s < 0) { - if ((remainderL * mcs_pow_10[-s]) > remainderR) + if ((d1.rem * divisor) > d2.rem) ret = 1; - else if ((remainderL * mcs_pow_10[-s]) < remainderR) + else if ((d1.rem * divisor) < d2.rem) ret = -1; } else { - if (remainderL > (remainderR * mcs_pow_10[s])) + if (d1.rem > (d2.rem * divisor)) ret = 1; - else if (remainderL < (remainderR * mcs_pow_10[s])) + else if (d1.rem < (d2.rem * divisor)) ret = -1; } } diff --git a/dbcon/execplan/constantcolumn.cpp b/dbcon/execplan/constantcolumn.cpp index 28a7611d4..775ad88ed 100644 --- a/dbcon/execplan/constantcolumn.cpp +++ b/dbcon/execplan/constantcolumn.cpp @@ -230,6 +230,7 @@ ConstantColumn::ConstantColumn(const int64_t val, TYPE type) : fResult.doubleVal = (double)fResult.intVal; fResult.longDoubleVal = (long double)fResult.intVal; fResult.decimalVal.value = fResult.intVal; + fResult.decimalVal.s128Value = fResult.intVal; fResult.decimalVal.scale = 0; fResultType.colDataType = CalpontSystemCatalog::BIGINT; fResultType.colWidth = 8; @@ -250,6 +251,7 @@ ConstantColumn::ConstantColumn(const uint64_t val, TYPE type) : fResult.doubleVal = (double)fResult.uintVal; fResult.longDoubleVal = (long double)fResult.uintVal; fResult.decimalVal.value = fResult.uintVal; + fResult.decimalVal.s128Value = fResult.uintVal; fResult.decimalVal.scale = 0; fResultType.colDataType = CalpontSystemCatalog::UBIGINT; fResultType.colWidth = 8; @@ -317,6 +319,7 @@ void ConstantColumn::serialize(messageqcpp::ByteStream& b) const b << (uint8_t)fResult.boolVal; b << fResult.strVal; b << (uint64_t)fResult.decimalVal.value; + b << (uint128_t)fResult.decimalVal.s128Value; b << (uint8_t)fResult.decimalVal.scale; b << (uint8_t)fResult.decimalVal.precision; } @@ -341,6 +344,7 @@ void ConstantColumn::unserialize(messageqcpp::ByteStream& b) b >> (uint8_t&)fResult.boolVal; b >> fResult.strVal; b >> (uint64_t&)fResult.decimalVal.value; + b >> (uint128_t&)fResult.decimalVal.s128Value; b >> (uint8_t&)fResult.decimalVal.scale; b >> (uint8_t&)fResult.decimalVal.precision; } diff --git a/dbcon/execplan/simplecolumn.cpp b/dbcon/execplan/simplecolumn.cpp index 34e02a4df..09328ae9f 100644 --- a/dbcon/execplan/simplecolumn.cpp +++ b/dbcon/execplan/simplecolumn.cpp @@ -637,38 +637,36 @@ void SimpleColumn::evaluate(Row& row, bool& isNull) { fResult.decimalVal.s128Value = *row.getBinaryField_offset(fInputOffset); - fResult.decimalVal.scale = (unsigned)fResultType.scale; break; } case 1: { fResult.decimalVal.value = row.getIntField<1>(fInputIndex); - fResult.decimalVal.scale = (unsigned)fResultType.scale; break; } case 2: { fResult.decimalVal.value = row.getIntField<2>(fInputIndex); - fResult.decimalVal.scale = (unsigned)fResultType.scale; break; } case 4: { fResult.decimalVal.value = row.getIntField<4>(fInputIndex); - fResult.decimalVal.scale = (unsigned)fResultType.scale; break; } default: { fResult.decimalVal.value = (int64_t)row.getUintField<8>(fInputIndex); - fResult.decimalVal.scale = (unsigned)fResultType.scale; break; } } + fResult.decimalVal.scale = (unsigned)fResultType.scale; + fResult.decimalVal.precision = (unsigned)fResultType.precision; + break; } diff --git a/dbcon/execplan/treenode.h b/dbcon/execplan/treenode.h index 99f5084a6..be105cf92 100644 --- a/dbcon/execplan/treenode.h +++ b/dbcon/execplan/treenode.h @@ -67,6 +67,7 @@ struct IDB_Decimal { s128Value = 0; } + IDB_Decimal(int64_t val, int8_t s, uint8_t p) : value (val), scale(s), @@ -131,6 +132,7 @@ struct IDB_Decimal return (decimalComp(rhs) == 0); } } + bool operator>(const IDB_Decimal& rhs) const { if (utils::widthByPrecision(precision) == 16) @@ -148,6 +150,7 @@ struct IDB_Decimal return (decimalComp(rhs) > 0); } } + bool operator<(const IDB_Decimal& rhs) const { if (utils::widthByPrecision(precision) == 16) @@ -165,6 +168,7 @@ struct IDB_Decimal return (decimalComp(rhs) < 0); } } + bool operator>=(const IDB_Decimal& rhs) const { if (utils::widthByPrecision(precision) == 16) @@ -182,6 +186,7 @@ struct IDB_Decimal return (decimalComp(rhs) >= 0); } } + bool operator<=(const IDB_Decimal& rhs) const { if (utils::widthByPrecision(precision) == 16) @@ -199,6 +204,7 @@ struct IDB_Decimal return (decimalComp(rhs) <= 0); } } + bool operator!=(const IDB_Decimal& rhs) const { if (utils::widthByPrecision(precision) == 16) diff --git a/dbcon/joblist/jlf_common.cpp b/dbcon/joblist/jlf_common.cpp index ef5964866..c07ab056e 100644 --- a/dbcon/joblist/jlf_common.cpp +++ b/dbcon/joblist/jlf_common.cpp @@ -333,7 +333,8 @@ string extractTableAlias(const SSC& sc) //------------------------------------------------------------------------------ CalpontSystemCatalog::OID isDictCol(const CalpontSystemCatalog::ColType& colType) { - if (datatypes::Decimal::isWideDecimalType(colType)) return 0; + if (datatypes::Decimal::isWideDecimalType(colType) || + colType.colDataType == CalpontSystemCatalog::BINARY) return 0; if (colType.colWidth > 8) return colType.ddn.dictOID; diff --git a/dbcon/joblist/jlf_execplantojoblist.cpp b/dbcon/joblist/jlf_execplantojoblist.cpp index 8b1211513..a184fbe24 100644 --- a/dbcon/joblist/jlf_execplantojoblist.cpp +++ b/dbcon/joblist/jlf_execplantojoblist.cpp @@ -130,9 +130,10 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo); /* This looks like an inefficient way to get NULL values. Much easier ways to do it. */ -int64_t valueNullNum(const CalpontSystemCatalog::ColType& ct, const string& timeZone) +template +void valueNullNum(const CalpontSystemCatalog::ColType& ct, const string& timeZone, T& val) { - int64_t n = 0; + T& n = val; bool pushWarning = false; boost::any anyVal = DataConvert::convertColumnData(ct, "", pushWarning, timeZone, true, false, false); @@ -292,24 +293,22 @@ int64_t valueNullNum(const CalpontSystemCatalog::ColType& ct, const string& time case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - n = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) - n = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::FOUR_BYTE) - n = boost::any_cast(anyVal); + if (LIKELY(ct.colWidth == datatypes::MAXDECIMALWIDTH)) + n = boost::any_cast(anyVal); else if (ct.colWidth == execplan::CalpontSystemCatalog::EIGHT_BYTE) n = boost::any_cast(anyVal); - else - n = 0xfffffffffffffffeLL; + else if (ct.colWidth == execplan::CalpontSystemCatalog::FOUR_BYTE) + n = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) + n = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) + n = boost::any_cast(anyVal); break; default: break; } - - return n; } template @@ -317,7 +316,7 @@ void convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct, { if (str.size() == 0 || isNull ) { - v = valueNullNum(ct, timeZone); + valueNullNum(ct, timeZone, v); return; } @@ -445,10 +444,10 @@ void convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct, case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) - v = boost::any_cast(anyVal); - else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) - v = boost::any_cast(anyVal); + if (LIKELY(ct.colWidth == datatypes::MAXDECIMALWIDTH)) + v = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::EIGHT_BYTE) + v = boost::any_cast(anyVal); else if (ct.colWidth == execplan::CalpontSystemCatalog::FOUR_BYTE) #ifdef _MSC_VER v = boost::any_cast(anyVal); @@ -456,10 +455,10 @@ void convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct, #else v = boost::any_cast(anyVal); #endif - else if (ct.colWidth == execplan::CalpontSystemCatalog::EIGHT_BYTE) - v = boost::any_cast(anyVal); - else - v = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE) + v = boost::any_cast(anyVal); + else if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE) + v = boost::any_cast(anyVal); break; diff --git a/dbcon/joblist/tupleconstantstep.cpp b/dbcon/joblist/tupleconstantstep.cpp index f6601cb1e..5b8f53810 100644 --- a/dbcon/joblist/tupleconstantstep.cpp +++ b/dbcon/joblist/tupleconstantstep.cpp @@ -529,7 +529,7 @@ void TupleConstantStep::fillInConstants(const rowgroup::Row& rowIn, rowgroup::Ro //fRowConst.copyField(rowOut.getData()+2, 0); // hardcoded 2 for rid length for (uint32_t i = 1; i < rowOut.getColumnCount(); i++) // WIP MCOL-641 implement copyBinaryField inside copyField - if (UNLIKELY(rowIn.getColumnWidth(i - 1) == 16)) + if (UNLIKELY(utils::isWide(rowIn.getColumnWidth(i - 1)))) rowIn.copyBinaryField(rowOut, i, i - 1); else rowIn.copyField(rowOut, i, i - 1); diff --git a/dbcon/mysql/ha_mcs_execplan.cpp b/dbcon/mysql/ha_mcs_execplan.cpp index bba566d7b..043da5168 100755 --- a/dbcon/mysql/ha_mcs_execplan.cpp +++ b/dbcon/mysql/ha_mcs_execplan.cpp @@ -4365,8 +4365,15 @@ ConstantColumn* buildDecimalColumn(Item* item, gp_walk_info& gwi) columnstore_decimal_val << str->ptr()[i]; } - columnstore_decimal.value = strtoll(columnstore_decimal_val.str().c_str(), 0, 10); + if (idp->decimal_precision() <= 18) + columnstore_decimal.value = strtoll(columnstore_decimal_val.str().c_str(), 0, 10); + else + { + bool dummy = false; + columnstore_decimal.s128Value = dataconvert::strtoll128(columnstore_decimal_val.str().c_str(), dummy, 0); + } + // TODO MCOL-641 Add support here if (gwi.internalDecimalScale >= 0 && idp->decimals > (uint)gwi.internalDecimalScale) { columnstore_decimal.scale = gwi.internalDecimalScale; diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index 1de0870f9..f80f9738b 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -805,7 +805,7 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - if (colType.colWidth == 16) + if (LIKELY(colType.colWidth == datatypes::MAXDECIMALWIDTH)) { // unset null bit first // Might be redundant diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 1161f2cda..d9724d727 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -41,6 +41,7 @@ using namespace boost; #include "primproc.h" #include "dataconvert.h" #include "widedecimalutils.h" +#include "mcs_decimal.h" using namespace logging; using namespace dbbc; using namespace primitives; @@ -639,8 +640,6 @@ inline string fixChar(int64_t intval) inline bool colCompare(int64_t val1, int64_t val2, uint8_t COP, uint8_t rf, int type, uint8_t width, const idb_regex_t& regex, bool isNull = false) { -// cout << "comparing " << hex << val1 << " to " << val2 << endl; - if (COMPARE_NIL == COP) return false; //@bug 425 added isNull condition @@ -689,6 +688,19 @@ inline bool colCompare(int64_t val1, int64_t val2, uint8_t COP, uint8_t rf, int } } +inline bool colCompare(int128_t val1, int128_t val2, uint8_t COP, uint8_t rf, int type, uint8_t width, bool isNull = false) +{ + if (COMPARE_NIL == COP) return false; + + /* isNullVal should work on the normalized value on little endian machines */ + bool val2Null = isNullVal(width, type, (uint8_t*) &val2); + + if (isNull == val2Null || (val2Null && COP == COMPARE_NE)) + return colCompare_(val1, val2, COP, rf); + else + return false; +} + inline bool colCompareUnsigned(uint64_t val1, uint64_t val2, uint8_t COP, uint8_t rf, int type, uint8_t width, const idb_regex_t& regex, bool isNull = false) { // cout << "comparing unsigned" << hex << val1 << " to " << val2 << endl; @@ -1602,7 +1614,7 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, int nextRidIndex = 0, argIndex = 0; bool done = false, cmp = false, isNull = false, isEmpty = false; uint16_t rid = 0; - prestored_set_t::const_iterator it; + prestored_set_t_128::const_iterator it; binWtype* argVals = (binWtype*)alloca(in->NOPS * W); uint8_t* std_cops = (uint8_t*)alloca(in->NOPS * sizeof(uint8_t)); @@ -1635,7 +1647,7 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, // we have a pre-parsed filter, and it's in the form of op and value arrays else if (parsedColumnFilter->columnFilterMode == TWO_ARRAYS) { - argVals = (binWtype*) parsedColumnFilter->prestored_argVals.get(); + argVals = (binWtype*) parsedColumnFilter->prestored_argVals128.get(); cops = parsedColumnFilter->prestored_cops.get(); rfs = parsedColumnFilter->prestored_rfs.get(); regex = parsedColumnFilter->prestored_regex.get(); @@ -1646,14 +1658,11 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, bval = (binWtype*)nextBinColValue(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull, &isEmpty, &rid, in->OutputType, reinterpret_cast(block), itemsPerBlk); - uint128_t val; - dataconvert::Int128Pod_t *valPod; - valPod = reinterpret_cast(&val); + int128_t val; while (!done) { - valPod->lo = *reinterpret_cast(*bval); - valPod->hi = *(reinterpret_cast(*bval) + 1); + val = *reinterpret_cast(bval); if (cops == NULL) // implies parsedColumnFilter && columnFilterMode == SET { @@ -1661,13 +1670,13 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, if (!(isNull && in->BOP == BOP_AND)) { - it = parsedColumnFilter->prestored_set->find(val); + it = parsedColumnFilter->prestored_set_128->find(val); if (in->BOP == BOP_OR) { // assume COP == COMPARE_EQ - if (it != parsedColumnFilter->prestored_set->end()) + if (it != parsedColumnFilter->prestored_set_128->end()) { store(in, out, outSize, written, rid, reinterpret_cast(block)); } @@ -1675,7 +1684,7 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, else if (in->BOP == BOP_AND) { // assume COP == COMPARE_NE - if (it == parsedColumnFilter->prestored_set->end()) + if (it == parsedColumnFilter->prestored_set_128->end()) { store(in, out, outSize, written, rid, reinterpret_cast(block)); } @@ -1686,45 +1695,11 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, { for (argIndex = 0; argIndex < in->NOPS; argIndex++) { - -// if((*((uint64_t *) (uval))) != 0) cout << "comparing " << dec << (*((uint64_t *) (uval))) << " to " << (*((uint64_t *) (argVals[argIndex]))) << endl; - // WIP MCOL-641 - uint128_t filterVal; - dataconvert::Int128Pod_t *filterValPod; - filterValPod = reinterpret_cast(&filterVal); + int128_t filterVal = *reinterpret_cast(argVals[argIndex]); - filterValPod->lo = *reinterpret_cast(argVals[argIndex]); - filterValPod->hi = *(reinterpret_cast(argVals[argIndex]) + 1); - - switch (cops[argIndex]) { - case COMPARE_NIL: - cmp = false; - break; - case COMPARE_LT: - cmp = val < filterVal; - break; - case COMPARE_EQ: - cmp = val == filterVal; - break; - case COMPARE_LE: - cmp = val <= filterVal; - break; - case COMPARE_GT: - cmp = val > filterVal; - break; - case COMPARE_NE: - cmp = val != filterVal; - break; - case COMPARE_GE: - cmp = val >= filterVal; - break; - default: - logIt(34, cops[argIndex], "colCompare"); - cmp = false; // throw an exception here? - } - -// cout << cmp << endl; + cmp = colCompare(val, filterVal, cops[argIndex], + rfs[argIndex], in->DataType, W, isNull); if (in->NOPS == 1) { @@ -1732,7 +1707,6 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, { store(in, out, outSize, written, rid, reinterpret_cast(block)); } - break; } else if (in->BOP == BOP_AND && cmp == false) @@ -1765,19 +1739,19 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in, } else if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType)) { - if (static_cast(out->Min) > val) - out->Min = static_cast(val); + if (static_cast(out->Min) > static_cast(val)) + out->Min = val; - if (static_cast(out->Max) < val) - out->Max = static_cast(val);; + if (static_cast(out->Max) < static_cast(val)) + out->Max = val; } else { - if (out->Min > (__int128) val) - out->Min = (__int128) val; + if (out->Min > val) + out->Min = val; - if (out->Max < (__int128) val) - out->Max = (__int128) val; + if (out->Max < val) + out->Max = val; } } @@ -1897,9 +1871,8 @@ boost::shared_ptr parseColumnFilter ret.reset(new ParsedColumnFilter()); ret->columnFilterMode = TWO_ARRAYS; - // WIP MCOL-641 - if (colWidth == 16) - ret->prestored_argVals.reset(new int64_t[filterCount * 2]); + if (colWidth == datatypes::MAXDECIMALWIDTH) + ret->prestored_argVals128.reset(new int128_t[filterCount]); else ret->prestored_argVals.reset(new int64_t[filterCount]); ret->prestored_cops.reset(new uint8_t[filterCount]); @@ -1997,9 +1970,8 @@ boost::shared_ptr parseColumnFilter ret->prestored_argVals[argIndex] = *reinterpret_cast(args->val); break; - // WIP MCOL-641 case 16: - memcpy(ret->prestored_argVals.get() + (argIndex * 2), args->val, 16); + ret->prestored_argVals128[argIndex] = *reinterpret_cast(args->val); } } @@ -2028,12 +2000,24 @@ boost::shared_ptr parseColumnFilter if (convertToSet) { ret->columnFilterMode = UNORDERED_SET; - ret->prestored_set.reset(new prestored_set_t()); + if (colWidth == datatypes::MAXDECIMALWIDTH) + { + ret->prestored_set_128.reset(new prestored_set_t_128()); - // @bug 2584, use COMPARE_NIL for "= null" to allow "is null" in OR expression - for (argIndex = 0; argIndex < filterCount; argIndex++) - if (ret->prestored_rfs[argIndex] == 0) - ret->prestored_set->insert(ret->prestored_argVals[argIndex]); + // @bug 2584, use COMPARE_NIL for "= null" to allow "is null" in OR expression + for (argIndex = 0; argIndex < filterCount; argIndex++) + if (ret->prestored_rfs[argIndex] == 0) + ret->prestored_set_128->insert(ret->prestored_argVals128[argIndex]); + } + else + { + ret->prestored_set.reset(new prestored_set_t()); + + // @bug 2584, use COMPARE_NIL for "= null" to allow "is null" in OR expression + for (argIndex = 0; argIndex < filterCount; argIndex++) + if (ret->prestored_rfs[argIndex] == 0) + ret->prestored_set->insert(ret->prestored_argVals[argIndex]); + } } return ret; diff --git a/primitives/linux-port/primitiveprocessor.h b/primitives/linux-port/primitiveprocessor.h index 0e8a93396..af0ea1909 100644 --- a/primitives/linux-port/primitiveprocessor.h +++ b/primitives/linux-port/primitiveprocessor.h @@ -82,7 +82,26 @@ public: } }; +class pcfHasher128 +{ +public: + inline size_t operator()(const int128_t i) const + { + return *reinterpret_cast(&i); + } +}; + +class pcfEqual128 +{ +public: + inline bool operator()(const int128_t f1, const int128_t f2) const + { + return f1 == f2; + } +}; + typedef std::tr1::unordered_set prestored_set_t; +typedef std::tr1::unordered_set prestored_set_t_128; typedef std::tr1::unordered_set DictEqualityFilter; struct idb_regex_t @@ -109,9 +128,11 @@ struct ParsedColumnFilter { ColumnFilterMode columnFilterMode; boost::shared_array prestored_argVals; + boost::shared_array prestored_argVals128; boost::shared_array prestored_cops; boost::shared_array prestored_rfs; boost::shared_ptr prestored_set; + boost::shared_ptr prestored_set_128; boost::shared_array prestored_regex; uint8_t likeOps; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 31e15e1c8..d0b88ffd1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -9,13 +9,13 @@ endif() if (WITH_ARITHMETICOPERATOR_UT) add_executable(arithmeticoperator_tests arithmeticoperator-tests.cpp) target_link_libraries(arithmeticoperator_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) - install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) + install(TARGETS arithmeticoperator_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) endif() if (WITH_CSDECIMAL_UT) add_executable(mcs_decimal_tests mcs_decimal-tests.cpp) target_link_libraries(mcs_decimal_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) - install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) + install(TARGETS mcs_decimal_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform) endif() if (WITH_DATACONVERT_UT) diff --git a/tests/dataconvert-tests.cpp b/tests/dataconvert-tests.cpp index 3e7f0d987..07b191b1a 100644 --- a/tests/dataconvert-tests.cpp +++ b/tests/dataconvert-tests.cpp @@ -729,5 +729,67 @@ TEST(DataConvertTest, DecimalToStringCheckScale38) EXPECT_EQ(string(buf), expected); } -TEST(DataConvertTest, ConvertColumnData) { +TEST(DataConvertTest, DecimalToStringCheckScale37) +{ + CalpontSystemCatalog::ColType ct; + ct.colDataType = CalpontSystemCatalog::DECIMAL; + char buf[42]; + string expected; + ct.precision = 38; + ct.scale = 37; + int128_t res; + + // test simple values + res = 0; + expected = "0.0000000000000000000000000000000000000"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = 2; + expected = "0.0000000000000000000000000000000000002"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -2; + expected = "-0.0000000000000000000000000000000000002"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = 123; + expected = "0.0000000000000000000000000000000000123"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -123; + expected = "-0.0000000000000000000000000000000000123"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = ((((((int128_t)1234567890 * 10000000000) + 1234567890) * 10000000000) + 1234567890) * 100000000 ) + 12345678; + expected = "1.2345678901234567890123456789012345678"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -res; + expected = "-1.2345678901234567890123456789012345678"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + + // test max/min decimal (i.e. 38 9's) + res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99; + expected = "9.9999999999999999999999999999999999999"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -res; + expected = "-9.9999999999999999999999999999999999999"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + + // test trailing zeros + res = 123000; + expected = "0.0000000000000000000000000000000123000"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); + res = -res; + expected = "-0.0000000000000000000000000000000123000"; + DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType); + EXPECT_EQ(string(buf), expected); +} + +TEST(DataConvertTest, ConvertColumnData) +{ } diff --git a/utils/common/columnwidth.h b/utils/common/columnwidth.h index 9d02a7312..d21e15230 100644 --- a/utils/common/columnwidth.h +++ b/utils/common/columnwidth.h @@ -23,7 +23,6 @@ namespace utils { const uint8_t MAXLEGACYWIDTH = 8ULL; - const uint8_t MAXCOLUMNWIDTH = 16ULL; inline bool isWide(uint8_t width) { diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 1a2086188..546c24057 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -391,29 +391,21 @@ void number_int_value(const string& data, case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - if (ct.colWidth == 1) + if (LIKELY(ct.colWidth == 16)) { - if (intVal < MIN_TINYINT) + int128_t tmp; + utils::int128Min(tmp); + if (intVal < tmp + 2) // + 2 for NULL and EMPTY values { - intVal = MIN_TINYINT; - pushwarning = true; - } - else if (intVal > MAX_TINYINT) - { - intVal = MAX_TINYINT; + intVal = tmp + 2; pushwarning = true; } } - else if (ct.colWidth == 2) + else if (ct.colWidth == 8) { - if (intVal < MIN_SMALLINT) + if (intVal < MIN_BIGINT) { - intVal = MIN_SMALLINT; - pushwarning = true; - } - else if (intVal > MAX_SMALLINT) - { - intVal = MAX_SMALLINT; + intVal = MIN_BIGINT; pushwarning = true; } } @@ -430,21 +422,29 @@ void number_int_value(const string& data, pushwarning = true; } } - else if (ct.colWidth == 8) + else if (ct.colWidth == 2) { - if (intVal < MIN_BIGINT) + if (intVal < MIN_SMALLINT) { - intVal = MIN_BIGINT; + intVal = MIN_SMALLINT; + pushwarning = true; + } + else if (intVal > MAX_SMALLINT) + { + intVal = MAX_SMALLINT; pushwarning = true; } } - else if (ct.colWidth == 16) + else if (ct.colWidth == 1) { - int128_t tmp; - utils::int128Min(tmp); - if (intVal < tmp + 2) // + 2 for NULL and EMPTY values + if (intVal < MIN_TINYINT) { - intVal = tmp + 2; + intVal = MIN_TINYINT; + pushwarning = true; + } + else if (intVal > MAX_TINYINT) + { + intVal = MAX_TINYINT; pushwarning = true; } } @@ -1469,31 +1469,31 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, break; case CalpontSystemCatalog::DECIMAL: - if (colType.colWidth == 1) + if (LIKELY(colType.colWidth == 16)) { - number_int_value(data, colType, pushWarning, noRoundup, val64); - value = (char) val64; - } - else if (colType.colWidth == 2) - { - number_int_value(data, colType, pushWarning, noRoundup, val64); - value = (short) val64; - } - else if (colType.colWidth == 4) - { - number_int_value(data, colType, pushWarning, noRoundup, val64); - value = (int) val64; + int128_t val128; + number_int_value(data, colType, pushWarning, noRoundup, val128); + value = (int128_t) val128; } else if (colType.colWidth == 8) { number_int_value(data, colType, pushWarning, noRoundup, val64); value = (long long) val64; } - else if (colType.colWidth == 16) + else if (colType.colWidth == 4) { - int128_t val128; - number_int_value(data, colType, pushWarning, noRoundup, val128); - value = (int128_t) val128; + number_int_value(data, colType, pushWarning, noRoundup, val64); + value = (int) val64; + } + else if (colType.colWidth == 2) + { + number_int_value(data, colType, pushWarning, noRoundup, val64); + value = (short) val64; + } + else if (colType.colWidth == 1) + { + number_int_value(data, colType, pushWarning, noRoundup, val64); + value = (char) val64; } //else if (colType.colWidth == 32) // value = data; @@ -1503,29 +1503,29 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, case CalpontSystemCatalog::UDECIMAL: // UDECIMAL numbers may not be negative - if (colType.colWidth == 1) + if (LIKELY(colType.colWidth == 16)) { - number_int_value(data, colType, pushWarning, noRoundup, val64); - char ival = (char) val64; + int128_t val128; + number_int_value(data, colType, pushWarning, noRoundup, val128); - if (ival < 0 && - ival != static_cast(joblist::TINYINTEMPTYROW) && - ival != static_cast(joblist::TINYINTNULL)) + if (val128 < 0 && + !utils::isWideDecimalNullValue(val128) && + !utils::isWideDecimalEmptyValue(val128)) { - ival = 0; + val128 = 0; pushWarning = true; } - value = ival; + value = val128; } - else if (colType.colWidth == 2) + else if (colType.colWidth == 8) { number_int_value(data, colType, pushWarning, noRoundup, val64); - short ival = (short) val64; + long long ival = static_cast(val64); if (ival < 0 && - ival != static_cast(joblist::SMALLINTEMPTYROW) && - ival != static_cast(joblist::SMALLINTNULL)) + ival != static_cast(joblist::BIGINTEMPTYROW) && + ival != static_cast(joblist::BIGINTNULL)) { ival = 0; pushWarning = true; @@ -1548,14 +1548,14 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, value = ival; } - else if (colType.colWidth == 8) + else if (colType.colWidth == 2) { number_int_value(data, colType, pushWarning, noRoundup, val64); - long long ival = static_cast(val64); + short ival = (short) val64; if (ival < 0 && - ival != static_cast(joblist::BIGINTEMPTYROW) && - ival != static_cast(joblist::BIGINTNULL)) + ival != static_cast(joblist::SMALLINTEMPTYROW) && + ival != static_cast(joblist::SMALLINTNULL)) { ival = 0; pushWarning = true; @@ -1563,20 +1563,20 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, value = ival; } - else if (colType.colWidth == 16) + else if (colType.colWidth == 1) { - int128_t val128; - number_int_value(data, colType, pushWarning, noRoundup, val128); + number_int_value(data, colType, pushWarning, noRoundup, val64); + char ival = (char) val64; - if (val128 < 0 && - !utils::isWideDecimalNullValue(val128) && - !utils::isWideDecimalEmptyValue(val128)) + if (ival < 0 && + ival != static_cast(joblist::TINYINTEMPTYROW) && + ival != static_cast(joblist::TINYINTNULL)) { - val128 = 0; + ival = 0; pushWarning = true; } - value = val128; + value = ival; } break; @@ -1865,31 +1865,31 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - if (colType.colWidth == CalpontSystemCatalog::ONE_BYTE) + if (LIKELY(colType.colWidth == 16)) { - char tinyintvalue = joblist::TINYINTNULL; - value = tinyintvalue; - } - else if (colType.colWidth == CalpontSystemCatalog::TWO_BYTE) - { - short smallintvalue = joblist::SMALLINTNULL; - value = smallintvalue; - } - else if (colType.colWidth == CalpontSystemCatalog::FOUR_BYTE) - { - int intvalue = joblist::INTNULL; - value = intvalue; + int128_t val; + utils::setWideDecimalNullValue(val); + value = val; } else if (colType.colWidth == CalpontSystemCatalog::EIGHT_BYTE) { long long eightbyte = joblist::BIGINTNULL; value = eightbyte; } - else if (colType.colWidth == 16) + else if (colType.colWidth == CalpontSystemCatalog::FOUR_BYTE) { - int128_t val; - utils::setWideDecimalNullValue(val); - value = val; + int intvalue = joblist::INTNULL; + value = intvalue; + } + else if (colType.colWidth == CalpontSystemCatalog::TWO_BYTE) + { + short smallintvalue = joblist::SMALLINTNULL; + value = smallintvalue; + } + else if (colType.colWidth == CalpontSystemCatalog::ONE_BYTE) + { + char tinyintvalue = joblist::TINYINTNULL; + value = tinyintvalue; } else { diff --git a/utils/joiner/tuplejoiner.cpp b/utils/joiner/tuplejoiner.cpp index 6297699a6..9282e7253 100644 --- a/utils/joiner/tuplejoiner.cpp +++ b/utils/joiner/tuplejoiner.cpp @@ -1147,7 +1147,7 @@ void TupleJoiner::updateCPData(const Row& r) } } } - else if (r.getColumnWidth(colIdx) == utils::MAXCOLUMNWIDTH + else if (r.getColumnWidth(colIdx) == datatypes::MAXDECIMALWIDTH && (r.getColType(colIdx) == CalpontSystemCatalog::DECIMAL || r.getColType(colIdx) == CalpontSystemCatalog::UDECIMAL)) { diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 3bb0f95ea..e8dc38f50 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -639,9 +639,9 @@ string Row::toString() const break; case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: - if (colWidths[i] == utils::MAXCOLUMNWIDTH) + if (colWidths[i] == datatypes::MAXDECIMALWIDTH) { - unsigned int buflen = precision[i] + 3; + unsigned int buflen = utils::MAXLENGTH16BYTES; char *buf = (char*)alloca(buflen); // empty the buffer dataconvert::DataConvert::decimalToString(getBinaryField(i), diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index f3c7fec39..edd552c93 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -1235,7 +1235,7 @@ inline void Row::copyField(Row& out, uint32_t destIndex, uint32_t srcIndex) cons // WIP MCOL-641 inline void Row::copyBinaryField(Row& out, uint32_t destIndex, uint32_t srcIndex) const { - out.setBinaryField(getBinaryField(srcIndex), 16, destIndex); + out.setBinaryField(getBinaryField(srcIndex), 16, destIndex); } inline void Row::setRid(uint64_t rid) diff --git a/writeengine/bulk/we_bulkloadbuffer.cpp b/writeengine/bulk/we_bulkloadbuffer.cpp index b899dcd8b..79d6cf275 100644 --- a/writeengine/bulk/we_bulkloadbuffer.cpp +++ b/writeengine/bulk/we_bulkloadbuffer.cpp @@ -39,6 +39,7 @@ #include "brmtypes.h" #include "dataconvert.h" #include "exceptclasses.h" +#include "mcs_decimal.h" #include "joblisttypes.h" @@ -986,18 +987,18 @@ void BulkLoadBuffer::convert(char* field, int fieldLength, if ( (column.dataType == CalpontSystemCatalog::DECIMAL) || (column.dataType == CalpontSystemCatalog::UDECIMAL)) { - if (width <= 8) - { - // errno is initialized and set in convertDecimalString - llVal = Convertor::convertDecimalString( - field, fieldLength, column.scale ); - } - else + if (LIKELY(width == datatypes::MAXDECIMALWIDTH)) { bool saturate = false; bigllVal = dataconvert::string_to_ll(string(field), saturate); // TODO MCOL-641 check saturate } + else if (width <= 8) + { + // errno is initialized and set in convertDecimalString + llVal = Convertor::convertDecimalString( + field, fieldLength, column.scale ); + } } else { diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 3f3f732a2..dbbc5162c 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -2622,7 +2622,6 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, uint64_t& blocksChanged) { uint8_t rc = 0; - //cout << " In processUpdate" << endl; uint32_t tmp32, sessionID; TxnID txnId; bs >> PMId; @@ -2639,7 +2638,6 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, uint8_t pkgType; bs >> pkgType; cpackages[txnId].read(bs); - //cout << "Processed meta data in update" << endl; rc = fWEWrapper.startTransaction(txnId); @@ -2811,9 +2809,20 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, rid = relativeRID; convertToRelativeRid (rid, extentNum, blockNum); rowIDLists.push_back(rid); - uint32_t colWidth = ((colTypes[j].colWidth > 8 && - !(colTypes[j].colDataType == CalpontSystemCatalog::DECIMAL || - colTypes[j].colDataType == CalpontSystemCatalog::UDECIMAL)) ? 8 : colTypes[j].colWidth); + + uint32_t colWidth = colTypes[j].colWidth; + + if (colWidth > 8 && + !(colTypes[j].colDataType == CalpontSystemCatalog::DECIMAL || + colTypes[j].colDataType == CalpontSystemCatalog::UDECIMAL)) + { + colWidth = 8; + } + else if (colWidth >= datatypes::MAXDECIMALWIDTH) + { + colWidth = datatypes::MAXDECIMALWIDTH; + } + int rrid = (int) relativeRID / (BYTE_PER_BLOCK / colWidth); // populate stats.blocksChanged if (rrid > preBlkNums[j]) @@ -2984,16 +2993,18 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, case CalpontSystemCatalog::UDECIMAL: { // WIP MCOL-641 - if (fetchColColwidths[fetchColPos] == 16) + if (fetchColColwidths[fetchColPos] == datatypes::MAXDECIMALWIDTH) { int128_t* dec; char buf[utils::MAXLENGTH16BYTES]; dec = row.getBinaryField(fetchColPos); + dataconvert::DataConvert::decimalToString(dec, (unsigned)fetchColScales[fetchColPos], buf, sizeof(buf), fetchColTypes[fetchColPos]); value.assign(buf); + break; } @@ -3347,17 +3358,19 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - if (fetchColColwidths[fetchColPos] == 16) + if (fetchColColwidths[fetchColPos] == datatypes::MAXDECIMALWIDTH) { // WIP MCOL-641 int128_t* dec; char buf[utils::MAXLENGTH16BYTES]; dec = row.getBinaryField(fetchColPos); + dataconvert::DataConvert::decimalToString(dec, (unsigned)fetchColScales[fetchColPos], buf, sizeof(buf), fetchColTypes[fetchColPos]); value = buf; + break; } @@ -3994,7 +4007,6 @@ uint8_t WE_DMLCommandProc::processDelete(messageqcpp::ByteStream& bs, uint64_t& blocksChanged) { uint8_t rc = 0; - //cout << " In processDelete" << endl; uint32_t tmp32, sessionID; TxnID txnId; bs >> PMId; @@ -4069,7 +4081,18 @@ uint8_t WE_DMLCommandProc::processDelete(messageqcpp::ByteStream& bs, for (uint32_t j = 0; j < row.getColumnCount(); j++) { preBlkNums[j] = -1; - colWidth[j] = (row.getColumnWidth(j) >= 16 ? 16 : row.getColumnWidth(j)); + colWidth[j] = row.getColumnWidth(j); + execplan::CalpontSystemCatalog::ColDataType colDataType = row.getColType(j); + if (colWidth[j] >= 8 && + !(colDataType == execplan::CalpontSystemCatalog::DECIMAL || + colDataType == execplan::CalpontSystemCatalog::UDECIMAL)) + { + colWidth[j] = 8; + } + else if (colWidth[j] >= datatypes::MAXDECIMALWIDTH) + { + colWidth[j] = datatypes::MAXDECIMALWIDTH; + } } //Get the file information from rowgroup diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index a89b48199..bb7489a48 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -70,9 +70,6 @@ namespace WriteEngine { StopWatch timer; -using dataconvert::int128_t; -using dataconvert::uint128_t; - /**@brief WriteEngineWrapper Constructor */ WriteEngineWrapper::WriteEngineWrapper() : m_opType(NOOP) @@ -803,21 +800,9 @@ int WriteEngineWrapper::deleteRow(const TxnID& txnid, const vector cscColType = cscColTypeList[i]; Convertor::convertColType(&curColStruct); - /*if (curColStruct.colType == WriteEngine::WR_BINARY) - { - uint128_t bigEmptyVal; - emptyVal = m_colOp[op(curColStruct.fCompressionType)]-> - getEmptyRowValue(curColStruct.colDataType, curColStruct.colWidth); - *(reinterpret_cast(&bigEmptyVal)) = emptyVal; - *(reinterpret_cast(&bigEmptyVal) + 1) = emptyVal; - curTuple.data = bigEmptyVal; - } - else - {*/ - m_colOp[op(curColStruct.fCompressionType)]-> - getEmptyRowValue(curColStruct.colDataType, curColStruct.colWidth, (uint8_t*)&emptyVal); - curTuple.data = emptyVal; - //} + m_colOp[op(curColStruct.fCompressionType)]-> + getEmptyRowValue(curColStruct.colDataType, curColStruct.colWidth, (uint8_t*)&emptyVal); + curTuple.data = emptyVal; curTupleList.push_back(curTuple); colValueList.push_back(curTupleList); @@ -4371,7 +4356,6 @@ int WriteEngineWrapper::updateColumnRecs(const TxnID& txnid, ColumnOp* colOp = NULL; bool successFlag = true; unsigned width = 0; - \ int curFbo = 0, curBio, lastFbo = -1; RID aRid = ridLists[0]; int rc = 0;