database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-05 16:15:50 +03:00
Code Activity

MCOL-641 1. Add support for int128_t in ParsedColumnFilter.

Browse Source 
2. Set Decimal precision in SimpleColumn::evaluate().
3. Add support for int128_t in ConstantColumn.
4. Set IDB_Decimal::s128Value in buildDecimalColumn().
5. Use width 16 as first if predicate for branching based on decimal width.
This commit is contained in:
Gagan Goel
2020-03-13 15:42:25 -04:00
committed by Roman Nozdrin
parent 0bd172cd6e
commit 74b64eb4f1
21 changed files with 348 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
datatypes/mcs_decimal.cpp
View File

@@ -24,6 +24,27 @@
namespace datatypes
{
struct lldiv_t_128
{
int128_t quot;
int128_t rem;
lldiv_t_128() : quot(0), rem(0) {}
};
inline lldiv_t_128 lldiv128(const int128_t& dividend, const int128_t& divisor)
{
lldiv_t_128 res;
if (UNLIKELY(divisor == 0) || UNLIKELY(dividend == 0))
return res;
res.quot = dividend / divisor;
res.rem = dividend % divisor;
return res;
}
template<typename BinaryOperation,
typename OpOverflowCheck,
typename MultiplicationOverflowCheck>
@@ -94,42 +115,42 @@ namespace datatypes
int Decimal::compare(const execplan::IDB_Decimal& l, const execplan::IDB_Decimal& r)
{
int128_t divL, divR;
getScaleDivisor(divL, l.scale);
getScaleDivisor(divR, r.scale);
int128_t quotinentL, quotinentR, remainderL, remainderR;
quotinentL = l.s128Value/divL;
remainderL = l.s128Value%divL;
quotinentR = r.s128Value/divR;
remainderR = r.s128Value%divR;
int128_t divisorL, divisorR;
getScaleDivisor(divisorL, l.scale);
getScaleDivisor(divisorR, r.scale);
lldiv_t_128 d1 = lldiv128(l.s128Value, divisorL);
lldiv_t_128 d2 = lldiv128(r.s128Value, divisorR);
int ret = 0;
if (quotinentL > quotinentR)
if (d1.quot > d2.quot)
{
ret = 1;
}
else if (quotinentL < quotinentR)
else if (d1.quot < d2.quot)
{
ret = -1;
}
else
{
// rem carries the value's sign, but needs to be normalized.
int32_t s = l.scale - r.scale;
int64_t s = l.scale - r.scale;
int128_t divisor;
getScaleDivisor(divisor, abs(s));
if (s < 0)
{
if ((remainderL * mcs_pow_10[-s]) > remainderR)
if ((d1.rem * divisor) > d2.rem)
ret = 1;
else if ((remainderL * mcs_pow_10[-s]) < remainderR)
else if ((d1.rem * divisor) < d2.rem)
ret = -1;
}
else
{
if (remainderL > (remainderR * mcs_pow_10[s]))
if (d1.rem > (d2.rem * divisor))
ret = 1;
else if (remainderL < (remainderR * mcs_pow_10[s]))
else if (d1.rem < (d2.rem * divisor))
ret = -1;
}
}

4
dbcon/execplan/constantcolumn.cpp
View File

@@ -230,6 +230,7 @@ ConstantColumn::ConstantColumn(const int64_t val, TYPE type) :
fResult.doubleVal = (double)fResult.intVal;
fResult.longDoubleVal = (long double)fResult.intVal;
fResult.decimalVal.value = fResult.intVal;
fResult.decimalVal.s128Value = fResult.intVal;
fResult.decimalVal.scale = 0;
fResultType.colDataType = CalpontSystemCatalog::BIGINT;
fResultType.colWidth = 8;
@@ -250,6 +251,7 @@ ConstantColumn::ConstantColumn(const uint64_t val, TYPE type) :
fResult.doubleVal = (double)fResult.uintVal;
fResult.longDoubleVal = (long double)fResult.uintVal;
fResult.decimalVal.value = fResult.uintVal;
fResult.decimalVal.s128Value = fResult.uintVal;
fResult.decimalVal.scale = 0;
fResultType.colDataType = CalpontSystemCatalog::UBIGINT;
fResultType.colWidth = 8;
@@ -317,6 +319,7 @@ void ConstantColumn::serialize(messageqcpp::ByteStream& b) const
b << (uint8_t)fResult.boolVal;
b << fResult.strVal;
b << (uint64_t)fResult.decimalVal.value;
b << (uint128_t)fResult.decimalVal.s128Value;
b << (uint8_t)fResult.decimalVal.scale;
b << (uint8_t)fResult.decimalVal.precision;
}
@@ -341,6 +344,7 @@ void ConstantColumn::unserialize(messageqcpp::ByteStream& b)
b >> (uint8_t&)fResult.boolVal;
b >> fResult.strVal;
b >> (uint64_t&)fResult.decimalVal.value;
b >> (uint128_t&)fResult.decimalVal.s128Value;
b >> (uint8_t&)fResult.decimalVal.scale;
b >> (uint8_t&)fResult.decimalVal.precision;
}

8
dbcon/execplan/simplecolumn.cpp
View File

@@ -637,38 +637,36 @@ void SimpleColumn::evaluate(Row& row, bool& isNull)
{
fResult.decimalVal.s128Value =
*row.getBinaryField_offset<decltype(fResult.decimalVal.s128Value)>(fInputOffset);
fResult.decimalVal.scale = (unsigned)fResultType.scale;
break;
}
case 1:
{
fResult.decimalVal.value = row.getIntField<1>(fInputIndex);
fResult.decimalVal.scale = (unsigned)fResultType.scale;
break;
}
case 2:
{
fResult.decimalVal.value = row.getIntField<2>(fInputIndex);
fResult.decimalVal.scale = (unsigned)fResultType.scale;
break;
}
case 4:
{
fResult.decimalVal.value = row.getIntField<4>(fInputIndex);
fResult.decimalVal.scale = (unsigned)fResultType.scale;
break;
}
default:
{
fResult.decimalVal.value = (int64_t)row.getUintField<8>(fInputIndex);
fResult.decimalVal.scale = (unsigned)fResultType.scale;
break;
}
}
fResult.decimalVal.scale = (unsigned)fResultType.scale;
fResult.decimalVal.precision = (unsigned)fResultType.precision;
break;
}

6
dbcon/execplan/treenode.h
View File

@@ -67,6 +67,7 @@ struct IDB_Decimal
{
s128Value = 0;
}
IDB_Decimal(int64_t val, int8_t s, uint8_t p) :
value (val),
scale(s),
@@ -131,6 +132,7 @@ struct IDB_Decimal
return (decimalComp(rhs) == 0);
}
}
bool operator>(const IDB_Decimal& rhs) const
{
if (utils::widthByPrecision(precision) == 16)
@@ -148,6 +150,7 @@ struct IDB_Decimal
return (decimalComp(rhs) > 0);
}
}
bool operator<(const IDB_Decimal& rhs) const
{
if (utils::widthByPrecision(precision) == 16)
@@ -165,6 +168,7 @@ struct IDB_Decimal
return (decimalComp(rhs) < 0);
}
}
bool operator>=(const IDB_Decimal& rhs) const
{
if (utils::widthByPrecision(precision) == 16)
@@ -182,6 +186,7 @@ struct IDB_Decimal
return (decimalComp(rhs) >= 0);
}
}
bool operator<=(const IDB_Decimal& rhs) const
{
if (utils::widthByPrecision(precision) == 16)
@@ -199,6 +204,7 @@ struct IDB_Decimal
return (decimalComp(rhs) <= 0);
}
}
bool operator!=(const IDB_Decimal& rhs) const
{
if (utils::widthByPrecision(precision) == 16)

3
dbcon/joblist/jlf_common.cpp
View File

@@ -333,7 +333,8 @@ string extractTableAlias(const SSC& sc)
//------------------------------------------------------------------------------
CalpontSystemCatalog::OID isDictCol(const CalpontSystemCatalog::ColType& colType)
{
if (datatypes::Decimal::isWideDecimalType(colType)) return 0;
if (datatypes::Decimal::isWideDecimalType(colType) ||
colType.colDataType == CalpontSystemCatalog::BINARY) return 0;
if (colType.colWidth > 8) return colType.ddn.dictOID;

41
dbcon/joblist/jlf_execplantojoblist.cpp
View File

@@ -130,9 +130,10 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo);
/* This looks like an inefficient way to get NULL values. Much easier ways
to do it. */
int64_t valueNullNum(const CalpontSystemCatalog::ColType& ct, const string& timeZone)
template<typename T>
void valueNullNum(const CalpontSystemCatalog::ColType& ct, const string& timeZone, T& val)
{
int64_t n = 0;
T& n = val;
bool pushWarning = false;
boost::any anyVal = DataConvert::convertColumnData(ct, "", pushWarning, timeZone, true, false, false);
@@ -292,24 +293,22 @@ int64_t valueNullNum(const CalpontSystemCatalog::ColType& ct, const string& time
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE)
n = boost::any_cast<char>(anyVal);
else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE)
n = boost::any_cast<short>(anyVal);
else if (ct.colWidth == execplan::CalpontSystemCatalog::FOUR_BYTE)
n = boost::any_cast<int>(anyVal);
if (LIKELY(ct.colWidth == datatypes::MAXDECIMALWIDTH))
n = boost::any_cast<int128_t>(anyVal);
else if (ct.colWidth == execplan::CalpontSystemCatalog::EIGHT_BYTE)
n = boost::any_cast<long long>(anyVal);
else
n = 0xfffffffffffffffeLL;
else if (ct.colWidth == execplan::CalpontSystemCatalog::FOUR_BYTE)
n = boost::any_cast<int>(anyVal);
else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE)
n = boost::any_cast<short>(anyVal);
else if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE)
n = boost::any_cast<char>(anyVal);
break;
default:
break;
}
return n;
}
template <typename T>
@@ -317,7 +316,7 @@ void convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct,
{
if (str.size() == 0 || isNull )
{
v = valueNullNum(ct, timeZone);
valueNullNum(ct, timeZone, v);
return;
}
@@ -445,10 +444,10 @@ void convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct,
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE)
v = boost::any_cast<char>(anyVal);
else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE)
v = boost::any_cast<int16_t>(anyVal);
if (LIKELY(ct.colWidth == datatypes::MAXDECIMALWIDTH))
v = boost::any_cast<int128_t>(anyVal);
else if (ct.colWidth == execplan::CalpontSystemCatalog::EIGHT_BYTE)
v = boost::any_cast<long long>(anyVal);
else if (ct.colWidth == execplan::CalpontSystemCatalog::FOUR_BYTE)
#ifdef _MSC_VER
v = boost::any_cast<int>(anyVal);
@@ -456,10 +455,10 @@ void convertValueNum(const string& str, const CalpontSystemCatalog::ColType& ct,
#else
v = boost::any_cast<int32_t>(anyVal);
#endif
else if (ct.colWidth == execplan::CalpontSystemCatalog::EIGHT_BYTE)
v = boost::any_cast<long long>(anyVal);
else
v = boost::any_cast<int128_t>(anyVal);
else if (ct.colWidth == execplan::CalpontSystemCatalog::TWO_BYTE)
v = boost::any_cast<int16_t>(anyVal);
else if (ct.colWidth == execplan::CalpontSystemCatalog::ONE_BYTE)
v = boost::any_cast<char>(anyVal);
break;

2
dbcon/joblist/tupleconstantstep.cpp
View File

@@ -529,7 +529,7 @@ void TupleConstantStep::fillInConstants(const rowgroup::Row& rowIn, rowgroup::Ro
//fRowConst.copyField(rowOut.getData()+2, 0); // hardcoded 2 for rid length
for (uint32_t i = 1; i < rowOut.getColumnCount(); i++)
// WIP MCOL-641 implement copyBinaryField inside copyField
if (UNLIKELY(rowIn.getColumnWidth(i - 1) == 16))
if (UNLIKELY(utils::isWide(rowIn.getColumnWidth(i - 1))))
rowIn.copyBinaryField(rowOut, i, i - 1);
else
rowIn.copyField(rowOut, i, i - 1);

7
dbcon/mysql/ha_mcs_execplan.cpp
View File

@@ -4365,8 +4365,15 @@ ConstantColumn* buildDecimalColumn(Item* item, gp_walk_info& gwi)
columnstore_decimal_val << str->ptr()[i];
}
if (idp->decimal_precision() <= 18)
columnstore_decimal.value = strtoll(columnstore_decimal_val.str().c_str(), 0, 10);
else
{
bool dummy = false;
columnstore_decimal.s128Value = dataconvert::strtoll128(columnstore_decimal_val.str().c_str(), dummy, 0);
}
// TODO MCOL-641 Add support here
if (gwi.internalDecimalScale >= 0 && idp->decimals > (uint)gwi.internalDecimalScale)
{
columnstore_decimal.scale = gwi.internalDecimalScale;

2
dbcon/mysql/ha_mcs_impl.cpp
View File

@@ -805,7 +805,7 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
if (colType.colWidth == 16)
if (LIKELY(colType.colWidth == datatypes::MAXDECIMALWIDTH))
{
// unset null bit first
// Might be redundant

110
primitives/linux-port/column.cpp
View File

@@ -41,6 +41,7 @@ using namespace boost;
#include "primproc.h"
#include "dataconvert.h"
#include "widedecimalutils.h"
#include "mcs_decimal.h"
using namespace logging;
using namespace dbbc;
using namespace primitives;
@@ -639,8 +640,6 @@ inline string fixChar(int64_t intval)
inline bool colCompare(int64_t val1, int64_t val2, uint8_t COP, uint8_t rf, int type, uint8_t width, const idb_regex_t& regex, bool isNull = false)
{
// cout << "comparing " << hex << val1 << " to " << val2 << endl;
if (COMPARE_NIL == COP) return false;
//@bug 425 added isNull condition
@@ -689,6 +688,19 @@ inline bool colCompare(int64_t val1, int64_t val2, uint8_t COP, uint8_t rf, int
}
}
inline bool colCompare(int128_t val1, int128_t val2, uint8_t COP, uint8_t rf, int type, uint8_t width, bool isNull = false)
{
if (COMPARE_NIL == COP) return false;
/* isNullVal should work on the normalized value on little endian machines */
bool val2Null = isNullVal(width, type, (uint8_t*) &val2);
if (isNull == val2Null || (val2Null && COP == COMPARE_NE))
return colCompare_(val1, val2, COP, rf);
else
return false;
}
inline bool colCompareUnsigned(uint64_t val1, uint64_t val2, uint8_t COP, uint8_t rf, int type, uint8_t width, const idb_regex_t& regex, bool isNull = false)
{
// cout << "comparing unsigned" << hex << val1 << " to " << val2 << endl;
@@ -1602,7 +1614,7 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in,
int nextRidIndex = 0, argIndex = 0;
bool done = false, cmp = false, isNull = false, isEmpty = false;
uint16_t rid = 0;
prestored_set_t::const_iterator it;
prestored_set_t_128::const_iterator it;
binWtype* argVals = (binWtype*)alloca(in->NOPS * W);
uint8_t* std_cops = (uint8_t*)alloca(in->NOPS * sizeof(uint8_t));
@@ -1635,7 +1647,7 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in,
// we have a pre-parsed filter, and it's in the form of op and value arrays
else if (parsedColumnFilter->columnFilterMode == TWO_ARRAYS)
{
argVals = (binWtype*) parsedColumnFilter->prestored_argVals.get();
argVals = (binWtype*) parsedColumnFilter->prestored_argVals128.get();
cops = parsedColumnFilter->prestored_cops.get();
rfs = parsedColumnFilter->prestored_rfs.get();
regex = parsedColumnFilter->prestored_regex.get();
@@ -1646,14 +1658,11 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in,
bval = (binWtype*)nextBinColValue<W>(in->DataType, ridArray, in->NVALS, &nextRidIndex, &done, &isNull,
&isEmpty, &rid, in->OutputType, reinterpret_cast<uint8_t*>(block), itemsPerBlk);
uint128_t val;
dataconvert::Int128Pod_t *valPod;
valPod = reinterpret_cast<dataconvert::Int128Pod_t*>(&val);
int128_t val;
while (!done)
{
valPod->lo = *reinterpret_cast<uint64_t*>(*bval);
valPod->hi = *(reinterpret_cast<uint64_t*>(*bval) + 1);
val = *reinterpret_cast<int128_t*>(bval);
if (cops == NULL) // implies parsedColumnFilter && columnFilterMode == SET
{
@@ -1661,13 +1670,13 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in,
if (!(isNull && in->BOP == BOP_AND))
{
it = parsedColumnFilter->prestored_set->find(val);
it = parsedColumnFilter->prestored_set_128->find(val);
if (in->BOP == BOP_OR)
{
// assume COP == COMPARE_EQ
if (it != parsedColumnFilter->prestored_set->end())
if (it != parsedColumnFilter->prestored_set_128->end())
{
store(in, out, outSize, written, rid, reinterpret_cast<const uint8_t*>(block));
}
@@ -1675,7 +1684,7 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in,
else if (in->BOP == BOP_AND)
{
// assume COP == COMPARE_NE
if (it == parsedColumnFilter->prestored_set->end())
if (it == parsedColumnFilter->prestored_set_128->end())
{
store(in, out, outSize, written, rid, reinterpret_cast<const uint8_t*>(block));
}
@@ -1686,45 +1695,11 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in,
{
for (argIndex = 0; argIndex < in->NOPS; argIndex++)
{
// if((*((uint64_t *) (uval))) != 0) cout << "comparing " << dec << (*((uint64_t *) (uval))) << " to " << (*((uint64_t *) (argVals[argIndex]))) << endl;
// WIP MCOL-641
uint128_t filterVal;
dataconvert::Int128Pod_t *filterValPod;
filterValPod = reinterpret_cast<dataconvert::Int128Pod_t*>(&filterVal);
int128_t filterVal = *reinterpret_cast<int128_t*>(argVals[argIndex]);
filterValPod->lo = *reinterpret_cast<uint64_t*>(argVals[argIndex]);
filterValPod->hi = *(reinterpret_cast<uint64_t*>(argVals[argIndex]) + 1);
switch (cops[argIndex]) {
case COMPARE_NIL:
cmp = false;
break;
case COMPARE_LT:
cmp = val < filterVal;
break;
case COMPARE_EQ:
cmp = val == filterVal;
break;
case COMPARE_LE:
cmp = val <= filterVal;
break;
case COMPARE_GT:
cmp = val > filterVal;
break;
case COMPARE_NE:
cmp = val != filterVal;
break;
case COMPARE_GE:
cmp = val >= filterVal;
break;
default:
logIt(34, cops[argIndex], "colCompare");
cmp = false; // throw an exception here?
}
// cout << cmp << endl;
cmp = colCompare(val, filterVal, cops[argIndex],
rfs[argIndex], in->DataType, W, isNull);
if (in->NOPS == 1)
{
@@ -1732,7 +1707,6 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in,
{
store(in, out, outSize, written, rid, reinterpret_cast<const uint8_t*>(block));
}
break;
}
else if (in->BOP == BOP_AND && cmp == false)
@@ -1765,19 +1739,19 @@ inline void p_Col_bin_ridArray(NewColRequestHeader* in,
}
else if (isUnsigned((CalpontSystemCatalog::ColDataType)in->DataType))
{
if (static_cast<unsigned __int128>(out->Min) > val)
out->Min = static_cast<unsigned __int128>(val);
if (static_cast<uint128_t>(out->Min) > static_cast<uint128_t>(val))
out->Min = val;
if (static_cast<unsigned __int128>(out->Max) < val)
out->Max = static_cast<unsigned __int128>(val);;
if (static_cast<uint128_t>(out->Max) < static_cast<uint128_t>(val))
out->Max = val;
}
else
{
if (out->Min > (__int128) val)
out->Min = (__int128) val;
if (out->Min > val)
out->Min = val;
if (out->Max < (__int128) val)
out->Max = (__int128) val;
if (out->Max < val)
out->Max = val;
}
}
@@ -1897,9 +1871,8 @@ boost::shared_ptr<ParsedColumnFilter> parseColumnFilter
ret.reset(new ParsedColumnFilter());
ret->columnFilterMode = TWO_ARRAYS;
// WIP MCOL-641
if (colWidth == 16)
ret->prestored_argVals.reset(new int64_t[filterCount * 2]);
if (colWidth == datatypes::MAXDECIMALWIDTH)
ret->prestored_argVals128.reset(new int128_t[filterCount]);
else
ret->prestored_argVals.reset(new int64_t[filterCount]);
ret->prestored_cops.reset(new uint8_t[filterCount]);
@@ -1997,9 +1970,8 @@ boost::shared_ptr<ParsedColumnFilter> parseColumnFilter
ret->prestored_argVals[argIndex] = *reinterpret_cast<const int64_t*>(args->val);
break;
// WIP MCOL-641
case 16:
memcpy(ret->prestored_argVals.get() + (argIndex * 2), args->val, 16);
ret->prestored_argVals128[argIndex] = *reinterpret_cast<const int128_t*>(args->val);
}
}
@@ -2028,6 +2000,17 @@ boost::shared_ptr<ParsedColumnFilter> parseColumnFilter
if (convertToSet)
{
ret->columnFilterMode = UNORDERED_SET;
if (colWidth == datatypes::MAXDECIMALWIDTH)
{
ret->prestored_set_128.reset(new prestored_set_t_128());
// @bug 2584, use COMPARE_NIL for "= null" to allow "is null" in OR expression
for (argIndex = 0; argIndex < filterCount; argIndex++)
if (ret->prestored_rfs[argIndex] == 0)
ret->prestored_set_128->insert(ret->prestored_argVals128[argIndex]);
}
else
{
ret->prestored_set.reset(new prestored_set_t());
// @bug 2584, use COMPARE_NIL for "= null" to allow "is null" in OR expression
@@ -2035,6 +2018,7 @@ boost::shared_ptr<ParsedColumnFilter> parseColumnFilter
if (ret->prestored_rfs[argIndex] == 0)
ret->prestored_set->insert(ret->prestored_argVals[argIndex]);
}
}
return ret;
}

21
primitives/linux-port/primitiveprocessor.h
View File

@@ -82,7 +82,26 @@ public:
}
};
class pcfHasher128
{
public:
inline size_t operator()(const int128_t i) const
{
return *reinterpret_cast<const uint64_t*>(&i);
}
};
class pcfEqual128
{
public:
inline bool operator()(const int128_t f1, const int128_t f2) const
{
return f1 == f2;
}
};
typedef std::tr1::unordered_set<int64_t, pcfHasher, pcfEqual> prestored_set_t;
typedef std::tr1::unordered_set<int128_t, pcfHasher128, pcfEqual128> prestored_set_t_128;
typedef std::tr1::unordered_set<std::string, utils::Hasher> DictEqualityFilter;
struct idb_regex_t
@@ -109,9 +128,11 @@ struct ParsedColumnFilter
{
ColumnFilterMode columnFilterMode;
boost::shared_array<int64_t> prestored_argVals;
boost::shared_array<int128_t> prestored_argVals128;
boost::shared_array<uint8_t> prestored_cops;
boost::shared_array<uint8_t> prestored_rfs;
boost::shared_ptr<prestored_set_t> prestored_set;
boost::shared_ptr<prestored_set_t_128> prestored_set_128;
boost::shared_array<idb_regex_t> prestored_regex;
uint8_t likeOps;

4
tests/CMakeLists.txt
View File

@@ -9,13 +9,13 @@ endif()
if (WITH_ARITHMETICOPERATOR_UT)
add_executable(arithmeticoperator_tests arithmeticoperator-tests.cpp)
target_link_libraries(arithmeticoperator_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS})
install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform)
install(TARGETS arithmeticoperator_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform)
endif()
if (WITH_CSDECIMAL_UT)
add_executable(mcs_decimal_tests mcs_decimal-tests.cpp)
target_link_libraries(mcs_decimal_tests ${ENGINE_LDFLAGS} ${GTEST_LIBRARIES} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS})
install(TARGETS rowgroup_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform)
install(TARGETS mcs_decimal_tests DESTINATION ${ENGINE_BINDIR} COMPONENT columnstore-platform)
endif()
if (WITH_DATACONVERT_UT)

64
tests/dataconvert-tests.cpp
View File

@@ -729,5 +729,67 @@ TEST(DataConvertTest, DecimalToStringCheckScale38)
EXPECT_EQ(string(buf), expected);
}
TEST(DataConvertTest, ConvertColumnData) {
TEST(DataConvertTest, DecimalToStringCheckScale37)
{
CalpontSystemCatalog::ColType ct;
ct.colDataType = CalpontSystemCatalog::DECIMAL;
char buf[42];
string expected;
ct.precision = 38;
ct.scale = 37;
int128_t res;
// test simple values
res = 0;
expected = "0.0000000000000000000000000000000000000";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = 2;
expected = "0.0000000000000000000000000000000000002";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -2;
expected = "-0.0000000000000000000000000000000000002";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = 123;
expected = "0.0000000000000000000000000000000000123";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -123;
expected = "-0.0000000000000000000000000000000000123";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = ((((((int128_t)1234567890 * 10000000000) + 1234567890) * 10000000000) + 1234567890) * 100000000 ) + 12345678;
expected = "1.2345678901234567890123456789012345678";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -res;
expected = "-1.2345678901234567890123456789012345678";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
// test max/min decimal (i.e. 38 9's)
res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99;
expected = "9.9999999999999999999999999999999999999";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -res;
expected = "-9.9999999999999999999999999999999999999";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
// test trailing zeros
res = 123000;
expected = "0.0000000000000000000000000000000123000";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -res;
expected = "-0.0000000000000000000000000000000123000";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
}
TEST(DataConvertTest, ConvertColumnData)
{
}

1
utils/common/columnwidth.h
View File

@@ -23,7 +23,6 @@
namespace utils
{
const uint8_t MAXLEGACYWIDTH = 8ULL;
const uint8_t MAXCOLUMNWIDTH = 16ULL;
inline bool isWide(uint8_t width)
{

164
utils/dataconvert/dataconvert.cpp
View File

@@ -391,29 +391,21 @@ void number_int_value(const string& data,
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
if (ct.colWidth == 1)
if (LIKELY(ct.colWidth == 16))
{
if (intVal < MIN_TINYINT)
int128_t tmp;
utils::int128Min(tmp);
if (intVal < tmp + 2) // + 2 for NULL and EMPTY values
{
intVal = MIN_TINYINT;
pushwarning = true;
}
else if (intVal > MAX_TINYINT)
{
intVal = MAX_TINYINT;
intVal = tmp + 2;
pushwarning = true;
}
}
else if (ct.colWidth == 2)
else if (ct.colWidth == 8)
{
if (intVal < MIN_SMALLINT)
if (intVal < MIN_BIGINT)
{
intVal = MIN_SMALLINT;
pushwarning = true;
}
else if (intVal > MAX_SMALLINT)
{
intVal = MAX_SMALLINT;
intVal = MIN_BIGINT;
pushwarning = true;
}
}
@@ -430,21 +422,29 @@ void number_int_value(const string& data,
pushwarning = true;
}
}
else if (ct.colWidth == 8)
else if (ct.colWidth == 2)
{
if (intVal < MIN_BIGINT)
if (intVal < MIN_SMALLINT)
{
intVal = MIN_BIGINT;
intVal = MIN_SMALLINT;
pushwarning = true;
}
else if (intVal > MAX_SMALLINT)
{
intVal = MAX_SMALLINT;
pushwarning = true;
}
}
else if (ct.colWidth == 16)
else if (ct.colWidth == 1)
{
int128_t tmp;
utils::int128Min(tmp);
if (intVal < tmp + 2) // + 2 for NULL and EMPTY values
if (intVal < MIN_TINYINT)
{
intVal = tmp + 2;
intVal = MIN_TINYINT;
pushwarning = true;
}
else if (intVal > MAX_TINYINT)
{
intVal = MAX_TINYINT;
pushwarning = true;
}
}
@@ -1469,31 +1469,31 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType,
break;
case CalpontSystemCatalog::DECIMAL:
if (colType.colWidth == 1)
if (LIKELY(colType.colWidth == 16))
{
number_int_value(data, colType, pushWarning, noRoundup, val64);
value = (char) val64;
}
else if (colType.colWidth == 2)
{
number_int_value(data, colType, pushWarning, noRoundup, val64);
value = (short) val64;
}
else if (colType.colWidth == 4)
{
number_int_value(data, colType, pushWarning, noRoundup, val64);
value = (int) val64;
int128_t val128;
number_int_value(data, colType, pushWarning, noRoundup, val128);
value = (int128_t) val128;
}
else if (colType.colWidth == 8)
{
number_int_value(data, colType, pushWarning, noRoundup, val64);
value = (long long) val64;
}
else if (colType.colWidth == 16)
else if (colType.colWidth == 4)
{
int128_t val128;
number_int_value(data, colType, pushWarning, noRoundup, val128);
value = (int128_t) val128;
number_int_value(data, colType, pushWarning, noRoundup, val64);
value = (int) val64;
}
else if (colType.colWidth == 2)
{
number_int_value(data, colType, pushWarning, noRoundup, val64);
value = (short) val64;
}
else if (colType.colWidth == 1)
{
number_int_value(data, colType, pushWarning, noRoundup, val64);
value = (char) val64;
}
//else if (colType.colWidth == 32)
// value = data;
@@ -1503,29 +1503,29 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType,
case CalpontSystemCatalog::UDECIMAL:
// UDECIMAL numbers may not be negative
if (colType.colWidth == 1)
if (LIKELY(colType.colWidth == 16))
{
number_int_value(data, colType, pushWarning, noRoundup, val64);
char ival = (char) val64;
int128_t val128;
number_int_value(data, colType, pushWarning, noRoundup, val128);
if (ival < 0 &&
ival != static_cast<int8_t>(joblist::TINYINTEMPTYROW) &&
ival != static_cast<int8_t>(joblist::TINYINTNULL))
if (val128 < 0 &&
!utils::isWideDecimalNullValue(val128) &&
!utils::isWideDecimalEmptyValue(val128))
{
ival = 0;
val128 = 0;
pushWarning = true;
}
value = ival;
value = val128;
}
else if (colType.colWidth == 2)
else if (colType.colWidth == 8)
{
number_int_value(data, colType, pushWarning, noRoundup, val64);
short ival = (short) val64;
long long ival = static_cast<long long>(val64);
if (ival < 0 &&
ival != static_cast<int16_t>(joblist::SMALLINTEMPTYROW) &&
ival != static_cast<int16_t>(joblist::SMALLINTNULL))
ival != static_cast<long long>(joblist::BIGINTEMPTYROW) &&
ival != static_cast<long long>(joblist::BIGINTNULL))
{
ival = 0;
pushWarning = true;
@@ -1548,14 +1548,14 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType,
value = ival;
}
else if (colType.colWidth == 8)
else if (colType.colWidth == 2)
{
number_int_value(data, colType, pushWarning, noRoundup, val64);
long long ival = static_cast<long long>(val64);
short ival = (short) val64;
if (ival < 0 &&
ival != static_cast<long long>(joblist::BIGINTEMPTYROW) &&
ival != static_cast<long long>(joblist::BIGINTNULL))
ival != static_cast<int16_t>(joblist::SMALLINTEMPTYROW) &&
ival != static_cast<int16_t>(joblist::SMALLINTNULL))
{
ival = 0;
pushWarning = true;
@@ -1563,20 +1563,20 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType,
value = ival;
}
else if (colType.colWidth == 16)
else if (colType.colWidth == 1)
{
int128_t val128;
number_int_value(data, colType, pushWarning, noRoundup, val128);
number_int_value(data, colType, pushWarning, noRoundup, val64);
char ival = (char) val64;
if (val128 < 0 &&
!utils::isWideDecimalNullValue(val128) &&
!utils::isWideDecimalEmptyValue(val128))
if (ival < 0 &&
ival != static_cast<int8_t>(joblist::TINYINTEMPTYROW) &&
ival != static_cast<int8_t>(joblist::TINYINTNULL))
{
val128 = 0;
ival = 0;
pushWarning = true;
}
value = val128;
value = ival;
}
break;
@@ -1865,31 +1865,31 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType,
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
if (colType.colWidth == CalpontSystemCatalog::ONE_BYTE)
if (LIKELY(colType.colWidth == 16))
{
char tinyintvalue = joblist::TINYINTNULL;
value = tinyintvalue;
}
else if (colType.colWidth == CalpontSystemCatalog::TWO_BYTE)
{
short smallintvalue = joblist::SMALLINTNULL;
value = smallintvalue;
}
else if (colType.colWidth == CalpontSystemCatalog::FOUR_BYTE)
{
int intvalue = joblist::INTNULL;
value = intvalue;
int128_t val;
utils::setWideDecimalNullValue(val);
value = val;
}
else if (colType.colWidth == CalpontSystemCatalog::EIGHT_BYTE)
{
long long eightbyte = joblist::BIGINTNULL;
value = eightbyte;
}
else if (colType.colWidth == 16)
else if (colType.colWidth == CalpontSystemCatalog::FOUR_BYTE)
{
int128_t val;
utils::setWideDecimalNullValue(val);
value = val;
int intvalue = joblist::INTNULL;
value = intvalue;
}
else if (colType.colWidth == CalpontSystemCatalog::TWO_BYTE)
{
short smallintvalue = joblist::SMALLINTNULL;
value = smallintvalue;
}
else if (colType.colWidth == CalpontSystemCatalog::ONE_BYTE)
{
char tinyintvalue = joblist::TINYINTNULL;
value = tinyintvalue;
}
else
{

2
utils/joiner/tuplejoiner.cpp
View File

@@ -1147,7 +1147,7 @@ void TupleJoiner::updateCPData(const Row& r)
}
}
}
else if (r.getColumnWidth(colIdx) == utils::MAXCOLUMNWIDTH
else if (r.getColumnWidth(colIdx) == datatypes::MAXDECIMALWIDTH
&& (r.getColType(colIdx) == CalpontSystemCatalog::DECIMAL
|| r.getColType(colIdx) == CalpontSystemCatalog::UDECIMAL))
{

4
utils/rowgroup/rowgroup.cpp
View File

@@ -639,9 +639,9 @@ string Row::toString() const
break;
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
if (colWidths[i] == utils::MAXCOLUMNWIDTH)
if (colWidths[i] == datatypes::MAXDECIMALWIDTH)
{
unsigned int buflen = precision[i] + 3;
unsigned int buflen = utils::MAXLENGTH16BYTES;
char *buf = (char*)alloca(buflen);
// empty the buffer
dataconvert::DataConvert::decimalToString(getBinaryField<int128_t>(i),

2
utils/rowgroup/rowgroup.h
View File

@@ -1235,7 +1235,7 @@ inline void Row::copyField(Row& out, uint32_t destIndex, uint32_t srcIndex) cons
// WIP MCOL-641
inline void Row::copyBinaryField(Row& out, uint32_t destIndex, uint32_t srcIndex) const
{
out.setBinaryField<int128_t>(getBinaryField<int128_t>(srcIndex), 16, destIndex);
out.setBinaryField(getBinaryField<int128_t>(srcIndex), 16, destIndex);
}
inline void Row::setRid(uint64_t rid)

15
writeengine/bulk/we_bulkloadbuffer.cpp
View File

@@ -39,6 +39,7 @@
#include "brmtypes.h"
#include "dataconvert.h"
#include "exceptclasses.h"
#include "mcs_decimal.h"
#include "joblisttypes.h"
@@ -986,18 +987,18 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
if ( (column.dataType == CalpontSystemCatalog::DECIMAL) ||
(column.dataType == CalpontSystemCatalog::UDECIMAL))
{
if (width <= 8)
{
// errno is initialized and set in convertDecimalString
llVal = Convertor::convertDecimalString(
field, fieldLength, column.scale );
}
else
if (LIKELY(width == datatypes::MAXDECIMALWIDTH))
{
bool saturate = false;
bigllVal = dataconvert::string_to_ll<int128_t>(string(field), saturate);
// TODO MCOL-641 check saturate
}
else if (width <= 8)
{
// errno is initialized and set in convertDecimalString
llVal = Convertor::convertDecimalString(
field, fieldLength, column.scale );
}
}
else
{

39
writeengine/server/we_dmlcommandproc.cpp
View File

@@ -2622,7 +2622,6 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs,
uint64_t& blocksChanged)
{
uint8_t rc = 0;
//cout << " In processUpdate" << endl;
uint32_t tmp32, sessionID;
TxnID txnId;
bs >> PMId;
@@ -2639,7 +2638,6 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs,
uint8_t pkgType;
bs >> pkgType;
cpackages[txnId].read(bs);
//cout << "Processed meta data in update" << endl;
rc = fWEWrapper.startTransaction(txnId);
@@ -2811,9 +2809,20 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs,
rid = relativeRID;
convertToRelativeRid (rid, extentNum, blockNum);
rowIDLists.push_back(rid);
uint32_t colWidth = ((colTypes[j].colWidth > 8 &&
uint32_t colWidth = colTypes[j].colWidth;
if (colWidth > 8 &&
!(colTypes[j].colDataType == CalpontSystemCatalog::DECIMAL ||
colTypes[j].colDataType == CalpontSystemCatalog::UDECIMAL)) ? 8 : colTypes[j].colWidth);
colTypes[j].colDataType == CalpontSystemCatalog::UDECIMAL))
{
colWidth = 8;
}
else if (colWidth >= datatypes::MAXDECIMALWIDTH)
{
colWidth = datatypes::MAXDECIMALWIDTH;
}
int rrid = (int) relativeRID / (BYTE_PER_BLOCK / colWidth);
// populate stats.blocksChanged
if (rrid > preBlkNums[j])
@@ -2984,16 +2993,18 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs,
case CalpontSystemCatalog::UDECIMAL:
{
// WIP MCOL-641
if (fetchColColwidths[fetchColPos] == 16)
if (fetchColColwidths[fetchColPos] == datatypes::MAXDECIMALWIDTH)
{
int128_t* dec;
char buf[utils::MAXLENGTH16BYTES];
dec = row.getBinaryField<int128_t>(fetchColPos);
dataconvert::DataConvert::decimalToString(dec,
(unsigned)fetchColScales[fetchColPos], buf,
sizeof(buf), fetchColTypes[fetchColPos]);
value.assign(buf);
break;
}
@@ -3347,17 +3358,19 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs,
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
if (fetchColColwidths[fetchColPos] == 16)
if (fetchColColwidths[fetchColPos] == datatypes::MAXDECIMALWIDTH)
{
// WIP MCOL-641
int128_t* dec;
char buf[utils::MAXLENGTH16BYTES];
dec = row.getBinaryField<int128_t>(fetchColPos);
dataconvert::DataConvert::decimalToString(dec,
(unsigned)fetchColScales[fetchColPos], buf,
sizeof(buf), fetchColTypes[fetchColPos]);
value = buf;
break;
}
@@ -3994,7 +4007,6 @@ uint8_t WE_DMLCommandProc::processDelete(messageqcpp::ByteStream& bs,
uint64_t& blocksChanged)
{
uint8_t rc = 0;
//cout << " In processDelete" << endl;
uint32_t tmp32, sessionID;
TxnID txnId;
bs >> PMId;
@@ -4069,7 +4081,18 @@ uint8_t WE_DMLCommandProc::processDelete(messageqcpp::ByteStream& bs,
for (uint32_t j = 0; j < row.getColumnCount(); j++)
{
preBlkNums[j] = -1;
colWidth[j] = (row.getColumnWidth(j) >= 16 ? 16 : row.getColumnWidth(j));
colWidth[j] = row.getColumnWidth(j);
execplan::CalpontSystemCatalog::ColDataType colDataType = row.getColType(j);
if (colWidth[j] >= 8 &&
!(colDataType == execplan::CalpontSystemCatalog::DECIMAL ||
colDataType == execplan::CalpontSystemCatalog::UDECIMAL))
{
colWidth[j] = 8;
}
else if (colWidth[j] >= datatypes::MAXDECIMALWIDTH)
{
colWidth[j] = datatypes::MAXDECIMALWIDTH;
}
}
//Get the file information from rowgroup

16
writeengine/wrapper/writeengine.cpp
View File

@@ -70,9 +70,6 @@ namespace WriteEngine
{
StopWatch timer;
using dataconvert::int128_t;
using dataconvert::uint128_t;
/**@brief WriteEngineWrapper Constructor
*/
WriteEngineWrapper::WriteEngineWrapper() : m_opType(NOOP)
@@ -803,21 +800,9 @@ int WriteEngineWrapper::deleteRow(const TxnID& txnid, const vector<CSCTypesList>
cscColType = cscColTypeList[i];
Convertor::convertColType(&curColStruct);
/*if (curColStruct.colType == WriteEngine::WR_BINARY)
{
uint128_t bigEmptyVal;
emptyVal = m_colOp[op(curColStruct.fCompressionType)]->
getEmptyRowValue(curColStruct.colDataType, curColStruct.colWidth);
*(reinterpret_cast<uint64_t*>(&bigEmptyVal)) = emptyVal;
*(reinterpret_cast<uint64_t*>(&bigEmptyVal) + 1) = emptyVal;
curTuple.data = bigEmptyVal;
}
else
{*/
m_colOp[op(curColStruct.fCompressionType)]->
getEmptyRowValue(curColStruct.colDataType, curColStruct.colWidth, (uint8_t*)&emptyVal);
curTuple.data = emptyVal;
//}
curTupleList.push_back(curTuple);
colValueList.push_back(curTupleList);
@@ -4371,7 +4356,6 @@ int WriteEngineWrapper::updateColumnRecs(const TxnID& txnid,
ColumnOp* colOp = NULL;
bool successFlag = true;
unsigned width = 0;
\
int curFbo = 0, curBio, lastFbo = -1;
RID aRid = ridLists[0];
int rc = 0;