diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index e246e8e95..463764cd0 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -414,15 +414,15 @@ template<> inline bool isNullVal<16>(uint8_t type, const uint8_t* ival) // For BINARY { const uint64_t* val = reinterpret_cast(ival); - return ((val[0] == joblist::BINARYEMPTYROW) && (val[1] == joblist::BINARYNULL)); + return ((val[0] == joblist::BINARYNULL) && (val[1] == joblist::BINARYEMPTYROW)); } template<> inline bool isNullVal<32>(uint8_t type, const uint8_t* ival) // For BINARY { const uint64_t* val = reinterpret_cast(ival); - return ((val[0] == joblist::BINARYEMPTYROW) && (val[1] == joblist::BINARYEMPTYROW) - && (val[2] == joblist::BINARYEMPTYROW) && (val[3] == joblist::BINARYNULL)); + return ((val[0] == joblist::BINARYNULL) && (val[1] == joblist::BINARYEMPTYROW) + && (val[2] == joblist::BINARYEMPTYROW) && (val[3] == joblist::BINARYEMPTYROW)); } template<> diff --git a/primitives/primproc/columncommand.cpp b/primitives/primproc/columncommand.cpp index 0270c8f9a..d2145807c 100644 --- a/primitives/primproc/columncommand.cpp +++ b/primitives/primproc/columncommand.cpp @@ -1067,8 +1067,6 @@ void ColumnCommand::getEmptyRowValue(const CSCDataType dataType, ptr[1] = joblist::BINARYEMPTYROW; } - - void ColumnCommand::getLBIDList(uint32_t loopCount, vector* lbids) { int64_t firstLBID = lbid, lastLBID = firstLBID + (loopCount * colType.colWidth) - 1, i; diff --git a/utils/common/columnwidth.h b/utils/common/columnwidth.h index 8e36f5356..b16086494 100644 --- a/utils/common/columnwidth.h +++ b/utils/common/columnwidth.h @@ -69,6 +69,18 @@ namespace utils return 16; } } + inline uint8_t precisionByWidth(unsigned w) + { + switch(w) + { + case 16: + return 38; + // In case we will support decimals that spans 32 bytes. + default: + return 65; + } + } + } diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 7db6e023d..619ac6c77 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -28,6 +28,7 @@ #include #include #include +#include using namespace std; #include #include @@ -1164,7 +1165,110 @@ bool stringToTimestampStruct(const string& data, TimeStamp& timeStamp, const str } // WIP MCOL-641 -// Check for overflows with buflen +// Second DT is for POD representation. +template +size_t DataConvert::writeIntPart(T* dec, char* p, + const uint16_t buflen, + const uint8_t scale) +{ + T intPart = *dec; + if (scale) + { + // optimize this + for (size_t i = 0; i < scale; i++) + intPart /= 10; + } + + // optimize for less then uint64_t values + uint64_t div = 10000000000000000000ULL; + T high = intPart; + T low; + low = high % div; + high /= div; + T mid; + mid = high % div; + high /= div; + + // pod[0] is high 8 byte, pod[1] is low + uint64_t* high_pod = reinterpret_cast(&high); + uint64_t* mid_pod = reinterpret_cast(&mid); + uint64_t* low_pod = reinterpret_cast(&low); + char* original_p = p; + int written = 0; + + // WIP replace snprintf with streams + if (high_pod[0] != 0) + { + written = sprintf(p, "%lu", high_pod[0]); + p += written; + written = sprintf(p, "%019lu", mid_pod[0]); + p += written; + sprintf(p, "%019lu", low_pod[0]); + } + else if (mid_pod[0] != 0) + { + written = sprintf(p, "%lu", mid_pod[0]); + p += written; + written = sprintf(p, "%019lu", low_pod[0]); + p += written; + } + else + { + written = sprintf(p, "%lu", low_pod[0]); + p += written; + } + + if (buflen <= p-original_p) + { + throw QueryDataExcept("toString() char buffer overflow.", formatErr); + } + return p-original_p; +} + +template +size_t DataConvert::writeFractionalPart(T* dec, char* p, + const uint16_t buflen, + const uint8_t scale) +{ + T scaleDivisor = 10; + for (size_t i = 1; i < scale; i++) + scaleDivisor *= 10; + + T fractionalPart = *dec % scaleDivisor; + return writeIntPart(&fractionalPart, p, buflen, 0); +} + +// WIP MCOL-641 +// Limit this to Decimal only +// Replace decimalToString with this one +template +void DataConvert::toString1(T* dec, char *p, const uint16_t buflen, + const uint8_t scale) +{ + if (*dec < static_cast(0)) + { + *p++ = '-'; + *dec *= -1; + } + + char* original_p = p; + size_t written = 0; + written = writeIntPart(dec, p, buflen, scale); + p += written; + + // WIP To be finished for 0.042 + if (scale) + { + *p++ = '.'; + p += writeFractionalPart(dec, p, p-original_p, scale); + } + + if (buflen <= p-original_p) + { + throw QueryDataExcept("toString() char buffer overflow.", formatErr); + } +} + template void DataConvert::toString(T* dec, char *p, size_t buflen) { @@ -1206,6 +1310,7 @@ void DataConvert::toString(T* dec, char *p, size_t buflen) if (buflen <= p-original_p) std::cout << "DataConvert::toString char buffer overflow" << std::endl; } + // WIP MCOL-641 // Template this // result must be calloc-ed @@ -1239,13 +1344,20 @@ void atoi128(const std::string& arg, uint128_t& res) } // WIP MCOL-641 +// Doesn't work for -0.042 template void DataConvert::decimalToString(T* valuePtr, uint8_t scale, char* buf, unsigned int buflen, - cscDataType colDataType) + cscDataType colDataType) // We don't need the last one { + if (*valuePtr < static_cast(0)) + { + *buf++ = '-'; + *valuePtr *= -1; + } + toString(valuePtr, buf, buflen); // Biggest ColumnStore supports is DECIMAL(38,x), or 38 total digits+dp+sign for column diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 4bd30ca33..2301d75ec 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -1034,8 +1034,19 @@ public: template EXPORT static void decimalToString(T* value, uint8_t scale, char* buf, unsigned int buflen, cscDataType colDataType); + template + static void toString(T* dec, char *p, size_t buflen); template - EXPORT static void toString(T* dec, char *p, size_t buflen); + EXPORT static void toString1(T* dec, char* p, const uint16_t buflen, + const uint8_t scale = 0); + template + static size_t writeIntPart(T* dec, char* p, const uint16_t buflen, + const uint8_t scale); + template + static size_t writeFractionalPart(T* dec, char* p, + const uint16_t buflen, const uint8_t scale); + + static inline void int128Max(int128_t& i) { diff --git a/utils/rowgroup/rowgroup-tests.cpp b/utils/rowgroup/rowgroup-tests.cpp index f68e5365e..e63f0b2db 100644 --- a/utils/rowgroup/rowgroup-tests.cpp +++ b/utils/rowgroup/rowgroup-tests.cpp @@ -1,8 +1,10 @@ #include // googletest header file +#include + #include "rowgroup.h" #include "columnwidth.h" #include "joblisttypes.h" -#include "iostream" +#include "dataconvert.h" #define WIDE_DEC_PRECISION 38U #define INITIAL_ROW_OFFSET 2 @@ -60,7 +62,6 @@ class RowDecimalTest : public ::testing::Test { false //useStringTable ); - //std::cout << inRG.toString() << std::endl; rg = inRG; rgD.reinit(rg); rg.setData(&rgD); @@ -74,9 +75,14 @@ class RowDecimalTest : public ::testing::Test { uint64_t* uint128_pod = reinterpret_cast(&nullValue); uint128_pod[0] = joblist::BINARYEMPTYROW; uint128_pod[1] = joblist::BINARYNULL; - bigValue = 42*0xFFFFFFFFFFFFFFFFLL; - - sValueVector.push_back(nullValue); + bigValue = -static_cast(0xFFFFFFFF)*0xFFFFFFFFFFFFFFFF; + //char buf[utils::precisionByWidth(16)+3]; + //memset(&buf[0], 0, sizeof(buf)); + //int scale1 = 3; + //dataconvert::DataConvert::decimalToString(&bigValue, scale1, buf, + // utils::precisionByWidth(sizeof(bigValue))+3,types[0]); + //std::cout << buf << std::endl; + sValueVector.push_back(nullValue-2); sValueVector.push_back(-42); sValueVector.push_back(bigValue); sValueVector.push_back(0); @@ -106,15 +112,15 @@ class RowDecimalTest : public ::testing::Test { s32ValueVector.push_back(0x81); s32ValueVector.push_back(joblist::INTNULL-1); - s64ValueVector.push_back(joblist::INTNULL); + s64ValueVector.push_back(joblist::BIGINTNULL); s64ValueVector.push_back(-0x79); s64ValueVector.push_back(0); s64ValueVector.push_back(0x81); - s64ValueVector.push_back(joblist::INTNULL-1); + s64ValueVector.push_back(joblist::BIGINTNULL-1); - r.initToNull(); - r.nextRow(rowSize); - for(size_t i = 1; i < sValueVector.size(); i++) { + //r.initToNull(); + //r.nextRow(rowSize); + for(size_t i = 0; i < sValueVector.size(); i++) { r.setBinaryField_offset(&sValueVector[i], sizeof(sValueVector[0]), offsets[0]); r.setBinaryField_offset(&uValueVector[i], @@ -123,7 +129,6 @@ class RowDecimalTest : public ::testing::Test { r.setIntField(s32ValueVector[i], 3); r.setIntField(s16ValueVector[i], 4); r.setIntField(s8ValueVector[i], 5); - //std::cout << r.toString() << std::endl; r.nextRow(rowSize); } rowCount = sValueVector.size(); @@ -159,6 +164,7 @@ TEST_F(RowDecimalTest, NonNULLValuesCheck) { TEST_F(RowDecimalTest, initToNullANDisNullValueValueCheck) { rg.getRow(0, &r); + r.initToNull(); EXPECT_TRUE(r.isNullValue(0)); EXPECT_TRUE(r.isNullValue(1)); EXPECT_TRUE(r.isNullValue(2)); @@ -176,29 +182,37 @@ TEST_F(RowDecimalTest, getBinaryFieldCheck) { for (size_t i = 0; i < sValueVector.size(); i++) { s128Value = r.getBinaryField(0); - EXPECT_EQ(*s128Value, sValueVector[i]); + EXPECT_EQ(sValueVector[i], *s128Value); u128Value = r.getBinaryField(1); - EXPECT_EQ(*u128Value, uValueVector[i]); - //EXPECT_EQ(r.getIntField(2),s64ValueVector[i]); - //EXPECT_EQ(r.getIntField(3),s32ValueVector[i]); + EXPECT_EQ(uValueVector[i], *u128Value); + //EXPECT_EQ(s64ValueVector[i], r.getIntField(2)); + //EXPECT_EQ(s32ValueVector[i],r.getIntField(3)); //EXPECT_EQ(r.getIntField(4),s16ValueVector[i]); //EXPECT_EQ(r.getIntField(5),s8ValueVector[i]); r.nextRow(rowSize); } } + +// TBD Need to add asserts when toString will be finished TEST_F(RowDecimalTest, toStringCheck) { - std::string exemplar1("0: NULL NULL NULL NULL NULL NULL "); + std::vector exemplarVector; + exemplarVector.push_back(std::string("0: NULL NULL NULL NULL NULL NULL ")); + exemplarVector.push_back(std::string("0: -42 42 -121 -121 -121 -121 ")); + exemplarVector.push_back(std::string("0: -79228162495817593515539431425 -79228162495817593515539431425 0 0 0 0 ")); + exemplarVector.push_back(std::string("0: 0 0 129 129 129 -127 ")); + exemplarVector.push_back(std::string("0: -18446744073709551618 -18446744073709551618 9223372036854775807 2147483647 32767 127 ")); + rg.getRow(0, &r); - EXPECT_EQ(r.toString(), exemplar1); - r.nextRow(rowSize); - std::cout << r.toString() << std::endl; - r.nextRow(rowSize); - std::cout << r.toString() << std::endl; - r.nextRow(rowSize); - std::cout << r.toString() << std::endl; + r.initToNull(); + for (auto &el: exemplarVector) { + EXPECT_EQ(el, r.toString()); + r.nextRow(rowSize); + } } -//toString -//toCSV -//applyMapping -//equals +TEST_F(RowDecimalTest, toCSVCheck) { +} +TEST_F(RowDecimalTest, applyMappingCheck) { +} +TEST_F(RowDecimalTest, equalsCheck) { +} diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 46971ae81..bd84117a2 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -642,8 +642,8 @@ string Row::toString() const { char *buf = (char*)alloca(precision[i] + 3); // empty the buffer - dataconvert::DataConvert::toString(getBinaryField(i), - buf, precision[i]+3); + dataconvert::DataConvert::toString(getBinaryField(i), + buf, precision[i]+3); //WIP scale[i] os << buf << " "; break; } @@ -851,8 +851,8 @@ void Row::initToNull() case 16 : { uint64_t *dec = reinterpret_cast(&data[offsets[i]]); - dec[0] = joblist::BINARYEMPTYROW; - dec[1] = joblist::BINARYNULL; + dec[0] = joblist::BINARYNULL; + dec[1] = joblist::BINARYEMPTYROW; break; } default: @@ -882,8 +882,8 @@ void Row::initToNull() case CalpontSystemCatalog::BINARY: { uint64_t *dec = reinterpret_cast(&data[offsets[i]]); - dec[0] = joblist::BINARYEMPTYROW; - dec[1] = joblist::BINARYNULL; + dec[0] = joblist::BINARYNULL; + dec[1] = joblist::BINARYEMPTYROW; } break; @@ -916,10 +916,10 @@ Row::isNullValue_offset( uint32_t offset) const { const int64_t *intPtr = reinterpret_cast(&data[offset]); - return ((intPtr[0] == static_cast(joblist::BINARYEMPTYROW)) && + return ((intPtr[0] == static_cast(joblist::BINARYNULL)) && (intPtr[1] == static_cast(joblist::BINARYEMPTYROW)) && (intPtr[2] == static_cast(joblist::BINARYEMPTYROW)) && - (intPtr[3] == static_cast(joblist::BINARYNULL))); + (intPtr[3] == static_cast(joblist::BINARYEMPTYROW))); } template<> @@ -928,8 +928,8 @@ Row::isNullValue_offset( uint32_t offset) const { const int64_t *intPtr = reinterpret_cast(&data[offset]); - return ((intPtr[0] == static_cast(joblist::BINARYEMPTYROW)) - && (intPtr[1] == static_cast(joblist::BINARYNULL))); + return ((intPtr[0] == static_cast(joblist::BINARYNULL)) + && (intPtr[1] == static_cast(joblist::BINARYEMPTYROW))); } template<> @@ -938,8 +938,8 @@ Row::isNullValue_offset( uint32_t offset) const { const int64_t *intPtr = reinterpret_cast(&data[offset]); - return ((intPtr[0] == static_cast(joblist::BINARYEMPTYROW)) - && (intPtr[1] == static_cast(joblist::BINARYNULL))); + return ((intPtr[0] == static_cast(joblist::BINARYNULL)) + && (intPtr[1] == static_cast(joblist::BINARYEMPTYROW))); } template<>