diff --git a/utils/dataconvert/dataconvert-tests.cpp b/utils/dataconvert/dataconvert-tests.cpp index 3a18571e8..b184030cb 100644 --- a/utils/dataconvert/dataconvert-tests.cpp +++ b/utils/dataconvert/dataconvert-tests.cpp @@ -24,6 +24,10 @@ using namespace std; using namespace execplan; #include "dataconvert.h" using namespace dataconvert; +#include "joblisttypes.h" +#include "columnwidth.h" + +using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; TEST(DataConvertTest, Strtoll128) { @@ -540,3 +544,102 @@ TEST(DataConvertTest, NumberIntValue) EXPECT_EQ(b2, b4); EXPECT_TRUE(pushWarning); } + +TEST(DataConvertTest, atoiCheck) { + std::vector expected; + std::vector decimalAsStringVec; + decimalAsStringVec.push_back("99999999999999999999999999999999999999"); + decimalAsStringVec.push_back("-99999999999999999999999999999999999999"); + decimalAsStringVec.push_back("0"); + decimalAsStringVec.push_back("-0.042"); + expected.push_back(10000000000000000000ULL*(uint128_t)9999999999999999999ULL + +9999999999999999999ULL); + expected.push_back(-1*expected[0]); + expected.push_back(0); + expected.push_back(-42); + for (int i=0; i < expected.size(); i++) { + int128_t value; + dataconvert::atoi128(decimalAsStringVec[i], value); + EXPECT_EQ(expected[i], value); + EXPECT_NE(expected[i], value-1); + } +} + +TEST(DataConvertTest, DecimalToStringCheckScale3) { + std::vector expected; + std::vector decimalAsStringVec; + // scale 3 + uint8_t scale3 = 3; + decimalAsStringVec.push_back("99999999999999999999999999999999999999"); + decimalAsStringVec.push_back("-99999999999999999999999999999999999999"); + decimalAsStringVec.push_back("0"); + decimalAsStringVec.push_back("-0.042"); + expected.push_back("99999999999999999999999999999999999.999"); + expected.push_back("-99999999999999999999999999999999999.999"); + expected.push_back("0.000"); + expected.push_back("-0.042"); + char buf[42]; + CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL; + for (int i=0; i < expected.size(); i++) { + int128_t value = -4; + memset(buf, 0, 42); + dataconvert::atoi128(decimalAsStringVec[i], value); + dataconvert::DataConvert::decimalToString(&value, scale3, buf, + utils::precisionByWidth(sizeof(value))+4, type); + EXPECT_EQ(expected[i], std::string(buf)); + } +} + +TEST(DataConvertTest, DecimalToStringCheckScale37) { + std::vector expected; + std::vector decimalAsStringVec; + uint8_t scale37 = 37; + decimalAsStringVec.push_back("99999999999999999999999999999999999999"); + decimalAsStringVec.push_back("-99999999999999999999999999999999999999"); + decimalAsStringVec.push_back("0"); + decimalAsStringVec.push_back("-42"); + decimalAsStringVec.push_back("-19999999999999999999999999999999999999"); + expected.push_back("9.9999999999999999999999999999999999999"); + expected.push_back("-9.9999999999999999999999999999999999999"); + expected.push_back("0.0000000000000000000000000000000000000"); + expected.push_back("-0.0000000000000000000000000000000000042"); + expected.push_back("-1.9999999999999999999999999999999999999"); + char buf[42]; + CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL; + for (int i=0; i < expected.size(); i++) { + int128_t value = -4; + memset(buf, 0, 41); + dataconvert::atoi128(decimalAsStringVec[i], value); + dataconvert::DataConvert::decimalToString(&value, scale37, buf, + utils::precisionByWidth(sizeof(value))+4, type); + EXPECT_EQ(expected[i], std::string(buf)); + } +} + +TEST(DataConvertTest, DecimalToStringCheckScale38) { + std::vector expected; + std::vector decimalAsStringVec; + uint8_t scale38 = 38; + decimalAsStringVec.push_back("99999999999999999999999999999999999999"); + decimalAsStringVec.push_back("-99999999999999999999999999999999999999"); + decimalAsStringVec.push_back("0"); + decimalAsStringVec.push_back("-42"); + expected.push_back("0.99999999999999999999999999999999999999"); + expected.push_back("-0.99999999999999999999999999999999999999"); + expected.push_back("0.00000000000000000000000000000000000000"); + expected.push_back("-0.00000000000000000000000000000000000042"); + char buf[42]; + CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL; + for (int i=0; i < expected.size(); i++) { + int128_t value = -4; + memset(buf, 0, 41); + dataconvert::atoi128(decimalAsStringVec[i], value); + dataconvert::DataConvert::decimalToString(&value, scale38, buf, + utils::precisionByWidth(sizeof(value))+4, type); + EXPECT_EQ(expected[i], std::string(buf)); + } +} + +TEST(DataConvertTest, ConvertColumnData) { +} + diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index 1273e134f..beaf5a7dd 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1227,16 +1227,16 @@ bool stringToTimestampStruct(const string& data, TimeStamp& timeStamp, const str } // WIP MCOL-641 -// Second DT is for POD representation. template size_t DataConvert::writeIntPart(T* dec, char* p, const uint16_t buflen, - const uint8_t scale) + const uint8_t scale) //don't need this { T intPart = *dec; if (scale) { - // optimize this + //TODO Use dictionary to produce divisor + // instead of a loop for (size_t i = 0; i < scale; i++) intPart /= 10; } @@ -1292,90 +1292,84 @@ size_t DataConvert::writeFractionalPart(T* dec, char* p, const uint16_t buflen, const uint8_t scale) { - T scaleDivisor = 10; - for (size_t i = 1; i < scale; i++) - scaleDivisor *= 10; + //TODO Use dictionary instead of multiplication. + T scaleDivisor = 10; + for (size_t i = 1; i < scale; i++) + scaleDivisor *= 10; - T fractionalPart = *dec % scaleDivisor; - return writeIntPart(&fractionalPart, p, buflen, 0); + T fractionalPart = *dec % scaleDivisor; + // divide by the base untill we have non-zero quotinent + size_t written = 0; + scaleDivisor /= 10; + while (scaleDivisor > 1 && *dec / scaleDivisor == 0) + { + *p++ = '0'; + written++; + scaleDivisor /= 10; + } + written += writeIntPart(&fractionalPart, p, buflen, 0); + if (written < scale) + { + p += written; + for (size_t left = written; left < scale; left++) + { + *p++ = '0'; + } + } + return scale; } -// WIP MCOL-641 -// Limit this to Decimal only -// Replace decimalToString with this one template -void DataConvert::toString1(T* dec, char *p, const uint16_t buflen, - const uint8_t scale) +void DataConvert::toString(T* dec, uint8_t scale, + char *p, unsigned int buflen) { - if (*dec < static_cast(0)) - { - *p++ = '-'; - *dec *= -1; - } + char* original_p = p; + size_t written = 0; + // Early return for NULL value + int128_t sign = 0; + // WIP use constants here + // WIP Treat both magics here + uint64_t* signPod = reinterpret_cast(&sign); + signPod[1] = utils::BINARYNULLVALUEHIGH; - char* original_p = p; - size_t written = 0; - written = writeIntPart(dec, p, buflen, scale); - p += written; + if (*dec == sign) + { + *p++ = '0'; + if (scale) + { + *p++ = '.'; + while (scale-- > 0) + *p++ = '0'; + } + return; + } - // WIP To be finished for 0.042 - if (scale) - { - *p++ = '.'; - p += writeFractionalPart(dec, p, p-original_p, scale); - } + if (*dec < static_cast(0)) + { + *p++ = '-'; + *dec *= -1; + } - if (buflen <= p-original_p) - { - throw QueryDataExcept("toString() char buffer overflow.", formatErr); - } + written = writeIntPart(dec, p, buflen, scale); + p += written; + + if (scale) + { + *p++ = '.'; + p += writeFractionalPart(dec, p, buflen-(p-original_p), scale); + } + + if (buflen <= p-original_p) + { + throw QueryDataExcept("toString() char buffer overflow.", formatErr); + } } -template -void DataConvert::toString(T* dec, char *p, size_t buflen) -{ - uint64_t div = 10000000000000000000ULL; - // template this - uint128_t high = *dec; - uint128_t low; - low = high % div; - high /= div; - uint128_t mid; - mid = high % div; - high /= div; - - // WIP How to treat PODs here ? - // use typeof - // Or a templated structure - // Use uint64* to access parts of uint128 and remove pods - Int128Pod_t *high_pod = reinterpret_cast(&high); - Int128Pod_t *mid_pod = reinterpret_cast(&mid); - Int128Pod_t *low_pod = reinterpret_cast(&low); - char* original_p = p; - int printed_chars = 0; - - // WIP replace snprintf with streams - if (high_pod->lo != 0) { - printed_chars = sprintf(p, "%lu", high_pod->lo); - p += printed_chars; - printed_chars = sprintf(p, "%019lu", mid_pod->lo); - p += printed_chars; - sprintf(p, "%019lu", low_pod->lo); - } else if (mid_pod->lo != 0) { - printed_chars = sprintf(p, "%lu", mid_pod->lo); - p += printed_chars; - sprintf(p, "%019lu", low_pod->lo); - } - else { - sprintf(p, "%lu", low_pod->lo); - } - if (buflen <= p-original_p) - std::cout << "DataConvert::toString char buffer overflow" << std::endl; -} +template +void DataConvert::toString(int128_t* dec, uint8_t scale, + char *p, unsigned int buflen); // WIP MCOL-641 -// Template this -// result must be calloc-ed void atoi128(const std::string& arg, int128_t& res) { res = 0; @@ -1386,14 +1380,12 @@ void atoi128(const std::string& arg, int128_t& res) if (LIKELY(arg[j]-'0' >= 0)) res = res*10 + arg[j] - '0'; } - // Use bit shift if possible if (idx) res *= -1; - //toString(res, buf); - //std::cerr << "atoi_ " << buf <(0)) - { - *ptr++ = '-'; - value *= -1; - idbassert(l1 >= 2); - l1--; - } - //we want to move the last scale chars right by one spot to insert the dp - //we want to move the trailing null as well, so it's really scale+1 chars - - toString(&value, ptr, buflen); - - // Biggest ColumnStore supports is DECIMAL(38,x), or 38 total digits+dp+sign for column - - if (scale == 0) - return; - - //need to make sure we have enough leading zeros for this to work... - //at this point scale is always > 0 - size_t l2 = 1; - - if ((unsigned)scale > l1) - { - const char* zeros = "00000000000000000000000000000000000000"; //38 0's - size_t diff = 0; - - if (*valuePtr != 0) - diff = scale - l1; //this will always be > 0 - else - diff = scale; - - memmove((ptr + diff), ptr, l1 + 1); //also move null - memcpy(ptr, zeros, diff); - - if (*valuePtr != 0) - l1 = 0; - else - l1 = 1; - } - else if ((unsigned)scale == l1) - { - l1 = 0; - l2 = 2; - } - else - { - l1 -= scale; - } - - memmove((ptr + l1 + l2), (ptr + l1), scale + 1); //also move null - - if (l2 == 2) - *(ptr + l1++) = '0'; - - *(ptr + l1) = '.'; + toString(valuePtr, scale, buf, buflen); } // Explicit instantiation template diff --git a/utils/dataconvert/dataconvert.h b/utils/dataconvert/dataconvert.h index 67cb3265c..78e3b8639 100644 --- a/utils/dataconvert/dataconvert.h +++ b/utils/dataconvert/dataconvert.h @@ -1044,11 +1044,8 @@ public: template EXPORT static void decimalToString(T* value, uint8_t scale, char* buf, unsigned int buflen, cscDataType colDataType); - template - static void toString(T* dec, char *p, size_t buflen); template - EXPORT static void toString1(T* dec, char* p, const uint16_t buflen, - const uint8_t scale = 0); + static void toString(T* dec, uint8_t scale, char* p, unsigned int buflen); template static size_t writeIntPart(T* dec, char* p, const uint16_t buflen, const uint8_t scale); diff --git a/utils/rowgroup/rowgroup-tests.cpp b/utils/rowgroup/rowgroup-tests.cpp index 7b9708526..a18f8d38b 100644 --- a/utils/rowgroup/rowgroup-tests.cpp +++ b/utils/rowgroup/rowgroup-tests.cpp @@ -1,3 +1,20 @@ +/* Copyright (C) 2020 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + #include // googletest header file #include @@ -195,7 +212,6 @@ TEST_F(RowDecimalTest, getBinaryFieldCheck) { } } -// TBD Need to add asserts when toString will be finished TEST_F(RowDecimalTest, toStringCheck) { std::vector exemplarVector; exemplarVector.push_back(std::string("0: NULL NULL NULL NULL NULL NULL "));