MCOL-641 DataConvert::decimalToString() refactoring.

utils/dataconvert/dataconvert-tests.cpp

@@ -24,6 +24,10 @@ using namespace std;
 using namespace execplan;
 #include "dataconvert.h"
 using namespace dataconvert;
+#include "joblisttypes.h"
+#include "columnwidth.h"
+
+using CSCDataType = execplan::CalpontSystemCatalog::ColDataType;
 
 TEST(DataConvertTest, Strtoll128)
 {
@@ -540,3 +544,102 @@ TEST(DataConvertTest, NumberIntValue)
     EXPECT_EQ(b2, b4);
     EXPECT_TRUE(pushWarning);
 }
+
+TEST(DataConvertTest, atoiCheck) {
+    std::vector<int128_t> expected;
+    std::vector<std::string> decimalAsStringVec;
+    decimalAsStringVec.push_back("99999999999999999999999999999999999999");
+    decimalAsStringVec.push_back("-99999999999999999999999999999999999999");
+    decimalAsStringVec.push_back("0");
+    decimalAsStringVec.push_back("-0.042");
+    expected.push_back(10000000000000000000ULL*(uint128_t)9999999999999999999ULL
+        +9999999999999999999ULL);
+    expected.push_back(-1*expected[0]);
+    expected.push_back(0);
+    expected.push_back(-42);
+    for (int i=0; i < expected.size(); i++) {
+        int128_t value;
+        dataconvert::atoi128(decimalAsStringVec[i], value);
+        EXPECT_EQ(expected[i], value);
+        EXPECT_NE(expected[i], value-1);
+    }
+}
+
+TEST(DataConvertTest, DecimalToStringCheckScale3) {
+    std::vector<std::string> expected;
+    std::vector<std::string> decimalAsStringVec;
+    // scale 3
+    uint8_t scale3 = 3;
+    decimalAsStringVec.push_back("99999999999999999999999999999999999999");
+    decimalAsStringVec.push_back("-99999999999999999999999999999999999999");
+    decimalAsStringVec.push_back("0");
+    decimalAsStringVec.push_back("-0.042");
+    expected.push_back("99999999999999999999999999999999999.999");
+    expected.push_back("-99999999999999999999999999999999999.999");
+    expected.push_back("0.000");
+    expected.push_back("-0.042");
+    char buf[42];
+    CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL;
+    for (int i=0; i < expected.size(); i++) {
+        int128_t value = -4;
+        memset(buf, 0, 42);
+        dataconvert::atoi128(decimalAsStringVec[i], value);
+        dataconvert::DataConvert::decimalToString(&value, scale3, buf,
+        utils::precisionByWidth(sizeof(value))+4, type);
+        EXPECT_EQ(expected[i], std::string(buf));
+    }
+}
+
+TEST(DataConvertTest, DecimalToStringCheckScale37) {
+    std::vector<std::string> expected;
+    std::vector<std::string> decimalAsStringVec;
+    uint8_t scale37 = 37;
+    decimalAsStringVec.push_back("99999999999999999999999999999999999999");
+    decimalAsStringVec.push_back("-99999999999999999999999999999999999999");
+    decimalAsStringVec.push_back("0");
+    decimalAsStringVec.push_back("-42");
+    decimalAsStringVec.push_back("-19999999999999999999999999999999999999");
+    expected.push_back("9.9999999999999999999999999999999999999");
+    expected.push_back("-9.9999999999999999999999999999999999999");
+    expected.push_back("0.0000000000000000000000000000000000000");
+    expected.push_back("-0.0000000000000000000000000000000000042");
+    expected.push_back("-1.9999999999999999999999999999999999999");
+    char buf[42];
+    CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL;
+    for (int i=0; i < expected.size(); i++) {
+        int128_t value = -4;
+        memset(buf, 0, 41);
+        dataconvert::atoi128(decimalAsStringVec[i], value);
+        dataconvert::DataConvert::decimalToString(&value, scale37, buf,
+        utils::precisionByWidth(sizeof(value))+4, type);
+        EXPECT_EQ(expected[i], std::string(buf));
+    }
+}
+
+TEST(DataConvertTest, DecimalToStringCheckScale38) {
+    std::vector<std::string> expected;
+    std::vector<std::string> decimalAsStringVec;
+    uint8_t scale38 = 38;
+    decimalAsStringVec.push_back("99999999999999999999999999999999999999");
+    decimalAsStringVec.push_back("-99999999999999999999999999999999999999");
+    decimalAsStringVec.push_back("0");
+    decimalAsStringVec.push_back("-42");
+    expected.push_back("0.99999999999999999999999999999999999999");
+    expected.push_back("-0.99999999999999999999999999999999999999");
+    expected.push_back("0.00000000000000000000000000000000000000");
+    expected.push_back("-0.00000000000000000000000000000000000042");
+    char buf[42];
+    CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL;
+    for (int i=0; i < expected.size(); i++) {
+        int128_t value = -4;
+        memset(buf, 0, 41);
+        dataconvert::atoi128(decimalAsStringVec[i], value);
+        dataconvert::DataConvert::decimalToString(&value, scale38, buf,
+        utils::precisionByWidth(sizeof(value))+4, type);
+        EXPECT_EQ(expected[i], std::string(buf));
+    }
+}
+
+TEST(DataConvertTest, ConvertColumnData) {
+}
+

utils/dataconvert/dataconvert.cpp

@@ -1227,16 +1227,16 @@ bool stringToTimestampStruct(const string& data, TimeStamp& timeStamp, const str
 }
 
 // WIP MCOL-641
-// Second DT is for POD representation.
 template <typename T>
 size_t DataConvert::writeIntPart(T* dec, char* p,
     const uint16_t buflen,
-    const uint8_t scale)
+    const uint8_t scale) //don't need this
 {
   T intPart = *dec;
   if (scale)
   {
-    // optimize this
+    //TODO Use dictionary to produce divisor
+    // instead of a loop
     for (size_t i = 0; i < scale; i++)
         intPart /= 10;
   }
@@ -1292,90 +1292,84 @@ size_t DataConvert::writeFractionalPart(T* dec, char* p,
     const uint16_t buflen,
     const uint8_t scale)
 {
-  T scaleDivisor = 10;
+    //TODO Use dictionary instead of multiplication.
-  for (size_t i = 1; i < scale; i++)
+    T scaleDivisor = 10;
-    scaleDivisor *= 10;
+    for (size_t i = 1; i < scale; i++)
+        scaleDivisor *= 10;
   
-  T fractionalPart = *dec % scaleDivisor;
+    T fractionalPart = *dec % scaleDivisor;
-  return writeIntPart(&fractionalPart, p, buflen, 0);
+    // divide by the base untill we have non-zero quotinent
+    size_t written = 0;
+    scaleDivisor /= 10;
+    while (scaleDivisor > 1 && *dec / scaleDivisor == 0)
+    {
+        *p++ = '0';
+        written++;
+        scaleDivisor /= 10;
+    }
+    written += writeIntPart(&fractionalPart, p, buflen, 0);
+    if (written < scale)
+    {
+        p += written;
+        for (size_t left = written; left < scale; left++)
+        {
+            *p++ = '0';
+        }
+    }
+    return scale;
 }
 
-// WIP MCOL-641
-// Limit this to Decimal only
-// Replace decimalToString with this one
 template<typename T>
-void DataConvert::toString1(T* dec, char *p, const uint16_t buflen,
+void DataConvert::toString(T* dec, uint8_t scale,
-    const uint8_t scale)
+    char *p, unsigned int buflen)
 {
-  if (*dec < static_cast<T>(0))
+    char* original_p = p;
-  {
+    size_t written = 0;
-    *p++ = '-';
+    // Early return for NULL value
-    *dec *= -1;
+    int128_t sign = 0;
-  }
+    // WIP use constants here
+    // WIP Treat both magics here
+    uint64_t* signPod = reinterpret_cast<uint64_t*>(&sign);
+    signPod[1] = utils::BINARYNULLVALUEHIGH;
 
-  char* original_p = p;
+    if (*dec == sign)
-  size_t written = 0;
+    {
-  written = writeIntPart<T>(dec, p, buflen, scale);
+        *p++ = '0';
-  p += written;
+        if (scale)
+        {
+            *p++ = '.';
+            while (scale-- > 0)
+                *p++ = '0';
+        }
+        return;
+    }
 
-  // WIP To be finished for 0.042
+    if (*dec < static_cast<T>(0))
-  if (scale)
+    {
-  {
+        *p++ = '-';
-    *p++ = '.';
+        *dec *= -1;
-    p += writeFractionalPart(dec, p, p-original_p, scale);
+    }
-  }
 
-  if (buflen <= p-original_p)
+    written = writeIntPart<T>(dec, p, buflen, scale);
-  {
+    p += written;
-    throw QueryDataExcept("toString() char buffer overflow.", formatErr);
+
-  }
+    if (scale)
+    {
+        *p++ = '.';
+        p += writeFractionalPart(dec, p, buflen-(p-original_p), scale);
+    }
+
+    if (buflen <= p-original_p)
+    {
+        throw QueryDataExcept("toString() char buffer overflow.", formatErr);
+    }
 }
 
-template<typename T>
+template
-void DataConvert::toString(T* dec, char *p, size_t buflen)
+void DataConvert::toString<int128_t>(int128_t* dec, uint8_t scale,
-{ 
+    char *p, unsigned int buflen);
-  uint64_t div = 10000000000000000000ULL;
-  // template this
-  uint128_t high = *dec;
-  uint128_t low;
-  low = high % div;
-  high /= div;
-  uint128_t mid;
-  mid = high % div;
-  high /= div;
- 
-  // WIP How to treat PODs here ?
-  // use typeof
-  // Or a templated structure
-  // Use uint64* to access parts of uint128 and remove pods
-  Int128Pod_t *high_pod = reinterpret_cast<Int128Pod_t*>(&high);
-  Int128Pod_t *mid_pod = reinterpret_cast<Int128Pod_t*>(&mid);
-  Int128Pod_t *low_pod = reinterpret_cast<Int128Pod_t*>(&low);
-  char* original_p = p;
-  int printed_chars = 0;
- 
-  // WIP replace snprintf with streams
-  if (high_pod->lo != 0) {
-    printed_chars = sprintf(p, "%lu", high_pod->lo);
-    p += printed_chars; 
-    printed_chars = sprintf(p, "%019lu", mid_pod->lo);
-    p += printed_chars;
-    sprintf(p, "%019lu", low_pod->lo);
-  } else if (mid_pod->lo != 0) {
-    printed_chars = sprintf(p, "%lu", mid_pod->lo);
-    p += printed_chars;
-    sprintf(p, "%019lu", low_pod->lo);
-  }
-  else {
-      sprintf(p, "%lu", low_pod->lo);
-  }
-  if (buflen <= p-original_p)
-    std::cout << "DataConvert::toString char buffer overflow" << std::endl;
-}
 
 // WIP MCOL-641
-// Template this
-// result must be calloc-ed
 void atoi128(const std::string& arg, int128_t& res)
 {
     res = 0;
@@ -1386,14 +1380,12 @@ void atoi128(const std::string& arg, int128_t& res)
         if (LIKELY(arg[j]-'0' >= 0))
             res = res*10 + arg[j] - '0';
     }
-    // Use bit shift if possible
     if (idx)
         res *= -1;
-    //toString(res, buf);
-    //std::cerr << "atoi_ " << buf <<endl;
-    //*res_ptr = res;
 }
 
+// WIP MCOL-641
+// remove this as we don't need this for wide-DECIMAL
 void atoi128(const std::string& arg, uint128_t& res)
 {
     res = 0;
@@ -1414,65 +1406,8 @@ void DataConvert::decimalToString(T* valuePtr,
     unsigned int buflen,
     cscDataType colDataType) // We don't need the last one
 {
-    T value = *valuePtr;
-    char* ptr = &buf[0];
-    size_t l1 = buflen;
-    if (value < static_cast<T>(0))
-    {
-        *ptr++ = '-';
-        value *= -1;
-        idbassert(l1 >= 2);
-        l1--;
-    }
 
-    //we want to move the last scale chars right by one spot to insert the dp
+    toString<T>(valuePtr, scale, buf, buflen);
-    //we want to move the trailing null as well, so it's really scale+1 chars
-
-    toString<T>(&value, ptr, buflen);
-
-    // Biggest ColumnStore supports is DECIMAL(38,x), or 38 total digits+dp+sign for column
-
-    if (scale == 0)
-        return;
-
-    //need to make sure we have enough leading zeros for this to work...
-    //at this point scale is always > 0
-    size_t l2 = 1;
-
-    if ((unsigned)scale > l1)
-    {
-        const char* zeros = "00000000000000000000000000000000000000"; //38 0's
-        size_t diff = 0;
-
-        if (*valuePtr != 0)
-            diff = scale - l1; //this will always be > 0
-        else
-            diff = scale;
-
-        memmove((ptr + diff), ptr, l1 + 1); //also move null
-        memcpy(ptr, zeros, diff);
-
-        if (*valuePtr != 0)
-            l1 = 0;
-        else
-            l1 = 1;
-    }
-    else if ((unsigned)scale == l1)
-    {
-        l1 = 0;
-        l2 = 2;
-    }
-    else
-    {
-        l1 -= scale;
-    }
-
-    memmove((ptr + l1 + l2), (ptr + l1), scale + 1); //also move null
-
-    if (l2 == 2)
-        *(ptr + l1++) = '0';
-
-    *(ptr + l1) = '.';
 }
 // Explicit instantiation
 template

utils/dataconvert/dataconvert.h

@@ -1044,11 +1044,8 @@ public:
     template <typename T>
     EXPORT static void decimalToString(T* value, uint8_t scale, char* buf, unsigned int buflen, cscDataType colDataType);
 
-    template<typename T>
-    static void toString(T* dec, char *p, size_t buflen);
     template <typename T>
-    EXPORT static void toString1(T* dec, char* p, const uint16_t buflen,
+    static void toString(T* dec, uint8_t scale, char* p, unsigned int buflen);
-        const uint8_t scale = 0);
     template <typename T>
     static size_t writeIntPart(T* dec, char* p, const uint16_t buflen,
         const uint8_t scale);

utils/rowgroup/rowgroup-tests.cpp

@@ -1,3 +1,20 @@
+/* Copyright (C) 2020 MariaDB Corporation
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License
+   as published by the Free Software Foundation; version 2 of
+   the License.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301, USA. */
+
 #include <gtest/gtest.h> // googletest header file
 #include <iostream>
 
@@ -195,7 +212,6 @@ TEST_F(RowDecimalTest, getBinaryFieldCheck) {
     }
 }
 
-// TBD Need to add asserts when toString will be finished
 TEST_F(RowDecimalTest, toStringCheck) {
     std::vector<std::string> exemplarVector;
     exemplarVector.push_back(std::string("0: NULL NULL NULL NULL NULL NULL "));