You've already forked mariadb-columnstore-engine
mirror of
https://github.com/mariadb-corporation/mariadb-columnstore-engine.git
synced 2025-07-30 19:23:07 +03:00
MCOL-641 DataConvert::decimalToString() refactoring.
This commit is contained in:
Roman Nozdrin
@ -24,6 +24,10 @@ using namespace std;
|
||||
using namespace execplan;
|
||||
#include "dataconvert.h"
|
||||
using namespace dataconvert;
|
||||
#include "joblisttypes.h"
|
||||
#include "columnwidth.h"
|
||||
|
||||
using CSCDataType = execplan::CalpontSystemCatalog::ColDataType;
|
||||
|
||||
TEST(DataConvertTest, Strtoll128)
|
||||
{
|
||||
@ -540,3 +544,102 @@ TEST(DataConvertTest, NumberIntValue)
|
||||
EXPECT_EQ(b2, b4);
|
||||
EXPECT_TRUE(pushWarning);
|
||||
}
|
||||
|
||||
TEST(DataConvertTest, atoiCheck) {
|
||||
std::vector<int128_t> expected;
|
||||
std::vector<std::string> decimalAsStringVec;
|
||||
decimalAsStringVec.push_back("99999999999999999999999999999999999999");
|
||||
decimalAsStringVec.push_back("-99999999999999999999999999999999999999");
|
||||
decimalAsStringVec.push_back("0");
|
||||
decimalAsStringVec.push_back("-0.042");
|
||||
expected.push_back(10000000000000000000ULL*(uint128_t)9999999999999999999ULL
|
||||
+9999999999999999999ULL);
|
||||
expected.push_back(-1*expected[0]);
|
||||
expected.push_back(0);
|
||||
expected.push_back(-42);
|
||||
for (int i=0; i < expected.size(); i++) {
|
||||
int128_t value;
|
||||
dataconvert::atoi128(decimalAsStringVec[i], value);
|
||||
EXPECT_EQ(expected[i], value);
|
||||
EXPECT_NE(expected[i], value-1);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(DataConvertTest, DecimalToStringCheckScale3) {
|
||||
std::vector<std::string> expected;
|
||||
std::vector<std::string> decimalAsStringVec;
|
||||
// scale 3
|
||||
uint8_t scale3 = 3;
|
||||
decimalAsStringVec.push_back("99999999999999999999999999999999999999");
|
||||
decimalAsStringVec.push_back("-99999999999999999999999999999999999999");
|
||||
decimalAsStringVec.push_back("0");
|
||||
decimalAsStringVec.push_back("-0.042");
|
||||
expected.push_back("99999999999999999999999999999999999.999");
|
||||
expected.push_back("-99999999999999999999999999999999999.999");
|
||||
expected.push_back("0.000");
|
||||
expected.push_back("-0.042");
|
||||
char buf[42];
|
||||
CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL;
|
||||
for (int i=0; i < expected.size(); i++) {
|
||||
int128_t value = -4;
|
||||
memset(buf, 0, 42);
|
||||
dataconvert::atoi128(decimalAsStringVec[i], value);
|
||||
dataconvert::DataConvert::decimalToString(&value, scale3, buf,
|
||||
utils::precisionByWidth(sizeof(value))+4, type);
|
||||
EXPECT_EQ(expected[i], std::string(buf));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(DataConvertTest, DecimalToStringCheckScale37) {
|
||||
std::vector<std::string> expected;
|
||||
std::vector<std::string> decimalAsStringVec;
|
||||
uint8_t scale37 = 37;
|
||||
decimalAsStringVec.push_back("99999999999999999999999999999999999999");
|
||||
decimalAsStringVec.push_back("-99999999999999999999999999999999999999");
|
||||
decimalAsStringVec.push_back("0");
|
||||
decimalAsStringVec.push_back("-42");
|
||||
decimalAsStringVec.push_back("-19999999999999999999999999999999999999");
|
||||
expected.push_back("9.9999999999999999999999999999999999999");
|
||||
expected.push_back("-9.9999999999999999999999999999999999999");
|
||||
expected.push_back("0.0000000000000000000000000000000000000");
|
||||
expected.push_back("-0.0000000000000000000000000000000000042");
|
||||
expected.push_back("-1.9999999999999999999999999999999999999");
|
||||
char buf[42];
|
||||
CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL;
|
||||
for (int i=0; i < expected.size(); i++) {
|
||||
int128_t value = -4;
|
||||
memset(buf, 0, 41);
|
||||
dataconvert::atoi128(decimalAsStringVec[i], value);
|
||||
dataconvert::DataConvert::decimalToString(&value, scale37, buf,
|
||||
utils::precisionByWidth(sizeof(value))+4, type);
|
||||
EXPECT_EQ(expected[i], std::string(buf));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(DataConvertTest, DecimalToStringCheckScale38) {
|
||||
std::vector<std::string> expected;
|
||||
std::vector<std::string> decimalAsStringVec;
|
||||
uint8_t scale38 = 38;
|
||||
decimalAsStringVec.push_back("99999999999999999999999999999999999999");
|
||||
decimalAsStringVec.push_back("-99999999999999999999999999999999999999");
|
||||
decimalAsStringVec.push_back("0");
|
||||
decimalAsStringVec.push_back("-42");
|
||||
expected.push_back("0.99999999999999999999999999999999999999");
|
||||
expected.push_back("-0.99999999999999999999999999999999999999");
|
||||
expected.push_back("0.00000000000000000000000000000000000000");
|
||||
expected.push_back("-0.00000000000000000000000000000000000042");
|
||||
char buf[42];
|
||||
CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL;
|
||||
for (int i=0; i < expected.size(); i++) {
|
||||
int128_t value = -4;
|
||||
memset(buf, 0, 41);
|
||||
dataconvert::atoi128(decimalAsStringVec[i], value);
|
||||
dataconvert::DataConvert::decimalToString(&value, scale38, buf,
|
||||
utils::precisionByWidth(sizeof(value))+4, type);
|
||||
EXPECT_EQ(expected[i], std::string(buf));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(DataConvertTest, ConvertColumnData) {
|
||||
}
|
||||
|
||||
|
||||
@ -1227,16 +1227,16 @@ bool stringToTimestampStruct(const string& data, TimeStamp& timeStamp, const str
|
||||
}
|
||||
|
||||
// WIP MCOL-641
|
||||
// Second DT is for POD representation.
|
||||
template <typename T>
|
||||
size_t DataConvert::writeIntPart(T* dec, char* p,
|
||||
const uint16_t buflen,
|
||||
const uint8_t scale)
|
||||
const uint8_t scale) //don't need this
|
||||
{
|
||||
T intPart = *dec;
|
||||
if (scale)
|
||||
{
|
||||
// optimize this
|
||||
//TODO Use dictionary to produce divisor
|
||||
// instead of a loop
|
||||
for (size_t i = 0; i < scale; i++)
|
||||
intPart /= 10;
|
||||
}
|
||||
@ -1292,90 +1292,84 @@ size_t DataConvert::writeFractionalPart(T* dec, char* p,
|
||||
const uint16_t buflen,
|
||||
const uint8_t scale)
|
||||
{
|
||||
T scaleDivisor = 10;
|
||||
for (size_t i = 1; i < scale; i++)
|
||||
scaleDivisor *= 10;
|
||||
//TODO Use dictionary instead of multiplication.
|
||||
T scaleDivisor = 10;
|
||||
for (size_t i = 1; i < scale; i++)
|
||||
scaleDivisor *= 10;
|
||||
|
||||
T fractionalPart = *dec % scaleDivisor;
|
||||
return writeIntPart(&fractionalPart, p, buflen, 0);
|
||||
T fractionalPart = *dec % scaleDivisor;
|
||||
// divide by the base untill we have non-zero quotinent
|
||||
size_t written = 0;
|
||||
scaleDivisor /= 10;
|
||||
while (scaleDivisor > 1 && *dec / scaleDivisor == 0)
|
||||
{
|
||||
*p++ = '0';
|
||||
written++;
|
||||
scaleDivisor /= 10;
|
||||
}
|
||||
written += writeIntPart(&fractionalPart, p, buflen, 0);
|
||||
if (written < scale)
|
||||
{
|
||||
p += written;
|
||||
for (size_t left = written; left < scale; left++)
|
||||
{
|
||||
*p++ = '0';
|
||||
}
|
||||
}
|
||||
return scale;
|
||||
}
|
||||
|
||||
// WIP MCOL-641
|
||||
// Limit this to Decimal only
|
||||
// Replace decimalToString with this one
|
||||
template<typename T>
|
||||
void DataConvert::toString1(T* dec, char *p, const uint16_t buflen,
|
||||
const uint8_t scale)
|
||||
void DataConvert::toString(T* dec, uint8_t scale,
|
||||
char *p, unsigned int buflen)
|
||||
{
|
||||
if (*dec < static_cast<T>(0))
|
||||
{
|
||||
*p++ = '-';
|
||||
*dec *= -1;
|
||||
}
|
||||
char* original_p = p;
|
||||
size_t written = 0;
|
||||
// Early return for NULL value
|
||||
int128_t sign = 0;
|
||||
// WIP use constants here
|
||||
// WIP Treat both magics here
|
||||
uint64_t* signPod = reinterpret_cast<uint64_t*>(&sign);
|
||||
signPod[1] = utils::BINARYNULLVALUEHIGH;
|
||||
|
||||
char* original_p = p;
|
||||
size_t written = 0;
|
||||
written = writeIntPart<T>(dec, p, buflen, scale);
|
||||
p += written;
|
||||
if (*dec == sign)
|
||||
{
|
||||
*p++ = '0';
|
||||
if (scale)
|
||||
{
|
||||
*p++ = '.';
|
||||
while (scale-- > 0)
|
||||
*p++ = '0';
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// WIP To be finished for 0.042
|
||||
if (scale)
|
||||
{
|
||||
*p++ = '.';
|
||||
p += writeFractionalPart(dec, p, p-original_p, scale);
|
||||
}
|
||||
if (*dec < static_cast<T>(0))
|
||||
{
|
||||
*p++ = '-';
|
||||
*dec *= -1;
|
||||
}
|
||||
|
||||
if (buflen <= p-original_p)
|
||||
{
|
||||
throw QueryDataExcept("toString() char buffer overflow.", formatErr);
|
||||
}
|
||||
written = writeIntPart<T>(dec, p, buflen, scale);
|
||||
p += written;
|
||||
|
||||
if (scale)
|
||||
{
|
||||
*p++ = '.';
|
||||
p += writeFractionalPart(dec, p, buflen-(p-original_p), scale);
|
||||
}
|
||||
|
||||
if (buflen <= p-original_p)
|
||||
{
|
||||
throw QueryDataExcept("toString() char buffer overflow.", formatErr);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
void DataConvert::toString(T* dec, char *p, size_t buflen)
|
||||
{
|
||||
uint64_t div = 10000000000000000000ULL;
|
||||
// template this
|
||||
uint128_t high = *dec;
|
||||
uint128_t low;
|
||||
low = high % div;
|
||||
high /= div;
|
||||
uint128_t mid;
|
||||
mid = high % div;
|
||||
high /= div;
|
||||
|
||||
// WIP How to treat PODs here ?
|
||||
// use typeof
|
||||
// Or a templated structure
|
||||
// Use uint64* to access parts of uint128 and remove pods
|
||||
Int128Pod_t *high_pod = reinterpret_cast<Int128Pod_t*>(&high);
|
||||
Int128Pod_t *mid_pod = reinterpret_cast<Int128Pod_t*>(&mid);
|
||||
Int128Pod_t *low_pod = reinterpret_cast<Int128Pod_t*>(&low);
|
||||
char* original_p = p;
|
||||
int printed_chars = 0;
|
||||
|
||||
// WIP replace snprintf with streams
|
||||
if (high_pod->lo != 0) {
|
||||
printed_chars = sprintf(p, "%lu", high_pod->lo);
|
||||
p += printed_chars;
|
||||
printed_chars = sprintf(p, "%019lu", mid_pod->lo);
|
||||
p += printed_chars;
|
||||
sprintf(p, "%019lu", low_pod->lo);
|
||||
} else if (mid_pod->lo != 0) {
|
||||
printed_chars = sprintf(p, "%lu", mid_pod->lo);
|
||||
p += printed_chars;
|
||||
sprintf(p, "%019lu", low_pod->lo);
|
||||
}
|
||||
else {
|
||||
sprintf(p, "%lu", low_pod->lo);
|
||||
}
|
||||
if (buflen <= p-original_p)
|
||||
std::cout << "DataConvert::toString char buffer overflow" << std::endl;
|
||||
}
|
||||
template
|
||||
void DataConvert::toString<int128_t>(int128_t* dec, uint8_t scale,
|
||||
char *p, unsigned int buflen);
|
||||
|
||||
// WIP MCOL-641
|
||||
// Template this
|
||||
// result must be calloc-ed
|
||||
void atoi128(const std::string& arg, int128_t& res)
|
||||
{
|
||||
res = 0;
|
||||
@ -1386,14 +1380,12 @@ void atoi128(const std::string& arg, int128_t& res)
|
||||
if (LIKELY(arg[j]-'0' >= 0))
|
||||
res = res*10 + arg[j] - '0';
|
||||
}
|
||||
// Use bit shift if possible
|
||||
if (idx)
|
||||
res *= -1;
|
||||
//toString(res, buf);
|
||||
//std::cerr << "atoi_ " << buf <<endl;
|
||||
//*res_ptr = res;
|
||||
}
|
||||
|
||||
// WIP MCOL-641
|
||||
// remove this as we don't need this for wide-DECIMAL
|
||||
void atoi128(const std::string& arg, uint128_t& res)
|
||||
{
|
||||
res = 0;
|
||||
@ -1414,65 +1406,8 @@ void DataConvert::decimalToString(T* valuePtr,
|
||||
unsigned int buflen,
|
||||
cscDataType colDataType) // We don't need the last one
|
||||
{
|
||||
T value = *valuePtr;
|
||||
char* ptr = &buf[0];
|
||||
size_t l1 = buflen;
|
||||
if (value < static_cast<T>(0))
|
||||
{
|
||||
*ptr++ = '-';
|
||||
value *= -1;
|
||||
idbassert(l1 >= 2);
|
||||
l1--;
|
||||
}
|
||||
|
||||
//we want to move the last scale chars right by one spot to insert the dp
|
||||
//we want to move the trailing null as well, so it's really scale+1 chars
|
||||
|
||||
toString<T>(&value, ptr, buflen);
|
||||
|
||||
// Biggest ColumnStore supports is DECIMAL(38,x), or 38 total digits+dp+sign for column
|
||||
|
||||
if (scale == 0)
|
||||
return;
|
||||
|
||||
//need to make sure we have enough leading zeros for this to work...
|
||||
//at this point scale is always > 0
|
||||
size_t l2 = 1;
|
||||
|
||||
if ((unsigned)scale > l1)
|
||||
{
|
||||
const char* zeros = "00000000000000000000000000000000000000"; //38 0's
|
||||
size_t diff = 0;
|
||||
|
||||
if (*valuePtr != 0)
|
||||
diff = scale - l1; //this will always be > 0
|
||||
else
|
||||
diff = scale;
|
||||
|
||||
memmove((ptr + diff), ptr, l1 + 1); //also move null
|
||||
memcpy(ptr, zeros, diff);
|
||||
|
||||
if (*valuePtr != 0)
|
||||
l1 = 0;
|
||||
else
|
||||
l1 = 1;
|
||||
}
|
||||
else if ((unsigned)scale == l1)
|
||||
{
|
||||
l1 = 0;
|
||||
l2 = 2;
|
||||
}
|
||||
else
|
||||
{
|
||||
l1 -= scale;
|
||||
}
|
||||
|
||||
memmove((ptr + l1 + l2), (ptr + l1), scale + 1); //also move null
|
||||
|
||||
if (l2 == 2)
|
||||
*(ptr + l1++) = '0';
|
||||
|
||||
*(ptr + l1) = '.';
|
||||
toString<T>(valuePtr, scale, buf, buflen);
|
||||
}
|
||||
// Explicit instantiation
|
||||
template
|
||||
|
||||
@ -1044,11 +1044,8 @@ public:
|
||||
template <typename T>
|
||||
EXPORT static void decimalToString(T* value, uint8_t scale, char* buf, unsigned int buflen, cscDataType colDataType);
|
||||
|
||||
template<typename T>
|
||||
static void toString(T* dec, char *p, size_t buflen);
|
||||
template <typename T>
|
||||
EXPORT static void toString1(T* dec, char* p, const uint16_t buflen,
|
||||
const uint8_t scale = 0);
|
||||
static void toString(T* dec, uint8_t scale, char* p, unsigned int buflen);
|
||||
template <typename T>
|
||||
static size_t writeIntPart(T* dec, char* p, const uint16_t buflen,
|
||||
const uint8_t scale);
|
||||
|
||||
@ -1,3 +1,20 @@
|
||||
/* Copyright (C) 2020 MariaDB Corporation
|
||||
|
||||
This program is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU General Public License
|
||||
as published by the Free Software Foundation; version 2 of
|
||||
the License.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
MA 02110-1301, USA. */
|
||||
|
||||
#include <gtest/gtest.h> // googletest header file
|
||||
#include <iostream>
|
||||
|
||||
@ -195,7 +212,6 @@ TEST_F(RowDecimalTest, getBinaryFieldCheck) {
|
||||
}
|
||||
}
|
||||
|
||||
// TBD Need to add asserts when toString will be finished
|
||||
TEST_F(RowDecimalTest, toStringCheck) {
|
||||
std::vector<std::string> exemplarVector;
|
||||
exemplarVector.push_back(std::string("0: NULL NULL NULL NULL NULL NULL "));
|
||||
|
||||
Reference in New Issue
Block a user