database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-641 1. Minor refactoring of decimalToString for int128_t.

Browse Source 
2. Update unit tests for decimalToString.
3. Allow support for wide decimal in TupleConstantStep::fillInConstants().
This commit is contained in:
Gagan Goel
2020-03-03 11:14:21 -05:00
committed by Roman Nozdrin
parent 2e8e7d52c3
commit 9b714274db
14 changed files with 504 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
dbcon/joblist/jlf_execplantojoblist.cpp
View File

@ -1893,7 +1893,9 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo)
if (ct.colDataType == CalpontSystemCatalog::DECIMAL && if (ct.colDataType == CalpontSystemCatalog::DECIMAL &&
ct.colWidth == 16) ct.colWidth == 16)
{ {
dataconvert::atoi128(constval, val128); bool saturate = false;
val128 = dataconvert::string_to_ll<int128_t>(constval, saturate);
// TODO MCOL-641 check saturate
} }
else else
{ {

3
dbcon/joblist/pcolscan.cpp
View File

@ -176,7 +176,8 @@ pColScanStep::pColScanStep(
// MCOL-641 WIP // MCOL-641 WIP
else if (fColType.colWidth > 8 else if (fColType.colWidth > 8
&& fColType.colDataType != CalpontSystemCatalog::BINARY && fColType.colDataType != CalpontSystemCatalog::BINARY
&& fColType.colDataType != CalpontSystemCatalog::DECIMAL) && fColType.colDataType != CalpontSystemCatalog::DECIMAL
&& fColType.colDataType != CalpontSystemCatalog::UDECIMAL)
{ {
fColType.colWidth = 8; fColType.colWidth = 8;
fIsDict = true; fIsDict = true;

3
dbcon/joblist/pcolstep.cpp
View File

@ -180,7 +180,8 @@ pColStep::pColStep(
// WIP MCOL-641 // WIP MCOL-641
else if (fColType.colWidth > 8 else if (fColType.colWidth > 8
&& fColType.colDataType != CalpontSystemCatalog::BINARY && fColType.colDataType != CalpontSystemCatalog::BINARY
&& fColType.colDataType != CalpontSystemCatalog::DECIMAL) && fColType.colDataType != CalpontSystemCatalog::DECIMAL
&& fColType.colDataType != CalpontSystemCatalog::UDECIMAL)
{ {
fColType.colWidth = 8; fColType.colWidth = 8;
fIsDict = true; fIsDict = true;

6
dbcon/joblist/tupleconstantstep.cpp
View File

@ -528,7 +528,11 @@ void TupleConstantStep::fillInConstants(const rowgroup::Row& rowIn, rowgroup::Ro
//fRowConst.copyField(rowOut.getData()+2, 0); // hardcoded 2 for rid length //fRowConst.copyField(rowOut.getData()+2, 0); // hardcoded 2 for rid length
for (uint32_t i = 1; i < rowOut.getColumnCount(); i++) for (uint32_t i = 1; i < rowOut.getColumnCount(); i++)
rowIn.copyField(rowOut, i, i - 1); // WIP MCOL-641 implement copyBinaryField inside copyField
if (UNLIKELY(rowIn.getColumnWidth(i - 1) == 16))
rowIn.copyBinaryField(rowOut, i, i - 1);
else
rowIn.copyField(rowOut, i, i - 1);
//memcpy(rowOut.getData()+rowOut.getOffset(1), rowIn.getData()+2, n); //memcpy(rowOut.getData()+rowOut.getOffset(1), rowIn.getData()+2, n);
} }

289
tests/dataconvert-tests.cpp
View File

@ -545,126 +545,189 @@ TEST(DataConvertTest, NumberIntValue)
EXPECT_TRUE(pushWarning); EXPECT_TRUE(pushWarning);
} }
TEST(DataConvertTest, atoiCheck) { TEST(DataConvertTest, DecimalToStringCheckScale0)
std::vector<int128_t> expected; {
std::vector<std::string> decimalAsStringVec; CalpontSystemCatalog::ColType ct;
decimalAsStringVec.push_back("99999999999999999999999999999999999999"); ct.colDataType = CalpontSystemCatalog::DECIMAL;
decimalAsStringVec.push_back("-99999999999999999999999999999999999999"); char buf[42];
decimalAsStringVec.push_back("0"); string input, expected;
decimalAsStringVec.push_back("-0.042"); ct.precision = 38;
expected.push_back(10000000000000000000ULL*(uint128_t)9999999999999999999ULL ct.scale = 0;
+9999999999999999999ULL); int128_t res;
expected.push_back(-1*expected[0]);
expected.push_back(0); // test simple values
expected.push_back(-42); res = 0;
for (int i=0; i < expected.size(); i++) { expected = "0";
int128_t value; DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
dataconvert::atoi128(decimalAsStringVec[i], value); EXPECT_EQ(string(buf), expected);
EXPECT_EQ(expected[i], value); res = 2;
EXPECT_NE(expected[i], value-1); expected = "2";
} DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -2;
expected = "-2";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = 123;
expected = "123";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -123;
expected = "-123";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
// test max/min decimal (i.e. 38 9's)
res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99;
expected = "99999999999999999999999999999999999999";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -res;
expected = "-99999999999999999999999999999999999999";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
// test trailing zeros
res = 123000;
expected = "123000";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -res;
expected = "-123000";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
}
TEST(DataConvertTest, DecimalToStringCheckScale10)
{
CalpontSystemCatalog::ColType ct;
ct.colDataType = CalpontSystemCatalog::DECIMAL;
char buf[42];
string input, expected;
ct.precision = 38;
ct.scale = 10;
int128_t res;
// test simple values
res = 0;
expected = "0.0000000000";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = 2;
expected = "0.0000000002";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -2;
expected = "-0.0000000002";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = 123;
expected = "0.0000000123";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -123;
expected = "-0.0000000123";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = 12345678901;
expected = "1.2345678901";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -12345678901;
expected = "-1.2345678901";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
// test max/min decimal (i.e. 38 9's)
res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99;
expected = "9999999999999999999999999999.9999999999";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -res;
expected = "-9999999999999999999999999999.9999999999";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
// test trailing zeros
res = 123000;
expected = "0.0000123000";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -res;
expected = "-0.0000123000";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
// test leading zeros
res = 10000000009;
expected = "1.0000000009";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -res;
expected = "-1.0000000009";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
} }
TEST(DataConvertTest, DecimalToStringCheckScale3) { TEST(DataConvertTest, DecimalToStringCheckScale38)
std::vector<std::string> expected; {
std::vector<std::string> decimalAsStringVec; CalpontSystemCatalog::ColType ct;
// scale 3 ct.colDataType = CalpontSystemCatalog::DECIMAL;
uint8_t scale3 = 3;
decimalAsStringVec.push_back("99999999999999999999999999999999999999");
decimalAsStringVec.push_back("-99999999999999999999999999999999999999");
decimalAsStringVec.push_back("0");
decimalAsStringVec.push_back("-0.042");
expected.push_back("99999999999999999999999999999999999.999");
expected.push_back("-99999999999999999999999999999999999.999");
expected.push_back("0.000");
expected.push_back("-0.042");
char buf[42]; char buf[42];
CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL; string input, expected;
for (int i=0; i < expected.size(); i++) { ct.precision = 38;
int128_t value = -4; ct.scale = 38;
memset(buf, 0, 42); int128_t res;
dataconvert::atoi128(decimalAsStringVec[i], value);
dataconvert::DataConvert::decimalToString(&value, scale3, buf,
utils::precisionByWidth(sizeof(value))+4, type);
EXPECT_EQ(expected[i], std::string(buf));
}
}
TEST(DataConvertTest, DecimalToStringCheckScale10) { // test simple values
std::vector<std::string> expected; res = 0;
std::vector<std::string> decimalAsStringVec; expected = "0.00000000000000000000000000000000000000";
// scale 3 DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
uint8_t scale3 = 10; EXPECT_EQ(string(buf), expected);
decimalAsStringVec.push_back("10000000009"); res = 2;
decimalAsStringVec.push_back("-10000000009"); expected = "0.00000000000000000000000000000000000002";
decimalAsStringVec.push_back("0"); DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
decimalAsStringVec.push_back("-10000000010"); EXPECT_EQ(string(buf), expected);
expected.push_back("1.0000000009"); res = -2;
expected.push_back("-1.0000000009"); expected = "-0.00000000000000000000000000000000000002";
expected.push_back("0.0000000000"); DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
expected.push_back("-1.0000000010"); EXPECT_EQ(string(buf), expected);
char buf[42]; res = 123;
CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL; expected = "0.00000000000000000000000000000000000123";
for (int i=0; i < expected.size(); i++) { DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
int128_t value = -4; EXPECT_EQ(string(buf), expected);
memset(buf, 0, 42); res = -123;
dataconvert::atoi128(decimalAsStringVec[i], value); expected = "-0.00000000000000000000000000000000000123";
dataconvert::DataConvert::decimalToString(&value, scale3, buf, DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
utils::precisionByWidth(sizeof(value))+4, type); EXPECT_EQ(string(buf), expected);
EXPECT_EQ(expected[i], std::string(buf)); res = ((((((int128_t)1234567890 * 10000000000) + 1234567890) * 10000000000) + 1234567890) * 100000000 ) + 12345678;
} expected = "0.12345678901234567890123456789012345678";
} DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -res;
expected = "-0.12345678901234567890123456789012345678";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
TEST(DataConvertTest, DecimalToStringCheckScale37) { // test max/min decimal (i.e. 38 9's)
std::vector<std::string> expected; res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99;
std::vector<std::string> decimalAsStringVec; expected = "0.99999999999999999999999999999999999999";
uint8_t scale37 = 37; DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
decimalAsStringVec.push_back("99999999999999999999999999999999999999"); EXPECT_EQ(string(buf), expected);
decimalAsStringVec.push_back("-99999999999999999999999999999999999999"); res = -res;
decimalAsStringVec.push_back("0"); expected = "-0.99999999999999999999999999999999999999";
decimalAsStringVec.push_back("-42"); DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
decimalAsStringVec.push_back("-19999999999999999999999999999999999999"); EXPECT_EQ(string(buf), expected);
expected.push_back("9.9999999999999999999999999999999999999");
expected.push_back("-9.9999999999999999999999999999999999999");
expected.push_back("0.0000000000000000000000000000000000000");
expected.push_back("-0.0000000000000000000000000000000000042");
expected.push_back("-1.9999999999999999999999999999999999999");
char buf[42];
CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL;
for (int i=0; i < expected.size(); i++) {
int128_t value = -4;
memset(buf, 0, 41);
dataconvert::atoi128(decimalAsStringVec[i], value);
dataconvert::DataConvert::decimalToString(&value, scale37, buf,
utils::precisionByWidth(sizeof(value))+4, type);
EXPECT_EQ(expected[i], std::string(buf));
}
}
TEST(DataConvertTest, DecimalToStringCheckScale38) { // test trailing zeros
std::vector<std::string> expected; res = 123000;
std::vector<std::string> decimalAsStringVec; expected = "0.00000000000000000000000000000000123000";
uint8_t scale38 = 38; DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
decimalAsStringVec.push_back("99999999999999999999999999999999999999"); EXPECT_EQ(string(buf), expected);
decimalAsStringVec.push_back("-99999999999999999999999999999999999999"); res = -res;
decimalAsStringVec.push_back("0"); expected = "-0.00000000000000000000000000000000123000";
decimalAsStringVec.push_back("-42"); DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
expected.push_back("0.99999999999999999999999999999999999999"); EXPECT_EQ(string(buf), expected);
expected.push_back("-0.99999999999999999999999999999999999999");
expected.push_back("0.00000000000000000000000000000000000000");
expected.push_back("-0.00000000000000000000000000000000000042");
char buf[42];
CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL;
for (int i=0; i < expected.size(); i++) {
int128_t value = -4;
memset(buf, 0, 41);
dataconvert::atoi128(decimalAsStringVec[i], value);
dataconvert::DataConvert::decimalToString(&value, scale38, buf,
utils::precisionByWidth(sizeof(value))+4, type);
EXPECT_EQ(expected[i], std::string(buf));
}
} }
TEST(DataConvertTest, ConvertColumnData) { TEST(DataConvertTest, ConvertColumnData) {
} }

299
tests/rowgroup-tests.cpp
View File

@ -20,6 +20,7 @@
#include "rowgroup.h" #include "rowgroup.h"
#include "columnwidth.h" #include "columnwidth.h"
#include "widedecimalutils.h"
#include "joblisttypes.h" #include "joblisttypes.h"
#include "dataconvert.h" #include "dataconvert.h"
@ -30,125 +31,138 @@ using int128_t = __int128;
using uint128_t = unsigned __int128; using uint128_t = unsigned __int128;
using CSCDataType = execplan::CalpontSystemCatalog::ColDataType; using CSCDataType = execplan::CalpontSystemCatalog::ColDataType;
class RowDecimalTest : public ::testing::Test { class RowDecimalTest : public ::testing::Test
protected: {
void SetUp() override { protected:
uint32_t precision = WIDE_DEC_PRECISION; void SetUp() override
uint32_t oid =3001; {
uint32_t precision = WIDE_DEC_PRECISION;
uint32_t oid = 3001;
std::vector<CSCDataType>types; std::vector<CSCDataType> types;
std::vector<decltype(precision)>precisionVec; std::vector<decltype(precision)> precisionVec;
std::vector<uint32_t> roids, tkeys, cscale; std::vector<uint32_t> roids, tkeys, cscale;
types.push_back(execplan::CalpontSystemCatalog::DECIMAL); std::vector<uint32_t> widthVec;
types.push_back(execplan::CalpontSystemCatalog::UDECIMAL);
for (size_t i=0; i <= 3; i++) {
types.push_back(execplan::CalpontSystemCatalog::DECIMAL); types.push_back(execplan::CalpontSystemCatalog::DECIMAL);
} types.push_back(execplan::CalpontSystemCatalog::UDECIMAL);
precisionVec.push_back(precision);
precisionVec.push_back(precision); for (size_t i = 0; i <= 3; i++)
precisionVec.push_back(18); types.push_back(execplan::CalpontSystemCatalog::DECIMAL);
precisionVec.push_back(9);
precisionVec.push_back(4); precisionVec.push_back(precision);
precisionVec.push_back(2); precisionVec.push_back(precision);
std::vector<uint32_t>widthVec; precisionVec.push_back(18);
uint32_t offset = INITIAL_ROW_OFFSET; precisionVec.push_back(9);
offsets.push_back(offset); precisionVec.push_back(4);
for (size_t i=0; i < types.size(); i++) { precisionVec.push_back(2);
uint8_t width = utils::widthByPrecision(precisionVec[i]);
widthVec.push_back(width); uint32_t offset = INITIAL_ROW_OFFSET;
offset += width; offsets.push_back(offset);
offsets.push_back(offset);
roids.push_back(oid+i); for (size_t i = 0; i < types.size(); i++)
tkeys.push_back(i+1); cscale.push_back(0); {
} uint8_t width = utils::widthByPrecision(precisionVec[i]);
/*offsets.push_back(INITIAL_ROW_OFFSET); widthVec.push_back(width);
offsets.push_back(16+INITIAL_ROW_OFFSET); offset += width;
offsets.push_back(16*2+INITIAL_ROW_OFFSET); offsets.push_back(offset);
roids.push_back(oid); roids.push_back(oid+1); roids.push_back(oid + i);
tkeys.push_back(1); tkeys.push_back(1); tkeys.push_back(i + 1);
cscale.push_back(0); cscale.push_back(0);*/ cscale.push_back(0);
}
/*offsets.push_back(INITIAL_ROW_OFFSET);
offsets.push_back(16+INITIAL_ROW_OFFSET);
offsets.push_back(16*2+INITIAL_ROW_OFFSET);
roids.push_back(oid); roids.push_back(oid+1);
tkeys.push_back(1); tkeys.push_back(1);
cscale.push_back(0); cscale.push_back(0);*/
rowgroup::RowGroup inRG(roids.size(), //column count rowgroup::RowGroup inRG(roids.size(), // column count
offsets, //oldOffset offsets, // oldOffset
roids, // column oids roids, // column oids
tkeys, //keys tkeys, // keys
types, // types types, // types
cscale, //scale cscale, // scale
precisionVec, // precision precisionVec, // precision
20, // sTableThreshold 20, // sTableThreshold
false //useStringTable false // useStringTable
); );
rg = inRG; rg = rgOut = inRG;
rgD.reinit(rg); rgD.reinit(rg);
rg.setData(&rgD); rgDOut.reinit(rgOut);
rg.setData(&rgD);
rgOut.setData(&rgDOut);
rg.initRow(&r); rg.initRow(&r);
rg.initRow(&rOutMappingCheck); rg.initRow(&rOutMappingCheck);
rowSize = r.getSize(); rowSize = r.getSize();
rg.getRow(0, &r); rg.getRow(0, &r);
int128_t nullValue = 0; int128_t nullValue = 0;
int128_t bigValue = 0; int128_t bigValue = 0;
uint64_t* uint128_pod = reinterpret_cast<uint64_t*>(&nullValue);
uint128_pod[0] = joblist::UBIGINTNULL;
uint128_pod[1] = joblist::UBIGINTEMPTYROW;
bigValue = -static_cast<int128_t>(0xFFFFFFFF)*0xFFFFFFFFFFFFFFFF;
sValueVector.push_back(nullValue);
sValueVector.push_back(-42);
sValueVector.push_back(bigValue);
sValueVector.push_back(0);
sValueVector.push_back(nullValue-1);
uValueVector.push_back(nullValue); utils::setWideDecimalNullValue(nullValue);
uValueVector.push_back(42); bigValue = -static_cast<int128_t>(0xFFFFFFFF)*0xFFFFFFFFFFFFFFFF;
uValueVector.push_back(bigValue);
uValueVector.push_back(0);
uValueVector.push_back(nullValue-1);
s8ValueVector.push_back(joblist::TINYINTNULL); sValueVector.push_back(nullValue);
s8ValueVector.push_back(-0x79); sValueVector.push_back(-42);
s8ValueVector.push_back(0); sValueVector.push_back(bigValue);
s8ValueVector.push_back(0x81); sValueVector.push_back(0);
s8ValueVector.push_back(joblist::TINYINTNULL-1); sValueVector.push_back(nullValue-1);
s16ValueVector.push_back(joblist::SMALLINTNULL); uValueVector.push_back(nullValue);
s16ValueVector.push_back(-0x79); uValueVector.push_back(42);
s16ValueVector.push_back(0); uValueVector.push_back(bigValue);
s16ValueVector.push_back(0x81); uValueVector.push_back(0);
s16ValueVector.push_back(joblist::SMALLINTNULL-1); uValueVector.push_back(nullValue-1);
s32ValueVector.push_back(joblist::INTNULL); s8ValueVector.push_back(joblist::TINYINTNULL);
s32ValueVector.push_back(-0x79); s8ValueVector.push_back(-0x79);
s32ValueVector.push_back(0); s8ValueVector.push_back(0);
s32ValueVector.push_back(0x81); s8ValueVector.push_back(0x81);
s32ValueVector.push_back(joblist::INTNULL-1); s8ValueVector.push_back(joblist::TINYINTNULL-1);
s64ValueVector.push_back(joblist::BIGINTNULL); s16ValueVector.push_back(joblist::SMALLINTNULL);
s64ValueVector.push_back(-0x79); s16ValueVector.push_back(-0x79);
s64ValueVector.push_back(0); s16ValueVector.push_back(0);
s64ValueVector.push_back(0x81); s16ValueVector.push_back(0x81);
s64ValueVector.push_back(joblist::BIGINTNULL-1); s16ValueVector.push_back(joblist::SMALLINTNULL-1);
for(size_t i = 0; i < sValueVector.size(); i++) { s32ValueVector.push_back(joblist::INTNULL);
r.setBinaryField_offset(&sValueVector[i], s32ValueVector.push_back(-0x79);
sizeof(sValueVector[0]), offsets[0]); s32ValueVector.push_back(0);
r.setBinaryField_offset(&uValueVector[i], s32ValueVector.push_back(0x81);
sizeof(uValueVector[0]), offsets[1]); s32ValueVector.push_back(joblist::INTNULL-1);
r.setIntField(s64ValueVector[i], 2);
r.setIntField(s32ValueVector[i], 3); s64ValueVector.push_back(joblist::BIGINTNULL);
r.setIntField(s16ValueVector[i], 4); s64ValueVector.push_back(-0x79);
r.setIntField(s8ValueVector[i], 5); s64ValueVector.push_back(0);
r.nextRow(rowSize); s64ValueVector.push_back(0x81);
s64ValueVector.push_back(joblist::BIGINTNULL-1);
for (size_t i = 0; i < sValueVector.size(); i++)
{
r.setBinaryField_offset(&sValueVector[i],
sizeof(sValueVector[0]), offsets[0]);
r.setBinaryField_offset(&uValueVector[i],
sizeof(uValueVector[0]), offsets[1]);
r.setIntField(s64ValueVector[i], 2);
r.setIntField(s32ValueVector[i], 3);
r.setIntField(s16ValueVector[i], 4);
r.setIntField(s8ValueVector[i], 5);
r.nextRow(rowSize);
}
rowCount = sValueVector.size();
} }
rowCount = sValueVector.size();
}
// void TearDown() override {}
rowgroup::Row r; // void TearDown() override {}
rowgroup::Row r, rOut;
rowgroup::Row rOutMappingCheck; rowgroup::Row rOutMappingCheck;
rowgroup::RowGroup rg; rowgroup::RowGroup rg, rgOut;
rowgroup::RGData rgD; rowgroup::RGData rgD, rgDOut;
uint32_t rowSize; uint32_t rowSize;
size_t rowCount; size_t rowCount;
std::vector<int128_t> sValueVector; std::vector<int128_t> sValueVector;
@ -160,9 +174,12 @@ class RowDecimalTest : public ::testing::Test {
std::vector<uint32_t> offsets; std::vector<uint32_t> offsets;
}; };
TEST_F(RowDecimalTest, NonNULLValuesCheck) { TEST_F(RowDecimalTest, NonNullValueCheck)
{
rg.getRow(1, &r); rg.getRow(1, &r);
for (size_t i = 1; i <= sValueVector.size(); i++) {
for (size_t i = 1; i <= sValueVector.size(); i++)
{
EXPECT_FALSE(r.isNullValue(0)); EXPECT_FALSE(r.isNullValue(0));
EXPECT_FALSE(r.isNullValue(1)); EXPECT_FALSE(r.isNullValue(1));
EXPECT_FALSE(r.isNullValue(2)); EXPECT_FALSE(r.isNullValue(2));
@ -173,9 +190,11 @@ TEST_F(RowDecimalTest, NonNULLValuesCheck) {
} }
} }
TEST_F(RowDecimalTest, initToNullANDisNullValueValueCheck) { TEST_F(RowDecimalTest, InitToNullAndIsNullValueCheck)
{
rg.getRow(0, &r); rg.getRow(0, &r);
r.initToNull(); r.initToNull();
EXPECT_TRUE(r.isNullValue(0)); EXPECT_TRUE(r.isNullValue(0));
EXPECT_TRUE(r.isNullValue(1)); EXPECT_TRUE(r.isNullValue(1));
EXPECT_TRUE(r.isNullValue(2)); EXPECT_TRUE(r.isNullValue(2));
@ -184,14 +203,16 @@ TEST_F(RowDecimalTest, initToNullANDisNullValueValueCheck) {
EXPECT_TRUE(r.isNullValue(5)); EXPECT_TRUE(r.isNullValue(5));
} }
TEST_F(RowDecimalTest, getBinaryFieldCheck) { TEST_F(RowDecimalTest, GetBinaryFieldCheck)
{
rg.getRow(0, &r); rg.getRow(0, &r);
uint128_t* u128Value; uint128_t* u128Value;
int128_t* s128Value; int128_t* s128Value;
// std::remove_reference<decltype(*u128Value)>::type uType; // std::remove_reference<decltype(*u128Value)>::type uType;
// std::remove_reference<decltype(*s128Value)>::type sType; // std::remove_reference<decltype(*s128Value)>::type sType;
for (size_t i = 0; i < sValueVector.size(); i++) { for (size_t i = 0; i < sValueVector.size(); i++)
{
s128Value = r.getBinaryField<int128_t>(0); s128Value = r.getBinaryField<int128_t>(0);
EXPECT_EQ(sValueVector[i], *s128Value); EXPECT_EQ(sValueVector[i], *s128Value);
u128Value = r.getBinaryField<uint128_t>(1); u128Value = r.getBinaryField<uint128_t>(1);
@ -204,26 +225,32 @@ TEST_F(RowDecimalTest, getBinaryFieldCheck) {
} }
} }
TEST_F(RowDecimalTest, toStringCheck) { TEST_F(RowDecimalTest, ToStringCheck)
{
std::vector<std::string> exemplarVector; std::vector<std::string> exemplarVector;
exemplarVector.push_back(std::string("0: NULL NULL NULL NULL NULL NULL ")); exemplarVector.push_back(std::string("0: NULL NULL NULL NULL NULL NULL "));
exemplarVector.push_back(std::string("0: -42 42 -121 -121 -121 -121 ")); exemplarVector.push_back(std::string("0: -42 42 -121 -121 -121 -121 "));
exemplarVector.push_back(std::string("0: -79228162495817593515539431425 -79228162495817593515539431425 0 0 0 0 ")); exemplarVector.push_back(std::string("0: -79228162495817593515539431425 -79228162495817593515539431425 0 0 0 0 "));
exemplarVector.push_back(std::string("0: 0 0 129 129 129 -127 ")); exemplarVector.push_back(std::string("0: 0 0 129 129 129 -127 "));
exemplarVector.push_back(std::string("0: -3 -3 9223372036854775807 2147483647 32767 127 ")); exemplarVector.push_back(std::string("0: 170141183460469231731687303715884105727 170141183460469231731687303715884105727 9223372036854775807 2147483647 32767 127 "));
rg.getRow(0, &r); rg.getRow(0, &r);
r.initToNull(); r.initToNull();
for (auto &el: exemplarVector) {
for (auto& el: exemplarVector)
{
EXPECT_EQ(el, r.toString()); EXPECT_EQ(el, r.toString());
r.nextRow(rowSize); r.nextRow(rowSize);
} }
} }
TEST_F(RowDecimalTest, toCSVCheck) { TEST_F(RowDecimalTest, ToCSVCheck)
{
} }
TEST_F(RowDecimalTest, applyMappingCheck) { TEST_F(RowDecimalTest, ApplyMappingCheck)
{
int mapping[] = {0, 1, -1, -1, -1, -1}; int mapping[] = {0, 1, -1, -1, -1, -1};
rg.getRow(1, &r); rg.getRow(1, &r);
rg.getRow(2, &rOutMappingCheck); rg.getRow(2, &rOutMappingCheck);
@ -233,11 +260,49 @@ TEST_F(RowDecimalTest, applyMappingCheck) {
EXPECT_NE(uValueVector[1], *u128Value); EXPECT_NE(uValueVector[1], *u128Value);
applyMapping(mapping, r, &rOutMappingCheck); applyMapping(mapping, r, &rOutMappingCheck);
s128Value = rOutMappingCheck.getBinaryField<int128_t>(0); s128Value = rOutMappingCheck.getBinaryField<int128_t>(0);
EXPECT_EQ(sValueVector[1], *s128Value);
u128Value = rOutMappingCheck.getBinaryField<uint128_t>(1); u128Value = rOutMappingCheck.getBinaryField<uint128_t>(1);
EXPECT_EQ(sValueVector[1], *s128Value);
EXPECT_EQ(uValueVector[1], *u128Value); EXPECT_EQ(uValueVector[1], *u128Value);
} }
// WIP TEST_F(RowDecimalTest, CopyBinaryFieldCheck)
TEST_F(RowDecimalTest, RowEqualsCheck) { {
int128_t constVal = 1;
int128_t *col1Out, *col1In;
uint128_t *col2Out, *col2In;
rgOut.getRow(0, &rOut);
for (size_t i = 0; i < sValueVector.size(); i++)
{
rOut.setBinaryField_offset(&constVal, 16, offsets[0]);
rOut.setBinaryField_offset((uint128_t*)&constVal, 16, offsets[1]);
rOut.nextRow(rowSize);
}
rgOut.initRow(&rOut);
rgOut.getRow(0, &rOut);
rg.getRow(0, &r);
for (size_t i = 0; i < sValueVector.size(); i++)
{
col1In = r.getBinaryField<int128_t>(0);
col1Out = rOut.getBinaryField<int128_t>(0);
col2In = r.getBinaryField<uint128_t>(1);
col2Out = rOut.getBinaryField<uint128_t>(1);
EXPECT_NE(*col1In, *col1Out);
EXPECT_NE(*col2In, *col2Out);
r.copyBinaryField(rOut, 0, 0);
r.copyBinaryField(rOut, 1, 1);
col1Out = rOut.getBinaryField<int128_t>(0);
col2Out = rOut.getBinaryField<uint128_t>(1);
EXPECT_EQ(*col1In, *col1Out);
EXPECT_EQ(*col2In, *col2Out);
r.nextRow(rowSize);
rOut.nextRow(rowSize);
}
}
// WIP
TEST_F(RowDecimalTest, RowEqualsCheck)
{
} }

216
utils/dataconvert/dataconvert.cpp
View File

@ -125,6 +125,7 @@ const uint64_t columnstore_pow_10[20] =
1000000000000000000ULL, 1000000000000000000ULL,
10000000000000000000ULL 10000000000000000000ULL
}; };
template <class T> template <class T>
bool from_string(T& t, const std::string& s, std::ios_base & (*f)(std::ios_base&)) bool from_string(T& t, const std::string& s, std::ios_base & (*f)(std::ios_base&))
{ {
@ -1249,115 +1250,144 @@ bool stringToTimestampStruct(const string& data, TimeStamp& timeStamp, const str
} }
size_t DataConvert::writeIntPart(int128_t* dec, char* p, size_t DataConvert::writeIntPart(int128_t* dec,
const uint16_t buflen, char* p,
const unsigned int buflen,
const uint8_t scale) const uint8_t scale)
{ {
int128_t intPart = *dec; int128_t intPart = *dec;
if (scale) int128_t high = 0, mid = 0, low = 0;
{ uint64_t div = 10000000000000000000ULL;
uint8_t maxPowOf10 = sizeof(columnstore_pow_10)/sizeof(columnstore_pow_10[0])-1;
switch (scale/maxPowOf10) if (scale)
{ {
case(2): const uint8_t maxPowOf10 =
intPart /= columnstore_pow_10[maxPowOf10]; (sizeof(columnstore_pow_10) / sizeof(columnstore_pow_10[0])) - 1;
intPart /= columnstore_pow_10[maxPowOf10];
break; // Assuming scale = [0, 56]
case(1): switch (scale / maxPowOf10)
intPart /= columnstore_pow_10[maxPowOf10]; {
case(0): case 2: // scale = [38, 56]
intPart /= columnstore_pow_10[scale%maxPowOf10]; intPart /= columnstore_pow_10[maxPowOf10];
intPart /= columnstore_pow_10[maxPowOf10];
low = intPart;
break;
case 1: // scale = [19, 37]
intPart /= columnstore_pow_10[maxPowOf10];
intPart /= columnstore_pow_10[scale % maxPowOf10];
low = intPart % div;
mid = intPart / div;
break;
case 0: // scale = [0, 18]
intPart /= columnstore_pow_10[scale % maxPowOf10];
low = intPart % div;
intPart /= div;
mid = intPart % div;
high = intPart / div;
break;
default:
throw QueryDataExcept("writeIntPart() bad scale", formatErr);
}
}
else
{
low = intPart % div;
intPart /= div;
mid = intPart % div;
high = intPart / div;
} }
}
uint64_t div = 10000000000000000000ULL; // pod[0] is low 8 bytes, pod[1] is high 8 bytes
int128_t high = intPart; uint64_t* high_pod = reinterpret_cast<uint64_t*>(&high);
int128_t low; uint64_t* mid_pod = reinterpret_cast<uint64_t*>(&mid);
low = high % div; uint64_t* low_pod = reinterpret_cast<uint64_t*>(&low);
high /= div; char* original_p = p;
int128_t mid;
mid = high % div;
high /= div;
// pod[0] is low 8 byte, pod[1] is high // WIP replace sprintf with streams
uint64_t* high_pod = reinterpret_cast<uint64_t*>(&high); if (high_pod[0] != 0)
uint64_t* mid_pod = reinterpret_cast<uint64_t*>(&mid); {
uint64_t* low_pod = reinterpret_cast<uint64_t*>(&low); p += sprintf(p, "%lu", high_pod[0]);
char* original_p = p; p += sprintf(p, "%019lu", mid_pod[0]);
int written = 0; p += sprintf(p, "%019lu", low_pod[0]);
}
else if (mid_pod[0] != 0)
{
p += sprintf(p, "%lu", mid_pod[0]);
p += sprintf(p, "%019lu", low_pod[0]);
}
else
{
p += sprintf(p, "%lu", low_pod[0]);
}
// WIP replace snprintf with streams size_t written = p - original_p;
if (high_pod[0] != 0)
{
written = sprintf(p, "%lu", high_pod[0]);
p += written;
written = sprintf(p, "%019lu", mid_pod[0]);
p += written;
sprintf(p, "%019lu", low_pod[0]);
}
else if (mid_pod[0] != 0)
{
written = sprintf(p, "%lu", mid_pod[0]);
p += written;
written = sprintf(p, "%019lu", low_pod[0]);
p += written;
}
else
{
written = sprintf(p, "%lu", low_pod[0]);
p += written;
}
if (buflen <= p-original_p) if (buflen <= written)
{ {
throw QueryDataExcept("writeIntPart() char buffer overflow.", formatErr); throw QueryDataExcept("writeIntPart() char buffer overflow.", formatErr);
} }
return p-original_p;
return written;
} }
size_t DataConvert::writeFractionalPart(int128_t* dec, char* p, size_t DataConvert::writeFractionalPart(int128_t* dec,
const uint16_t buflen, char* p,
const unsigned int buflen,
const uint8_t scale) const uint8_t scale)
{ {
int128_t scaleDivisor = 1; int128_t scaleDivisor = 1;
uint8_t maxPowOf10 = sizeof(columnstore_pow_10)/sizeof(columnstore_pow_10[0])-1; const uint8_t maxPowOf10 =
switch (scale/maxPowOf10) (sizeof(columnstore_pow_10) / sizeof(columnstore_pow_10[0])) - 1;
switch (scale / maxPowOf10)
{ {
case(2): case 2:
scaleDivisor *= columnstore_pow_10[maxPowOf10]; scaleDivisor *= columnstore_pow_10[maxPowOf10];
scaleDivisor *= columnstore_pow_10[maxPowOf10]; scaleDivisor *= columnstore_pow_10[maxPowOf10];
break; break;
case(1): case 1:
scaleDivisor *= columnstore_pow_10[maxPowOf10]; scaleDivisor *= columnstore_pow_10[maxPowOf10];
case(0): case 0:
scaleDivisor *= columnstore_pow_10[scale%maxPowOf10]; scaleDivisor *= columnstore_pow_10[scale % maxPowOf10];
} }
int128_t fractionalPart = *dec % scaleDivisor; int128_t fractionalPart = *dec % scaleDivisor;
// divide by the base untill we have non-zero quotinent
// divide by the base until we have non-zero quotient
size_t written = 0; size_t written = 0;
scaleDivisor /= 10; scaleDivisor /= 10;
while (scaleDivisor > 1 && fractionalPart/scaleDivisor == 0)
char* original_p = p;
while (scaleDivisor > 1 && fractionalPart / scaleDivisor == 0)
{ {
*p++ = '0'; *p++ = '0';
written++; written++;
scaleDivisor /= 10; scaleDivisor /= 10;
} }
written += writeIntPart(&fractionalPart, p, buflen, 0);
p += writeIntPart(&fractionalPart, p, buflen - written, 0);
written = p - original_p;
// this should never be true
if (written < scale) if (written < scale)
{ {
p += written;
for (size_t left = written; left < scale; left++) for (size_t left = written; left < scale; left++)
{ {
*p++ = '0'; *p++ = '0';
} }
} }
return scale;
return written;
} }
void DataConvert::toString(int128_t* dec, uint8_t scale, void DataConvert::decimalToString(int128_t* dec,
char *p, unsigned int buflen) const uint8_t scale,
char *p,
const unsigned int buflen,
cscDataType colDataType) // colDataType is redundant
{ {
char* original_p = p; char* original_p = p;
size_t written = 0; size_t written = 0;
@ -1379,54 +1409,20 @@ void DataConvert::toString(int128_t* dec, uint8_t scale,
if (scale) if (scale)
{ {
*p++ = '.'; *p++ = '.';
p += writeFractionalPart(dec, p, buflen-(p-original_p), scale); written = p - original_p;
p += writeFractionalPart(dec, p, buflen - written, scale);
} }
*p = '\0'; *p = '\0';
if (buflen <= p-original_p) written = p - original_p;
if (buflen <= written)
{ {
throw QueryDataExcept("toString() char buffer overflow.", formatErr); throw QueryDataExcept("toString() char buffer overflow.", formatErr);
} }
} }
// WIP MCOL-641
void atoi128(const std::string& arg, int128_t& res)
{
res = 0;
size_t idx = (arg[0] == '-') ? 1 : 0;
for (size_t j = idx; j < arg.size(); j++)
{
// WIP Optimize this
if (LIKELY(arg[j]-'0' >= 0))
res = res*10 + arg[j] - '0';
}
if (idx)
res *= -1;
}
// WIP MCOL-641
// remove this as we don't need this for wide-DECIMAL
void atoi128(const std::string& arg, uint128_t& res)
{
res = 0;
for (size_t j = 0; j < arg.size(); j++)
{
// WIP Optimize this
if (LIKELY(arg[j]-'0' >= 0))
res = res*10 + arg[j] - '0';
}
}
void DataConvert::decimalToString(int128_t* valuePtr,
uint8_t scale,
char* buf,
unsigned int buflen,
cscDataType colDataType) // We don't need the last one
{
toString(valuePtr, scale, buf, buflen);
}
boost::any boost::any
DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType,
const std::string& dataOrig, bool& pushWarning, const std::string& dataOrig, bool& pushWarning,

12
utils/dataconvert/dataconvert.h
View File

@ -152,8 +152,6 @@ struct Int128Pod_struct
typedef Int128Pod_struct Int128Pod_t; typedef Int128Pod_struct Int128Pod_t;
void atoi128(const std::string& arg, int128_t& res);
enum CalpontDateTimeFormat enum CalpontDateTimeFormat
{ {
CALPONTDATE_ENUM = 1, // date format is: "YYYY-MM-DD" CALPONTDATE_ENUM = 1, // date format is: "YYYY-MM-DD"
@ -177,9 +175,6 @@ struct MySQLTime
} }
}; };
void atoi128(const std::string& arg, int128_t& res);
void atoi128(const std::string& arg, uint128_t& res);
/** /**
* This function converts the timezone represented as a string * This function converts the timezone represented as a string
* in the format "+HH:MM" or "-HH:MM" to a signed offset in seconds * in the format "+HH:MM" or "-HH:MM" to a signed offset in seconds
@ -1041,12 +1036,11 @@ public:
EXPORT static bool isNullData(execplan::ColumnResult* cr, int rownum, execplan::CalpontSystemCatalog::ColType colType); EXPORT static bool isNullData(execplan::ColumnResult* cr, int rownum, execplan::CalpontSystemCatalog::ColType colType);
static inline std::string decimalToString(int64_t value, uint8_t scale, cscDataType colDataType); static inline std::string decimalToString(int64_t value, uint8_t scale, cscDataType colDataType);
static inline void decimalToString(int64_t value, uint8_t scale, char* buf, unsigned int buflen, cscDataType colDataType); static inline void decimalToString(int64_t value, uint8_t scale, char* buf, unsigned int buflen, cscDataType colDataType);
static void decimalToString(int128_t* value, uint8_t scale, char* buf, unsigned int buflen, cscDataType colDataType);
static void toString(int128_t* dec, uint8_t scale, char* p, unsigned int buflen); static void decimalToString(int128_t* dec, const uint8_t scale, char* buf, const unsigned int buflen, cscDataType colDataType);
static size_t writeIntPart(int128_t* dec, char* p, const uint16_t buflen, static size_t writeIntPart(int128_t* dec, char* p, const unsigned int buflen,
const uint8_t scale); const uint8_t scale);
static size_t writeFractionalPart(int128_t* dec, char* p, const uint16_t buflen, static size_t writeFractionalPart(int128_t* dec, char* p, const unsigned int buflen,
const uint8_t scale); const uint8_t scale);
static inline void int128Max(int128_t& i) static inline void int128Max(int128_t& i)

13
utils/rowgroup/rowgroup.cpp
View File

@ -633,22 +633,23 @@ string Row::toString() const
os << " " << dec; os << " " << dec;
break; break;
} }
// WIP MCOL-641
case CalpontSystemCatalog::BINARY: case CalpontSystemCatalog::BINARY:
std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl; std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl;
break; break;
// WIP
case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL: case CalpontSystemCatalog::UDECIMAL:
if (colWidths[i] == sizeof(int128_t)) if (utils::isWide(colWidths[i]))
{ {
char *buf = (char*)alloca(precision[i] + 3); unsigned int buflen = precision[i] + 3;
char *buf = (char*)alloca(buflen);
// empty the buffer // empty the buffer
dataconvert::DataConvert::decimalToString(getBinaryField<int128_t>(i), dataconvert::DataConvert::decimalToString(getBinaryField<int128_t>(i),
scale[i], buf, precision[i]+3, types[i]); scale[i], buf, buflen, types[i]);
os << buf << " "; os << buf << " ";
break; break;
} }
// fallback if the the legacy DECIMAL // fallthrough if the legacy DECIMAL
default: default:
os << getIntField(i) << " "; os << getIntField(i) << " ";
break; break;
@ -850,10 +851,8 @@ void Row::initToNull()
break; break;
case 16 : case 16 :
{
utils::setWideDecimalNullValue(reinterpret_cast<int128_t&>(data[offsets[i]])); utils::setWideDecimalNullValue(reinterpret_cast<int128_t&>(data[offsets[i]]));
break; break;
}
default: default:
*((int64_t*) &data[offsets[i]]) = static_cast<int64_t>(joblist::BIGINTNULL); *((int64_t*) &data[offsets[i]]) = static_cast<int64_t>(joblist::BIGINTNULL);
break; break;

9
utils/rowgroup/rowgroup.h
View File

@ -475,6 +475,9 @@ public:
// that's not string-table safe, this one is // that's not string-table safe, this one is
inline void copyField(Row& dest, uint32_t destIndex, uint32_t srcIndex) const; inline void copyField(Row& dest, uint32_t destIndex, uint32_t srcIndex) const;
// WIP MCOL-641
inline void copyBinaryField(Row& dest, uint32_t destIndex, uint32_t srcIndex) const;
std::string toString() const; std::string toString() const;
std::string toCSV() const; std::string toCSV() const;
@ -1229,6 +1232,12 @@ inline void Row::copyField(Row& out, uint32_t destIndex, uint32_t srcIndex) cons
out.setIntField(getIntField(srcIndex), destIndex); out.setIntField(getIntField(srcIndex), destIndex);
} }
// WIP MCOL-641
inline void Row::copyBinaryField(Row& out, uint32_t destIndex, uint32_t srcIndex) const
{
out.setBinaryField<int128_t>(getBinaryField<int128_t>(srcIndex), 16, destIndex);
}
inline void Row::setRid(uint64_t rid) inline void Row::setRid(uint64_t rid)
{ {
*((uint16_t*) data) = rid & 0xffff; *((uint16_t*) data) = rid & 0xffff;

4
writeengine/bulk/we_bulkloadbuffer.cpp
View File

@ -994,7 +994,9 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
} }
else else
{ {
dataconvert::atoi128(string(field), bigllVal); bool saturate = false;
bigllVal = dataconvert::string_to_ll<int128_t>(string(field), saturate);
// TODO MCOL-641 check saturate
} }
} }
else else

18
writeengine/server/we_dmlcommandproc.cpp
View File

@ -815,24 +815,19 @@ uint8_t WE_DMLCommandProc::rollbackVersion(ByteStream& bs, std::string& err)
uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId) uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId)
{ {
int rc = 0; int rc = 0;
//cout << "processBatchInsert received bytestream length " << bs.length() << endl;
InsertDMLPackage insertPkg; InsertDMLPackage insertPkg;
ByteStream::quadbyte tmp32; ByteStream::quadbyte tmp32;
bs >> tmp32; bs >> tmp32;
//cout << "processBatchInsert got transaction id " << tmp32 << endl;
bs >> PMId; bs >> PMId;
//cout << "processBatchInsert gor PMId " << PMId << endl;
insertPkg.read( bs); insertPkg.read( bs);
uint32_t sessionId = insertPkg.get_SessionID(); uint32_t sessionId = insertPkg.get_SessionID();
//cout << " processBatchInsert for session " << sessionId << endl;
DMLTable* tablePtr = insertPkg.get_Table(); DMLTable* tablePtr = insertPkg.get_Table();
bool isAutocommitOn = insertPkg.get_isAutocommitOn(); bool isAutocommitOn = insertPkg.get_isAutocommitOn();
if (idbdatafile::IDBPolicy::useHdfs()) if (idbdatafile::IDBPolicy::useHdfs())
isAutocommitOn = true; isAutocommitOn = true;
//cout << "This session isAutocommitOn is " << isAutocommitOn << endl;
BRM::TxnID txnid; BRM::TxnID txnid;
txnid.id = tmp32; txnid.id = tmp32;
txnid.valid = true; txnid.valid = true;
@ -858,7 +853,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
try try
{ {
ridList = systemCatalogPtr->columnRIDs(tableName, true); ridList = systemCatalogPtr->columnRIDs(tableName, true);
roPair = systemCatalogPtr->tableRID( tableName); roPair = systemCatalogPtr->tableRID(tableName);
} }
catch (std::exception& ex) catch (std::exception& ex)
{ {
@ -867,7 +862,6 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
return rc; return rc;
} }
std::vector<OID> dctnryStoreOids(ridList.size()) ; std::vector<OID> dctnryStoreOids(ridList.size()) ;
std::vector<Column> columns; std::vector<Column> columns;
DctnryStructList dctnryList; DctnryStructList dctnryList;
@ -919,14 +913,10 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
if (i == 0) if (i == 0)
{ {
rc = pDBRootExtentTracker->selectFirstSegFile(dbRootExtent, bFirstExtentOnThisPM, bEmptyPM, trkErrMsg); rc = pDBRootExtentTracker->selectFirstSegFile(dbRootExtent, bFirstExtentOnThisPM, bEmptyPM, trkErrMsg);
/* cout << "bEmptyPM = " << (int) bEmptyPM << " bFirstExtentOnThisPM= " << (int)bFirstExtentOnThisPM <<
" oid:dbroot:hwm = " << ridList[i].objnum << ":"<<dbRootExtent.fDbRoot << ":"
<<":"<<dbRootExtent.fLocalHwm << " err = " << trkErrMsg << endl; */
} }
else else
pDBRootExtentTracker->assignFirstSegFile(*(dbRootExtTrackerVec[0].get()), dbRootExtent); pDBRootExtentTracker->assignFirstSegFile(*(dbRootExtTrackerVec[0].get()), dbRootExtent);
colDBRootExtentInfo.push_back(dbRootExtent); colDBRootExtentInfo.push_back(dbRootExtent);
Column aColumn; Column aColumn;
@ -984,7 +974,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
std::vector<BRM::LBIDRange> rangeList; std::vector<BRM::LBIDRange> rangeList;
// use of MetaFile for bulk rollback support // use of MetaFile for bulk rollback support
if ( fIsFirstBatchPm && isAutocommitOn) if (fIsFirstBatchPm && isAutocommitOn)
{ {
//save meta data, version last block for each dbroot at the start of batch insert //save meta data, version last block for each dbroot at the start of batch insert
try try
@ -992,10 +982,8 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
fRBMetaWriter->init(tblOid, tableName.table); fRBMetaWriter->init(tblOid, tableName.table);
fRBMetaWriter->saveBulkRollbackMetaData(columns, dctnryStoreOids, dbRootHWMInfoColVec); fRBMetaWriter->saveBulkRollbackMetaData(columns, dctnryStoreOids, dbRootHWMInfoColVec);
//cout << "Saved meta files" << endl;
if (!bFirstExtentOnThisPM) if (!bFirstExtentOnThisPM)
{ {
//cout << "Backing up hwm chunks" << endl;
for (unsigned i = 0; i < dctnryList.size(); i++) //back up chunks for compressed dictionary for (unsigned i = 0; i < dctnryList.size(); i++) //back up chunks for compressed dictionary
{ {
// @bug 5572 HDFS tmp file - Ignoring return flag, don't need in this context // @bug 5572 HDFS tmp file - Ignoring return flag, don't need in this context
@ -1310,7 +1298,6 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
int error = NO_ERROR; int error = NO_ERROR;
//fWriteEngine.setDebugLevel(WriteEngine::DEBUG_3); //fWriteEngine.setDebugLevel(WriteEngine::DEBUG_3);
//cout << "Batch inserting a row with transaction id " << txnid.id << endl;
if (colValuesList.size() > 0) if (colValuesList.size() > 0)
{ {
if (colValuesList[0].size() > 0) if (colValuesList[0].size() > 0)
@ -1361,7 +1348,6 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
if ( isWarningSet && ( rc == NO_ERROR ) ) if ( isWarningSet && ( rc == NO_ERROR ) )
{ {
rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING; rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING;
//cout << "Got warning" << endl;
Message::Args args; Message::Args args;
string cols = "'" + colNames[0] + "'"; string cols = "'" + colNames[0] + "'";

3
writeengine/server/we_readthread.cpp
View File

@ -154,10 +154,7 @@ void DmlReadThread::operator()()
case WE_SVR_BATCH_INSERT: case WE_SVR_BATCH_INSERT:
{ {
//timer.start("processBatchInsert");
rc = fWeDMLprocessor->processBatchInsert(ibs, errMsg, PMId); rc = fWeDMLprocessor->processBatchInsert(ibs, errMsg, PMId);
//timer.stop("processBatchInsert");
//cout << "fWeDMLprocessor " << fWeDMLprocessor << " is processing batchinsert ..." << endl;
break; break;
} }

25
writeengine/wrapper/writeengine.cpp
View File

@ -1129,7 +1129,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
//---------------------------------------------------------------------- //----------------------------------------------------------------------
if (bFirstExtentOnThisPM) if (bFirstExtentOnThisPM)
{ {
//cout << "bFirstExtentOnThisPM is " << bFirstExtentOnThisPM << endl;
std::vector<BRM::CreateStripeColumnExtentsArgIn> cols; std::vector<BRM::CreateStripeColumnExtentsArgIn> cols;
BRM::CreateStripeColumnExtentsArgIn createStripeColumnExtentsArgIn; BRM::CreateStripeColumnExtentsArgIn createStripeColumnExtentsArgIn;
@ -1234,8 +1233,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
colStructList[i].fColPartition = tmpExtentInfo[currentDBrootIdx].fPartition; colStructList[i].fColPartition = tmpExtentInfo[currentDBrootIdx].fPartition;
colStructList[i].fColSegment = tmpExtentInfo[currentDBrootIdx].fSegment; colStructList[i].fColSegment = tmpExtentInfo[currentDBrootIdx].fSegment;
colStructList[i].fColDbRoot = tmpExtentInfo[currentDBrootIdx].fDbRoot; colStructList[i].fColDbRoot = tmpExtentInfo[currentDBrootIdx].fDbRoot;
//cout << "Load from dbrootExtenttracker oid:dbroot:part:seg = " <<colStructList[i].dataOid<<":"
//<<colStructList[i].fColDbRoot<<":"<<colStructList[i].fColPartition<<":"<<colStructList[i].fColSegment<<endl;
dctnryStructList[i].fColPartition = tmpExtentInfo[currentDBrootIdx].fPartition; dctnryStructList[i].fColPartition = tmpExtentInfo[currentDBrootIdx].fPartition;
dctnryStructList[i].fColSegment = tmpExtentInfo[currentDBrootIdx].fSegment; dctnryStructList[i].fColSegment = tmpExtentInfo[currentDBrootIdx].fSegment;
dctnryStructList[i].fColDbRoot = tmpExtentInfo[currentDBrootIdx].fDbRoot; dctnryStructList[i].fColDbRoot = tmpExtentInfo[currentDBrootIdx].fDbRoot;
@ -1271,7 +1268,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
{ {
aExt.hwm = extents[i].startBlkOffset; aExt.hwm = extents[i].startBlkOffset;
aExt.isNewExt = true; aExt.isNewExt = true;
//cout << "adding a ext to metadata" << endl;
} }
else else
{ {
@ -1279,12 +1275,10 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
tmpExtentInfo = dbRootExtentTrackers[i]->getDBRootExtentList(); tmpExtentInfo = dbRootExtentTrackers[i]->getDBRootExtentList();
aExt.isNewExt = false; aExt.isNewExt = false;
aExt.hwm = tmpExtentInfo[currentDBrootIdx].fLocalHwm; aExt.hwm = tmpExtentInfo[currentDBrootIdx].fLocalHwm;
//cout << "oid " << colStructList[i].dataOid << " gets hwm " << aExt.hwm << endl;
} }
aExt.current = true; aExt.current = true;
aColExtsInfo.push_back(aExt); aColExtsInfo.push_back(aExt);
//cout << "get from extentinfo oid:hwm = " << colStructList[i].dataOid << ":" << aExt.hwm << endl;
} }
tableMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo); tableMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo);
@ -1387,7 +1381,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
if (it != aColExtsInfo.end()) if (it != aColExtsInfo.end())
{ {
hwm = it->hwm; hwm = it->hwm;
//cout << "Got from colextinfo hwm for oid " << colStructList[colId].dataOid << " is " << hwm << " and seg is " << colStructList[0].fColSegment << endl;
} }
oldHwm = hwm; //Save this info for rollback oldHwm = hwm; //Save this info for rollback
@ -1422,8 +1415,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
curCol, (uint64_t)totalRow, rowIdArray, hwm, newExtent, rowsLeft, newHwm, newFile, curCol, (uint64_t)totalRow, rowIdArray, hwm, newExtent, rowsLeft, newHwm, newFile,
newColStructList, newDctnryStructList, dbRootExtentTrackers, insertSelect, true, tableOid, isFirstBatchPm); newColStructList, newDctnryStructList, dbRootExtentTrackers, insertSelect, true, tableOid, isFirstBatchPm);
//cout << "after allocrowid, total row = " <<totalRow << " newExtent is " << newExtent << endl;
//cout << "column oid " << curColStruct.dataOid << " has hwm:newHwm = " << hwm <<":" << newHwm<< endl;
if (rc != NO_ERROR) //Clean up is already done if (rc != NO_ERROR) //Clean up is already done
return rc; return rc;
@ -1441,7 +1432,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
((totalRow - rowsLeft) > 0) && ((totalRow - rowsLeft) > 0) &&
(rowIdArray[totalRow - rowsLeft - 1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK)) (rowIdArray[totalRow - rowsLeft - 1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK))
{ {
for (unsigned k=0; k<colStructList.size(); k++) for (unsigned k=0; k<colStructList.size(); k++)
{ {
// Skip the selected column // Skip the selected column
if (k == colId) if (k == colId)
@ -1505,7 +1496,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
return rc; return rc;
} }
for (uint32_t rows = 0; rows < (totalRow - rowsLeft); rows++) for (uint32_t rows = 0; rows < (totalRow - rowsLeft); rows++)
{ {
if (dctStr_iter->length() == 0) if (dctStr_iter->length() == 0)
{ {
@ -1560,7 +1551,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
if (rc != NO_ERROR) if (rc != NO_ERROR)
return rc; return rc;
for (uint32_t rows = 0; rows < rowsLeft; rows++) for (uint32_t rows = 0; rows < rowsLeft; rows++)
{ {
if (dctStr_iter->length() == 0) if (dctStr_iter->length() == 0)
{ {
@ -1627,13 +1618,11 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
lastRidNew = rowIdArray[totalRow - 1]; lastRidNew = rowIdArray[totalRow - 1];
} }
//cout << "rowid allocated is " << lastRid << endl;
//if a new extent is created, all the columns in this table should have their own new extent //if a new extent is created, all the columns in this table should have their own new extent
//First column already processed //First column already processed
//@Bug 1701. Close the file (if uncompressed) //@Bug 1701. Close the file (if uncompressed)
m_colOp[op(curCol.compressionType)]->clearColumn(curCol); m_colOp[op(curCol.compressionType)]->clearColumn(curCol);
//cout << "Saving hwm info for new ext batch" << endl;
//Update hwm to set them in the end //Update hwm to set them in the end
bool succFlag = false; bool succFlag = false;
unsigned colWidth = 0; unsigned colWidth = 0;
@ -1663,7 +1652,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
colWidth = colStructList[i].colWidth; colWidth = colStructList[i].colWidth;
succFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio); succFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio);
//cout << "insertcolumnrec oid:rid:fbo:oldhwm = " << colStructList[i].dataOid << ":" << lastRid << ":" << curFbo << ":" << oldHwm << endl;
if (succFlag) if (succFlag)
{ {
if ((HWM)curFbo >= oldHwm) if ((HWM)curFbo >= oldHwm)
@ -1677,8 +1665,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
it->current = false; it->current = false;
} }
//cout << "updated old ext info for oid " << colStructList[i].dataOid << " dbroot:part:seg:hwm:current = "
//<< it->dbRoot<<":"<<it->partNum<<":"<<it->segNum<<":"<<it->hwm<<":"<< it->current<< " and newExtent is " << newExtent << endl;
} }
else else
return ERR_INVALID_PARAM; return ERR_INVALID_PARAM;
@ -1734,7 +1720,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
colNewValueList.push_back(newColTupleList); colNewValueList.push_back(newColTupleList);
newColTupleList.clear(); newColTupleList.clear();
//upate the oldvalue list for the old extent //update the oldvalue list for the old extent
for (uint64_t j = 0; j < (totalRow - rowsLeft); j++) for (uint64_t j = 0; j < (totalRow - rowsLeft); j++)
{ {
firstPartTupleList.push_back(colTupleList[j]); firstPartTupleList.push_back(colTupleList[j]);
@ -1749,7 +1735,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
#ifdef PROFILE #ifdef PROFILE
timer.start("writeColumnRec"); timer.start("writeColumnRec");
#endif #endif
//cout << "Writing column record" << endl;
if (rc == NO_ERROR) if (rc == NO_ERROR)
{ {
@ -3979,7 +3964,7 @@ void WriteEngineWrapper::printInputValue(const ColStructList& colStructList,
// WIP replace with a single call // WIP replace with a single call
char buf[utils::MAXLENGTH16BYTES]; char buf[utils::MAXLENGTH16BYTES];
int128_t val = boost::any_cast<int128_t>(curTuple.data); int128_t val = boost::any_cast<int128_t>(curTuple.data);
dataconvert::DataConvert::toString(&val, 0, buf, utils::MAXLENGTH16BYTES); dataconvert::DataConvert::decimalToString(&val, 0, buf, utils::MAXLENGTH16BYTES, curColStruct.colDataType);
curStr.assign(buf); curStr.assign(buf);
} }
else if (curTuple.data.type() == typeid(double)) else if (curTuple.data.type() == typeid(double))