database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL-641 1. Minor refactoring of decimalToString for int128_t.

Browse Source 
2. Update unit tests for decimalToString.
3. Allow support for wide decimal in TupleConstantStep::fillInConstants().
This commit is contained in:
Gagan Goel
2020-03-03 11:14:21 -05:00
committed by Roman Nozdrin
parent 2e8e7d52c3
commit 9b714274db
14 changed files with 504 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
dbcon/joblist/jlf_execplantojoblist.cpp
View File

@ -1893,7 +1893,9 @@ const JobStepVector doSimpleFilter(SimpleFilter* sf, JobInfo& jobInfo)
if (ct.colDataType == CalpontSystemCatalog::DECIMAL &&
ct.colWidth == 16)
{
dataconvert::atoi128(constval, val128);
bool saturate = false;
val128 = dataconvert::string_to_ll<int128_t>(constval, saturate);
// TODO MCOL-641 check saturate
}
else
{

3
dbcon/joblist/pcolscan.cpp
View File

@ -176,7 +176,8 @@ pColScanStep::pColScanStep(
// MCOL-641 WIP
else if (fColType.colWidth > 8
&& fColType.colDataType != CalpontSystemCatalog::BINARY
&& fColType.colDataType != CalpontSystemCatalog::DECIMAL)
&& fColType.colDataType != CalpontSystemCatalog::DECIMAL
&& fColType.colDataType != CalpontSystemCatalog::UDECIMAL)
{
fColType.colWidth = 8;
fIsDict = true;

3
dbcon/joblist/pcolstep.cpp
View File

@ -180,7 +180,8 @@ pColStep::pColStep(
// WIP MCOL-641
else if (fColType.colWidth > 8
&& fColType.colDataType != CalpontSystemCatalog::BINARY
&& fColType.colDataType != CalpontSystemCatalog::DECIMAL)
&& fColType.colDataType != CalpontSystemCatalog::DECIMAL
&& fColType.colDataType != CalpontSystemCatalog::UDECIMAL)
{
fColType.colWidth = 8;
fIsDict = true;

6
dbcon/joblist/tupleconstantstep.cpp
View File

@ -528,7 +528,11 @@ void TupleConstantStep::fillInConstants(const rowgroup::Row& rowIn, rowgroup::Ro
//fRowConst.copyField(rowOut.getData()+2, 0); // hardcoded 2 for rid length
for (uint32_t i = 1; i < rowOut.getColumnCount(); i++)
rowIn.copyField(rowOut, i, i - 1);
// WIP MCOL-641 implement copyBinaryField inside copyField
if (UNLIKELY(rowIn.getColumnWidth(i - 1) == 16))
rowIn.copyBinaryField(rowOut, i, i - 1);
else
rowIn.copyField(rowOut, i, i - 1);
//memcpy(rowOut.getData()+rowOut.getOffset(1), rowIn.getData()+2, n);
}

289
tests/dataconvert-tests.cpp
View File

@ -545,126 +545,189 @@ TEST(DataConvertTest, NumberIntValue)
EXPECT_TRUE(pushWarning);
}
TEST(DataConvertTest, atoiCheck) {
std::vector<int128_t> expected;
std::vector<std::string> decimalAsStringVec;
decimalAsStringVec.push_back("99999999999999999999999999999999999999");
decimalAsStringVec.push_back("-99999999999999999999999999999999999999");
decimalAsStringVec.push_back("0");
decimalAsStringVec.push_back("-0.042");
expected.push_back(10000000000000000000ULL*(uint128_t)9999999999999999999ULL
+9999999999999999999ULL);
expected.push_back(-1*expected[0]);
expected.push_back(0);
expected.push_back(-42);
for (int i=0; i < expected.size(); i++) {
int128_t value;
dataconvert::atoi128(decimalAsStringVec[i], value);
EXPECT_EQ(expected[i], value);
EXPECT_NE(expected[i], value-1);
}
TEST(DataConvertTest, DecimalToStringCheckScale0)
{
CalpontSystemCatalog::ColType ct;
ct.colDataType = CalpontSystemCatalog::DECIMAL;
char buf[42];
string input, expected;
ct.precision = 38;
ct.scale = 0;
int128_t res;
// test simple values
res = 0;
expected = "0";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = 2;
expected = "2";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -2;
expected = "-2";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = 123;
expected = "123";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -123;
expected = "-123";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
// test max/min decimal (i.e. 38 9's)
res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99;
expected = "99999999999999999999999999999999999999";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -res;
expected = "-99999999999999999999999999999999999999";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
// test trailing zeros
res = 123000;
expected = "123000";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -res;
expected = "-123000";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
}
TEST(DataConvertTest, DecimalToStringCheckScale10)
{
CalpontSystemCatalog::ColType ct;
ct.colDataType = CalpontSystemCatalog::DECIMAL;
char buf[42];
string input, expected;
ct.precision = 38;
ct.scale = 10;
int128_t res;
// test simple values
res = 0;
expected = "0.0000000000";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = 2;
expected = "0.0000000002";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -2;
expected = "-0.0000000002";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = 123;
expected = "0.0000000123";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -123;
expected = "-0.0000000123";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = 12345678901;
expected = "1.2345678901";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -12345678901;
expected = "-1.2345678901";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
// test max/min decimal (i.e. 38 9's)
res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99;
expected = "9999999999999999999999999999.9999999999";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -res;
expected = "-9999999999999999999999999999.9999999999";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
// test trailing zeros
res = 123000;
expected = "0.0000123000";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -res;
expected = "-0.0000123000";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
// test leading zeros
res = 10000000009;
expected = "1.0000000009";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -res;
expected = "-1.0000000009";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
}
TEST(DataConvertTest, DecimalToStringCheckScale3) {
std::vector<std::string> expected;
std::vector<std::string> decimalAsStringVec;
// scale 3
uint8_t scale3 = 3;
decimalAsStringVec.push_back("99999999999999999999999999999999999999");
decimalAsStringVec.push_back("-99999999999999999999999999999999999999");
decimalAsStringVec.push_back("0");
decimalAsStringVec.push_back("-0.042");
expected.push_back("99999999999999999999999999999999999.999");
expected.push_back("-99999999999999999999999999999999999.999");
expected.push_back("0.000");
expected.push_back("-0.042");
TEST(DataConvertTest, DecimalToStringCheckScale38)
{
CalpontSystemCatalog::ColType ct;
ct.colDataType = CalpontSystemCatalog::DECIMAL;
char buf[42];
CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL;
for (int i=0; i < expected.size(); i++) {
int128_t value = -4;
memset(buf, 0, 42);
dataconvert::atoi128(decimalAsStringVec[i], value);
dataconvert::DataConvert::decimalToString(&value, scale3, buf,
utils::precisionByWidth(sizeof(value))+4, type);
EXPECT_EQ(expected[i], std::string(buf));
}
}
string input, expected;
ct.precision = 38;
ct.scale = 38;
int128_t res;
TEST(DataConvertTest, DecimalToStringCheckScale10) {
std::vector<std::string> expected;
std::vector<std::string> decimalAsStringVec;
// scale 3
uint8_t scale3 = 10;
decimalAsStringVec.push_back("10000000009");
decimalAsStringVec.push_back("-10000000009");
decimalAsStringVec.push_back("0");
decimalAsStringVec.push_back("-10000000010");
expected.push_back("1.0000000009");
expected.push_back("-1.0000000009");
expected.push_back("0.0000000000");
expected.push_back("-1.0000000010");
char buf[42];
CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL;
for (int i=0; i < expected.size(); i++) {
int128_t value = -4;
memset(buf, 0, 42);
dataconvert::atoi128(decimalAsStringVec[i], value);
dataconvert::DataConvert::decimalToString(&value, scale3, buf,
utils::precisionByWidth(sizeof(value))+4, type);
EXPECT_EQ(expected[i], std::string(buf));
}
}
// test simple values
res = 0;
expected = "0.00000000000000000000000000000000000000";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = 2;
expected = "0.00000000000000000000000000000000000002";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -2;
expected = "-0.00000000000000000000000000000000000002";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = 123;
expected = "0.00000000000000000000000000000000000123";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -123;
expected = "-0.00000000000000000000000000000000000123";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = ((((((int128_t)1234567890 * 10000000000) + 1234567890) * 10000000000) + 1234567890) * 100000000 ) + 12345678;
expected = "0.12345678901234567890123456789012345678";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -res;
expected = "-0.12345678901234567890123456789012345678";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
TEST(DataConvertTest, DecimalToStringCheckScale37) {
std::vector<std::string> expected;
std::vector<std::string> decimalAsStringVec;
uint8_t scale37 = 37;
decimalAsStringVec.push_back("99999999999999999999999999999999999999");
decimalAsStringVec.push_back("-99999999999999999999999999999999999999");
decimalAsStringVec.push_back("0");
decimalAsStringVec.push_back("-42");
decimalAsStringVec.push_back("-19999999999999999999999999999999999999");
expected.push_back("9.9999999999999999999999999999999999999");
expected.push_back("-9.9999999999999999999999999999999999999");
expected.push_back("0.0000000000000000000000000000000000000");
expected.push_back("-0.0000000000000000000000000000000000042");
expected.push_back("-1.9999999999999999999999999999999999999");
char buf[42];
CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL;
for (int i=0; i < expected.size(); i++) {
int128_t value = -4;
memset(buf, 0, 41);
dataconvert::atoi128(decimalAsStringVec[i], value);
dataconvert::DataConvert::decimalToString(&value, scale37, buf,
utils::precisionByWidth(sizeof(value))+4, type);
EXPECT_EQ(expected[i], std::string(buf));
}
}
// test max/min decimal (i.e. 38 9's)
res = ((((((((int128_t)999999999 * 1000000000) + 999999999) * 1000000000) + 999999999) * 1000000000 ) + 999999999) * 100) + 99;
expected = "0.99999999999999999999999999999999999999";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -res;
expected = "-0.99999999999999999999999999999999999999";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
TEST(DataConvertTest, DecimalToStringCheckScale38) {
std::vector<std::string> expected;
std::vector<std::string> decimalAsStringVec;
uint8_t scale38 = 38;
decimalAsStringVec.push_back("99999999999999999999999999999999999999");
decimalAsStringVec.push_back("-99999999999999999999999999999999999999");
decimalAsStringVec.push_back("0");
decimalAsStringVec.push_back("-42");
expected.push_back("0.99999999999999999999999999999999999999");
expected.push_back("-0.99999999999999999999999999999999999999");
expected.push_back("0.00000000000000000000000000000000000000");
expected.push_back("-0.00000000000000000000000000000000000042");
char buf[42];
CSCDataType type = execplan::CalpontSystemCatalog::DECIMAL;
for (int i=0; i < expected.size(); i++) {
int128_t value = -4;
memset(buf, 0, 41);
dataconvert::atoi128(decimalAsStringVec[i], value);
dataconvert::DataConvert::decimalToString(&value, scale38, buf,
utils::precisionByWidth(sizeof(value))+4, type);
EXPECT_EQ(expected[i], std::string(buf));
}
// test trailing zeros
res = 123000;
expected = "0.00000000000000000000000000000000123000";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
res = -res;
expected = "-0.00000000000000000000000000000000123000";
DataConvert::decimalToString(&res, ct.scale, buf, 42, ct.colDataType);
EXPECT_EQ(string(buf), expected);
}
TEST(DataConvertTest, ConvertColumnData) {
}

299
tests/rowgroup-tests.cpp
View File

@ -20,6 +20,7 @@
#include "rowgroup.h"
#include "columnwidth.h"
#include "widedecimalutils.h"
#include "joblisttypes.h"
#include "dataconvert.h"
@ -30,125 +31,138 @@ using int128_t = __int128;
using uint128_t = unsigned __int128;
using CSCDataType = execplan::CalpontSystemCatalog::ColDataType;
class RowDecimalTest : public ::testing::Test {
protected:
void SetUp() override {
uint32_t precision = WIDE_DEC_PRECISION;
uint32_t oid =3001;
class RowDecimalTest : public ::testing::Test
{
protected:
void SetUp() override
{
uint32_t precision = WIDE_DEC_PRECISION;
uint32_t oid = 3001;
std::vector<CSCDataType>types;
std::vector<decltype(precision)>precisionVec;
std::vector<uint32_t> roids, tkeys, cscale;
types.push_back(execplan::CalpontSystemCatalog::DECIMAL);
types.push_back(execplan::CalpontSystemCatalog::UDECIMAL);
for (size_t i=0; i <= 3; i++) {
std::vector<CSCDataType> types;
std::vector<decltype(precision)> precisionVec;
std::vector<uint32_t> roids, tkeys, cscale;
std::vector<uint32_t> widthVec;
types.push_back(execplan::CalpontSystemCatalog::DECIMAL);
}
precisionVec.push_back(precision);
precisionVec.push_back(precision);
precisionVec.push_back(18);
precisionVec.push_back(9);
precisionVec.push_back(4);
precisionVec.push_back(2);
std::vector<uint32_t>widthVec;
uint32_t offset = INITIAL_ROW_OFFSET;
offsets.push_back(offset);
for (size_t i=0; i < types.size(); i++) {
uint8_t width = utils::widthByPrecision(precisionVec[i]);
widthVec.push_back(width);
offset += width;
offsets.push_back(offset);
roids.push_back(oid+i);
tkeys.push_back(i+1); cscale.push_back(0);
}
/*offsets.push_back(INITIAL_ROW_OFFSET);
offsets.push_back(16+INITIAL_ROW_OFFSET);
offsets.push_back(16*2+INITIAL_ROW_OFFSET);
roids.push_back(oid); roids.push_back(oid+1);
tkeys.push_back(1); tkeys.push_back(1);
cscale.push_back(0); cscale.push_back(0);*/
types.push_back(execplan::CalpontSystemCatalog::UDECIMAL);
for (size_t i = 0; i <= 3; i++)
types.push_back(execplan::CalpontSystemCatalog::DECIMAL);
precisionVec.push_back(precision);
precisionVec.push_back(precision);
precisionVec.push_back(18);
precisionVec.push_back(9);
precisionVec.push_back(4);
precisionVec.push_back(2);
uint32_t offset = INITIAL_ROW_OFFSET;
offsets.push_back(offset);
for (size_t i = 0; i < types.size(); i++)
{
uint8_t width = utils::widthByPrecision(precisionVec[i]);
widthVec.push_back(width);
offset += width;
offsets.push_back(offset);
roids.push_back(oid + i);
tkeys.push_back(i + 1);
cscale.push_back(0);
}
/*offsets.push_back(INITIAL_ROW_OFFSET);
offsets.push_back(16+INITIAL_ROW_OFFSET);
offsets.push_back(16*2+INITIAL_ROW_OFFSET);
roids.push_back(oid); roids.push_back(oid+1);
tkeys.push_back(1); tkeys.push_back(1);
cscale.push_back(0); cscale.push_back(0);*/
rowgroup::RowGroup inRG(roids.size(), //column count
offsets, //oldOffset
roids, // column oids
tkeys, //keys
types, // types
cscale, //scale
precisionVec, // precision
20, // sTableThreshold
false //useStringTable
);
rowgroup::RowGroup inRG(roids.size(), // column count
offsets, // oldOffset
roids, // column oids
tkeys, // keys
types, // types
cscale, // scale
precisionVec, // precision
20, // sTableThreshold
false // useStringTable
);
rg = inRG;
rgD.reinit(rg);
rg.setData(&rgD);
rg = rgOut = inRG;
rgD.reinit(rg);
rgDOut.reinit(rgOut);
rg.setData(&rgD);
rgOut.setData(&rgDOut);
rg.initRow(&r);
rg.initRow(&rOutMappingCheck);
rowSize = r.getSize();
rg.getRow(0, &r);
rg.initRow(&r);
rg.initRow(&rOutMappingCheck);
rowSize = r.getSize();
rg.getRow(0, &r);
int128_t nullValue = 0;
int128_t bigValue = 0;
uint64_t* uint128_pod = reinterpret_cast<uint64_t*>(&nullValue);
uint128_pod[0] = joblist::UBIGINTNULL;
uint128_pod[1] = joblist::UBIGINTEMPTYROW;
bigValue = -static_cast<int128_t>(0xFFFFFFFF)*0xFFFFFFFFFFFFFFFF;
sValueVector.push_back(nullValue);
sValueVector.push_back(-42);
sValueVector.push_back(bigValue);
sValueVector.push_back(0);
sValueVector.push_back(nullValue-1);
int128_t nullValue = 0;
int128_t bigValue = 0;
uValueVector.push_back(nullValue);
uValueVector.push_back(42);
uValueVector.push_back(bigValue);
uValueVector.push_back(0);
uValueVector.push_back(nullValue-1);
utils::setWideDecimalNullValue(nullValue);
bigValue = -static_cast<int128_t>(0xFFFFFFFF)*0xFFFFFFFFFFFFFFFF;
s8ValueVector.push_back(joblist::TINYINTNULL);
s8ValueVector.push_back(-0x79);
s8ValueVector.push_back(0);
s8ValueVector.push_back(0x81);
s8ValueVector.push_back(joblist::TINYINTNULL-1);
sValueVector.push_back(nullValue);
sValueVector.push_back(-42);
sValueVector.push_back(bigValue);
sValueVector.push_back(0);
sValueVector.push_back(nullValue-1);
s16ValueVector.push_back(joblist::SMALLINTNULL);
s16ValueVector.push_back(-0x79);
s16ValueVector.push_back(0);
s16ValueVector.push_back(0x81);
s16ValueVector.push_back(joblist::SMALLINTNULL-1);
uValueVector.push_back(nullValue);
uValueVector.push_back(42);
uValueVector.push_back(bigValue);
uValueVector.push_back(0);
uValueVector.push_back(nullValue-1);
s32ValueVector.push_back(joblist::INTNULL);
s32ValueVector.push_back(-0x79);
s32ValueVector.push_back(0);
s32ValueVector.push_back(0x81);
s32ValueVector.push_back(joblist::INTNULL-1);
s8ValueVector.push_back(joblist::TINYINTNULL);
s8ValueVector.push_back(-0x79);
s8ValueVector.push_back(0);
s8ValueVector.push_back(0x81);
s8ValueVector.push_back(joblist::TINYINTNULL-1);
s64ValueVector.push_back(joblist::BIGINTNULL);
s64ValueVector.push_back(-0x79);
s64ValueVector.push_back(0);
s64ValueVector.push_back(0x81);
s64ValueVector.push_back(joblist::BIGINTNULL-1);
s16ValueVector.push_back(joblist::SMALLINTNULL);
s16ValueVector.push_back(-0x79);
s16ValueVector.push_back(0);
s16ValueVector.push_back(0x81);
s16ValueVector.push_back(joblist::SMALLINTNULL-1);
for(size_t i = 0; i < sValueVector.size(); i++) {
r.setBinaryField_offset(&sValueVector[i],
sizeof(sValueVector[0]), offsets[0]);
r.setBinaryField_offset(&uValueVector[i],
sizeof(uValueVector[0]), offsets[1]);
r.setIntField(s64ValueVector[i], 2);
r.setIntField(s32ValueVector[i], 3);
r.setIntField(s16ValueVector[i], 4);
r.setIntField(s8ValueVector[i], 5);
r.nextRow(rowSize);
s32ValueVector.push_back(joblist::INTNULL);
s32ValueVector.push_back(-0x79);
s32ValueVector.push_back(0);
s32ValueVector.push_back(0x81);
s32ValueVector.push_back(joblist::INTNULL-1);
s64ValueVector.push_back(joblist::BIGINTNULL);
s64ValueVector.push_back(-0x79);
s64ValueVector.push_back(0);
s64ValueVector.push_back(0x81);
s64ValueVector.push_back(joblist::BIGINTNULL-1);
for (size_t i = 0; i < sValueVector.size(); i++)
{
r.setBinaryField_offset(&sValueVector[i],
sizeof(sValueVector[0]), offsets[0]);
r.setBinaryField_offset(&uValueVector[i],
sizeof(uValueVector[0]), offsets[1]);
r.setIntField(s64ValueVector[i], 2);
r.setIntField(s32ValueVector[i], 3);
r.setIntField(s16ValueVector[i], 4);
r.setIntField(s8ValueVector[i], 5);
r.nextRow(rowSize);
}
rowCount = sValueVector.size();
}
rowCount = sValueVector.size();
}
// void TearDown() override {}
rowgroup::Row r;
// void TearDown() override {}
rowgroup::Row r, rOut;
rowgroup::Row rOutMappingCheck;
rowgroup::RowGroup rg;
rowgroup::RGData rgD;
rowgroup::RowGroup rg, rgOut;
rowgroup::RGData rgD, rgDOut;
uint32_t rowSize;
size_t rowCount;
std::vector<int128_t> sValueVector;
@ -160,9 +174,12 @@ class RowDecimalTest : public ::testing::Test {
std::vector<uint32_t> offsets;
};
TEST_F(RowDecimalTest, NonNULLValuesCheck) {
TEST_F(RowDecimalTest, NonNullValueCheck)
{
rg.getRow(1, &r);
for (size_t i = 1; i <= sValueVector.size(); i++) {
for (size_t i = 1; i <= sValueVector.size(); i++)
{
EXPECT_FALSE(r.isNullValue(0));
EXPECT_FALSE(r.isNullValue(1));
EXPECT_FALSE(r.isNullValue(2));
@ -173,9 +190,11 @@ TEST_F(RowDecimalTest, NonNULLValuesCheck) {
}
}
TEST_F(RowDecimalTest, initToNullANDisNullValueValueCheck) {
TEST_F(RowDecimalTest, InitToNullAndIsNullValueCheck)
{
rg.getRow(0, &r);
r.initToNull();
EXPECT_TRUE(r.isNullValue(0));
EXPECT_TRUE(r.isNullValue(1));
EXPECT_TRUE(r.isNullValue(2));
@ -184,14 +203,16 @@ TEST_F(RowDecimalTest, initToNullANDisNullValueValueCheck) {
EXPECT_TRUE(r.isNullValue(5));
}
TEST_F(RowDecimalTest, getBinaryFieldCheck) {
TEST_F(RowDecimalTest, GetBinaryFieldCheck)
{
rg.getRow(0, &r);
uint128_t* u128Value;
int128_t* s128Value;
// std::remove_reference<decltype(*u128Value)>::type uType;
// std::remove_reference<decltype(*s128Value)>::type sType;
for (size_t i = 0; i < sValueVector.size(); i++) {
for (size_t i = 0; i < sValueVector.size(); i++)
{
s128Value = r.getBinaryField<int128_t>(0);
EXPECT_EQ(sValueVector[i], *s128Value);
u128Value = r.getBinaryField<uint128_t>(1);
@ -204,26 +225,32 @@ TEST_F(RowDecimalTest, getBinaryFieldCheck) {
}
}
TEST_F(RowDecimalTest, toStringCheck) {
TEST_F(RowDecimalTest, ToStringCheck)
{
std::vector<std::string> exemplarVector;
exemplarVector.push_back(std::string("0: NULL NULL NULL NULL NULL NULL "));
exemplarVector.push_back(std::string("0: -42 42 -121 -121 -121 -121 "));
exemplarVector.push_back(std::string("0: -79228162495817593515539431425 -79228162495817593515539431425 0 0 0 0 "));
exemplarVector.push_back(std::string("0: 0 0 129 129 129 -127 "));
exemplarVector.push_back(std::string("0: -3 -3 9223372036854775807 2147483647 32767 127 "));
exemplarVector.push_back(std::string("0: 170141183460469231731687303715884105727 170141183460469231731687303715884105727 9223372036854775807 2147483647 32767 127 "));
rg.getRow(0, &r);
r.initToNull();
for (auto &el: exemplarVector) {
for (auto& el: exemplarVector)
{
EXPECT_EQ(el, r.toString());
r.nextRow(rowSize);
}
}
TEST_F(RowDecimalTest, toCSVCheck) {
TEST_F(RowDecimalTest, ToCSVCheck)
{
}
TEST_F(RowDecimalTest, applyMappingCheck) {
TEST_F(RowDecimalTest, ApplyMappingCheck)
{
int mapping[] = {0, 1, -1, -1, -1, -1};
rg.getRow(1, &r);
rg.getRow(2, &rOutMappingCheck);
@ -233,11 +260,49 @@ TEST_F(RowDecimalTest, applyMappingCheck) {
EXPECT_NE(uValueVector[1], *u128Value);
applyMapping(mapping, r, &rOutMappingCheck);
s128Value = rOutMappingCheck.getBinaryField<int128_t>(0);
EXPECT_EQ(sValueVector[1], *s128Value);
u128Value = rOutMappingCheck.getBinaryField<uint128_t>(1);
EXPECT_EQ(sValueVector[1], *s128Value);
EXPECT_EQ(uValueVector[1], *u128Value);
}
// WIP
TEST_F(RowDecimalTest, RowEqualsCheck) {
TEST_F(RowDecimalTest, CopyBinaryFieldCheck)
{
int128_t constVal = 1;
int128_t *col1Out, *col1In;
uint128_t *col2Out, *col2In;
rgOut.getRow(0, &rOut);
for (size_t i = 0; i < sValueVector.size(); i++)
{
rOut.setBinaryField_offset(&constVal, 16, offsets[0]);
rOut.setBinaryField_offset((uint128_t*)&constVal, 16, offsets[1]);
rOut.nextRow(rowSize);
}
rgOut.initRow(&rOut);
rgOut.getRow(0, &rOut);
rg.getRow(0, &r);
for (size_t i = 0; i < sValueVector.size(); i++)
{
col1In = r.getBinaryField<int128_t>(0);
col1Out = rOut.getBinaryField<int128_t>(0);
col2In = r.getBinaryField<uint128_t>(1);
col2Out = rOut.getBinaryField<uint128_t>(1);
EXPECT_NE(*col1In, *col1Out);
EXPECT_NE(*col2In, *col2Out);
r.copyBinaryField(rOut, 0, 0);
r.copyBinaryField(rOut, 1, 1);
col1Out = rOut.getBinaryField<int128_t>(0);
col2Out = rOut.getBinaryField<uint128_t>(1);
EXPECT_EQ(*col1In, *col1Out);
EXPECT_EQ(*col2In, *col2Out);
r.nextRow(rowSize);
rOut.nextRow(rowSize);
}
}
// WIP
TEST_F(RowDecimalTest, RowEqualsCheck)
{
}

216
utils/dataconvert/dataconvert.cpp
View File

@ -125,6 +125,7 @@ const uint64_t columnstore_pow_10[20] =
1000000000000000000ULL,
10000000000000000000ULL
};
template <class T>
bool from_string(T& t, const std::string& s, std::ios_base & (*f)(std::ios_base&))
{
@ -1249,115 +1250,144 @@ bool stringToTimestampStruct(const string& data, TimeStamp& timeStamp, const str
}
size_t DataConvert::writeIntPart(int128_t* dec, char* p,
const uint16_t buflen,
size_t DataConvert::writeIntPart(int128_t* dec,
char* p,
const unsigned int buflen,
const uint8_t scale)
{
int128_t intPart = *dec;
if (scale)
{
uint8_t maxPowOf10 = sizeof(columnstore_pow_10)/sizeof(columnstore_pow_10[0])-1;
switch (scale/maxPowOf10)
int128_t intPart = *dec;
int128_t high = 0, mid = 0, low = 0;
uint64_t div = 10000000000000000000ULL;
if (scale)
{
case(2):
intPart /= columnstore_pow_10[maxPowOf10];
intPart /= columnstore_pow_10[maxPowOf10];
break;
case(1):
intPart /= columnstore_pow_10[maxPowOf10];
case(0):
intPart /= columnstore_pow_10[scale%maxPowOf10];
const uint8_t maxPowOf10 =
(sizeof(columnstore_pow_10) / sizeof(columnstore_pow_10[0])) - 1;
// Assuming scale = [0, 56]
switch (scale / maxPowOf10)
{
case 2: // scale = [38, 56]
intPart /= columnstore_pow_10[maxPowOf10];
intPart /= columnstore_pow_10[maxPowOf10];
low = intPart;
break;
case 1: // scale = [19, 37]
intPart /= columnstore_pow_10[maxPowOf10];
intPart /= columnstore_pow_10[scale % maxPowOf10];
low = intPart % div;
mid = intPart / div;
break;
case 0: // scale = [0, 18]
intPart /= columnstore_pow_10[scale % maxPowOf10];
low = intPart % div;
intPart /= div;
mid = intPart % div;
high = intPart / div;
break;
default:
throw QueryDataExcept("writeIntPart() bad scale", formatErr);
}
}
else
{
low = intPart % div;
intPart /= div;
mid = intPart % div;
high = intPart / div;
}
}
uint64_t div = 10000000000000000000ULL;
int128_t high = intPart;
int128_t low;
low = high % div;
high /= div;
int128_t mid;
mid = high % div;
high /= div;
// pod[0] is low 8 bytes, pod[1] is high 8 bytes
uint64_t* high_pod = reinterpret_cast<uint64_t*>(&high);
uint64_t* mid_pod = reinterpret_cast<uint64_t*>(&mid);
uint64_t* low_pod = reinterpret_cast<uint64_t*>(&low);
char* original_p = p;
// pod[0] is low 8 byte, pod[1] is high
uint64_t* high_pod = reinterpret_cast<uint64_t*>(&high);
uint64_t* mid_pod = reinterpret_cast<uint64_t*>(&mid);
uint64_t* low_pod = reinterpret_cast<uint64_t*>(&low);
char* original_p = p;
int written = 0;
// WIP replace sprintf with streams
if (high_pod[0] != 0)
{
p += sprintf(p, "%lu", high_pod[0]);
p += sprintf(p, "%019lu", mid_pod[0]);
p += sprintf(p, "%019lu", low_pod[0]);
}
else if (mid_pod[0] != 0)
{
p += sprintf(p, "%lu", mid_pod[0]);
p += sprintf(p, "%019lu", low_pod[0]);
}
else
{
p += sprintf(p, "%lu", low_pod[0]);
}
// WIP replace snprintf with streams
if (high_pod[0] != 0)
{
written = sprintf(p, "%lu", high_pod[0]);
p += written;
written = sprintf(p, "%019lu", mid_pod[0]);
p += written;
sprintf(p, "%019lu", low_pod[0]);
}
else if (mid_pod[0] != 0)
{
written = sprintf(p, "%lu", mid_pod[0]);
p += written;
written = sprintf(p, "%019lu", low_pod[0]);
p += written;
}
else
{
written = sprintf(p, "%lu", low_pod[0]);
p += written;
}
size_t written = p - original_p;
if (buflen <= p-original_p)
{
throw QueryDataExcept("writeIntPart() char buffer overflow.", formatErr);
}
return p-original_p;
if (buflen <= written)
{
throw QueryDataExcept("writeIntPart() char buffer overflow.", formatErr);
}
return written;
}
size_t DataConvert::writeFractionalPart(int128_t* dec, char* p,
const uint16_t buflen,
size_t DataConvert::writeFractionalPart(int128_t* dec,
char* p,
const unsigned int buflen,
const uint8_t scale)
{
int128_t scaleDivisor = 1;
uint8_t maxPowOf10 = sizeof(columnstore_pow_10)/sizeof(columnstore_pow_10[0])-1;
switch (scale/maxPowOf10)
const uint8_t maxPowOf10 =
(sizeof(columnstore_pow_10) / sizeof(columnstore_pow_10[0])) - 1;
switch (scale / maxPowOf10)
{
case(2):
case 2:
scaleDivisor *= columnstore_pow_10[maxPowOf10];
scaleDivisor *= columnstore_pow_10[maxPowOf10];
break;
case(1):
case 1:
scaleDivisor *= columnstore_pow_10[maxPowOf10];
case(0):
scaleDivisor *= columnstore_pow_10[scale%maxPowOf10];
case 0:
scaleDivisor *= columnstore_pow_10[scale % maxPowOf10];
}
int128_t fractionalPart = *dec % scaleDivisor;
// divide by the base untill we have non-zero quotinent
// divide by the base until we have non-zero quotient
size_t written = 0;
scaleDivisor /= 10;
while (scaleDivisor > 1 && fractionalPart/scaleDivisor == 0)
char* original_p = p;
while (scaleDivisor > 1 && fractionalPart / scaleDivisor == 0)
{
*p++ = '0';
written++;
scaleDivisor /= 10;
}
written += writeIntPart(&fractionalPart, p, buflen, 0);
p += writeIntPart(&fractionalPart, p, buflen - written, 0);
written = p - original_p;
// this should never be true
if (written < scale)
{
p += written;
for (size_t left = written; left < scale; left++)
{
*p++ = '0';
}
}
return scale;
return written;
}
void DataConvert::toString(int128_t* dec, uint8_t scale,
char *p, unsigned int buflen)
void DataConvert::decimalToString(int128_t* dec,
const uint8_t scale,
char *p,
const unsigned int buflen,
cscDataType colDataType) // colDataType is redundant
{
char* original_p = p;
size_t written = 0;
@ -1379,54 +1409,20 @@ void DataConvert::toString(int128_t* dec, uint8_t scale,
if (scale)
{
*p++ = '.';
p += writeFractionalPart(dec, p, buflen-(p-original_p), scale);
written = p - original_p;
p += writeFractionalPart(dec, p, buflen - written, scale);
}
*p = '\0';
if (buflen <= p-original_p)
written = p - original_p;
if (buflen <= written)
{
throw QueryDataExcept("toString() char buffer overflow.", formatErr);
}
}
// WIP MCOL-641
void atoi128(const std::string& arg, int128_t& res)
{
res = 0;
size_t idx = (arg[0] == '-') ? 1 : 0;
for (size_t j = idx; j < arg.size(); j++)
{
// WIP Optimize this
if (LIKELY(arg[j]-'0' >= 0))
res = res*10 + arg[j] - '0';
}
if (idx)
res *= -1;
}
// WIP MCOL-641
// remove this as we don't need this for wide-DECIMAL
void atoi128(const std::string& arg, uint128_t& res)
{
res = 0;
for (size_t j = 0; j < arg.size(); j++)
{
// WIP Optimize this
if (LIKELY(arg[j]-'0' >= 0))
res = res*10 + arg[j] - '0';
}
}
void DataConvert::decimalToString(int128_t* valuePtr,
uint8_t scale,
char* buf,
unsigned int buflen,
cscDataType colDataType) // We don't need the last one
{
toString(valuePtr, scale, buf, buflen);
}
boost::any
DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType,
const std::string& dataOrig, bool& pushWarning,

12
utils/dataconvert/dataconvert.h
View File

@ -152,8 +152,6 @@ struct Int128Pod_struct
typedef Int128Pod_struct Int128Pod_t;
void atoi128(const std::string& arg, int128_t& res);
enum CalpontDateTimeFormat
{
CALPONTDATE_ENUM = 1, // date format is: "YYYY-MM-DD"
@ -177,9 +175,6 @@ struct MySQLTime
}
};
void atoi128(const std::string& arg, int128_t& res);
void atoi128(const std::string& arg, uint128_t& res);
/**
* This function converts the timezone represented as a string
* in the format "+HH:MM" or "-HH:MM" to a signed offset in seconds
@ -1041,12 +1036,11 @@ public:
EXPORT static bool isNullData(execplan::ColumnResult* cr, int rownum, execplan::CalpontSystemCatalog::ColType colType);
static inline std::string decimalToString(int64_t value, uint8_t scale, cscDataType colDataType);
static inline void decimalToString(int64_t value, uint8_t scale, char* buf, unsigned int buflen, cscDataType colDataType);
static void decimalToString(int128_t* value, uint8_t scale, char* buf, unsigned int buflen, cscDataType colDataType);
static void toString(int128_t* dec, uint8_t scale, char* p, unsigned int buflen);
static size_t writeIntPart(int128_t* dec, char* p, const uint16_t buflen,
static void decimalToString(int128_t* dec, const uint8_t scale, char* buf, const unsigned int buflen, cscDataType colDataType);
static size_t writeIntPart(int128_t* dec, char* p, const unsigned int buflen,
const uint8_t scale);
static size_t writeFractionalPart(int128_t* dec, char* p, const uint16_t buflen,
static size_t writeFractionalPart(int128_t* dec, char* p, const unsigned int buflen,
const uint8_t scale);
static inline void int128Max(int128_t& i)

13
utils/rowgroup/rowgroup.cpp
View File

@ -633,22 +633,23 @@ string Row::toString() const
os << " " << dec;
break;
}
// WIP MCOL-641
case CalpontSystemCatalog::BINARY:
std::cout << __FILE__<< ":" <<__LINE__ << " Fix for 16 Bytes ?" << std::endl;
break;
// WIP
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
if (colWidths[i] == sizeof(int128_t))
if (utils::isWide(colWidths[i]))
{
char *buf = (char*)alloca(precision[i] + 3);
unsigned int buflen = precision[i] + 3;
char *buf = (char*)alloca(buflen);
// empty the buffer
dataconvert::DataConvert::decimalToString(getBinaryField<int128_t>(i),
scale[i], buf, precision[i]+3, types[i]);
scale[i], buf, buflen, types[i]);
os << buf << " ";
break;
}
// fallback if the the legacy DECIMAL
// fallthrough if the legacy DECIMAL
default:
os << getIntField(i) << " ";
break;
@ -850,10 +851,8 @@ void Row::initToNull()
break;
case 16 :
{
utils::setWideDecimalNullValue(reinterpret_cast<int128_t&>(data[offsets[i]]));
break;
}
default:
*((int64_t*) &data[offsets[i]]) = static_cast<int64_t>(joblist::BIGINTNULL);
break;

9
utils/rowgroup/rowgroup.h
View File

@ -475,6 +475,9 @@ public:
// that's not string-table safe, this one is
inline void copyField(Row& dest, uint32_t destIndex, uint32_t srcIndex) const;
// WIP MCOL-641
inline void copyBinaryField(Row& dest, uint32_t destIndex, uint32_t srcIndex) const;
std::string toString() const;
std::string toCSV() const;
@ -1229,6 +1232,12 @@ inline void Row::copyField(Row& out, uint32_t destIndex, uint32_t srcIndex) cons
out.setIntField(getIntField(srcIndex), destIndex);
}
// WIP MCOL-641
inline void Row::copyBinaryField(Row& out, uint32_t destIndex, uint32_t srcIndex) const
{
out.setBinaryField<int128_t>(getBinaryField<int128_t>(srcIndex), 16, destIndex);
}
inline void Row::setRid(uint64_t rid)
{
*((uint16_t*) data) = rid & 0xffff;

4
writeengine/bulk/we_bulkloadbuffer.cpp
View File

@ -994,7 +994,9 @@ void BulkLoadBuffer::convert(char* field, int fieldLength,
}
else
{
dataconvert::atoi128(string(field), bigllVal);
bool saturate = false;
bigllVal = dataconvert::string_to_ll<int128_t>(string(field), saturate);
// TODO MCOL-641 check saturate
}
}
else

18
writeengine/server/we_dmlcommandproc.cpp
View File

@ -815,24 +815,19 @@ uint8_t WE_DMLCommandProc::rollbackVersion(ByteStream& bs, std::string& err)
uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::string& err, ByteStream::quadbyte& PMId)
{
int rc = 0;
//cout << "processBatchInsert received bytestream length " << bs.length() << endl;
InsertDMLPackage insertPkg;
ByteStream::quadbyte tmp32;
bs >> tmp32;
//cout << "processBatchInsert got transaction id " << tmp32 << endl;
bs >> PMId;
//cout << "processBatchInsert gor PMId " << PMId << endl;
insertPkg.read( bs);
uint32_t sessionId = insertPkg.get_SessionID();
//cout << " processBatchInsert for session " << sessionId << endl;
DMLTable* tablePtr = insertPkg.get_Table();
bool isAutocommitOn = insertPkg.get_isAutocommitOn();
if (idbdatafile::IDBPolicy::useHdfs())
isAutocommitOn = true;
//cout << "This session isAutocommitOn is " << isAutocommitOn << endl;
BRM::TxnID txnid;
txnid.id = tmp32;
txnid.valid = true;
@ -858,7 +853,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
try
{
ridList = systemCatalogPtr->columnRIDs(tableName, true);
roPair = systemCatalogPtr->tableRID( tableName);
roPair = systemCatalogPtr->tableRID(tableName);
}
catch (std::exception& ex)
{
@ -867,7 +862,6 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
return rc;
}
std::vector<OID> dctnryStoreOids(ridList.size()) ;
std::vector<Column> columns;
DctnryStructList dctnryList;
@ -919,14 +913,10 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
if (i == 0)
{
rc = pDBRootExtentTracker->selectFirstSegFile(dbRootExtent, bFirstExtentOnThisPM, bEmptyPM, trkErrMsg);
/* cout << "bEmptyPM = " << (int) bEmptyPM << " bFirstExtentOnThisPM= " << (int)bFirstExtentOnThisPM <<
" oid:dbroot:hwm = " << ridList[i].objnum << ":"<<dbRootExtent.fDbRoot << ":"
<<":"<<dbRootExtent.fLocalHwm << " err = " << trkErrMsg << endl; */
}
else
pDBRootExtentTracker->assignFirstSegFile(*(dbRootExtTrackerVec[0].get()), dbRootExtent);
colDBRootExtentInfo.push_back(dbRootExtent);
Column aColumn;
@ -984,7 +974,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
std::vector<BRM::LBIDRange> rangeList;
// use of MetaFile for bulk rollback support
if ( fIsFirstBatchPm && isAutocommitOn)
if (fIsFirstBatchPm && isAutocommitOn)
{
//save meta data, version last block for each dbroot at the start of batch insert
try
@ -992,10 +982,8 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
fRBMetaWriter->init(tblOid, tableName.table);
fRBMetaWriter->saveBulkRollbackMetaData(columns, dctnryStoreOids, dbRootHWMInfoColVec);
//cout << "Saved meta files" << endl;
if (!bFirstExtentOnThisPM)
{
//cout << "Backing up hwm chunks" << endl;
for (unsigned i = 0; i < dctnryList.size(); i++) //back up chunks for compressed dictionary
{
// @bug 5572 HDFS tmp file - Ignoring return flag, don't need in this context
@ -1310,7 +1298,6 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
int error = NO_ERROR;
//fWriteEngine.setDebugLevel(WriteEngine::DEBUG_3);
//cout << "Batch inserting a row with transaction id " << txnid.id << endl;
if (colValuesList.size() > 0)
{
if (colValuesList[0].size() > 0)
@ -1361,7 +1348,6 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
if ( isWarningSet && ( rc == NO_ERROR ) )
{
rc = dmlpackageprocessor::DMLPackageProcessor::IDBRANGE_WARNING;
//cout << "Got warning" << endl;
Message::Args args;
string cols = "'" + colNames[0] + "'";

3
writeengine/server/we_readthread.cpp
View File

@ -154,10 +154,7 @@ void DmlReadThread::operator()()
case WE_SVR_BATCH_INSERT:
{
//timer.start("processBatchInsert");
rc = fWeDMLprocessor->processBatchInsert(ibs, errMsg, PMId);
//timer.stop("processBatchInsert");
//cout << "fWeDMLprocessor " << fWeDMLprocessor << " is processing batchinsert ..." << endl;
break;
}

25
writeengine/wrapper/writeengine.cpp
View File

@ -1129,7 +1129,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
//----------------------------------------------------------------------
if (bFirstExtentOnThisPM)
{
//cout << "bFirstExtentOnThisPM is " << bFirstExtentOnThisPM << endl;
std::vector<BRM::CreateStripeColumnExtentsArgIn> cols;
BRM::CreateStripeColumnExtentsArgIn createStripeColumnExtentsArgIn;
@ -1234,8 +1233,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
colStructList[i].fColPartition = tmpExtentInfo[currentDBrootIdx].fPartition;
colStructList[i].fColSegment = tmpExtentInfo[currentDBrootIdx].fSegment;
colStructList[i].fColDbRoot = tmpExtentInfo[currentDBrootIdx].fDbRoot;
//cout << "Load from dbrootExtenttracker oid:dbroot:part:seg = " <<colStructList[i].dataOid<<":"
//<<colStructList[i].fColDbRoot<<":"<<colStructList[i].fColPartition<<":"<<colStructList[i].fColSegment<<endl;
dctnryStructList[i].fColPartition = tmpExtentInfo[currentDBrootIdx].fPartition;
dctnryStructList[i].fColSegment = tmpExtentInfo[currentDBrootIdx].fSegment;
dctnryStructList[i].fColDbRoot = tmpExtentInfo[currentDBrootIdx].fDbRoot;
@ -1271,7 +1268,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
{
aExt.hwm = extents[i].startBlkOffset;
aExt.isNewExt = true;
//cout << "adding a ext to metadata" << endl;
}
else
{
@ -1279,12 +1275,10 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
tmpExtentInfo = dbRootExtentTrackers[i]->getDBRootExtentList();
aExt.isNewExt = false;
aExt.hwm = tmpExtentInfo[currentDBrootIdx].fLocalHwm;
//cout << "oid " << colStructList[i].dataOid << " gets hwm " << aExt.hwm << endl;
}
aExt.current = true;
aColExtsInfo.push_back(aExt);
//cout << "get from extentinfo oid:hwm = " << colStructList[i].dataOid << ":" << aExt.hwm << endl;
}
tableMetaData->setColExtsInfo(colStructList[i].dataOid, aColExtsInfo);
@ -1387,7 +1381,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
if (it != aColExtsInfo.end())
{
hwm = it->hwm;
//cout << "Got from colextinfo hwm for oid " << colStructList[colId].dataOid << " is " << hwm << " and seg is " << colStructList[0].fColSegment << endl;
}
oldHwm = hwm; //Save this info for rollback
@ -1422,8 +1415,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
curCol, (uint64_t)totalRow, rowIdArray, hwm, newExtent, rowsLeft, newHwm, newFile,
newColStructList, newDctnryStructList, dbRootExtentTrackers, insertSelect, true, tableOid, isFirstBatchPm);
//cout << "after allocrowid, total row = " <<totalRow << " newExtent is " << newExtent << endl;
//cout << "column oid " << curColStruct.dataOid << " has hwm:newHwm = " << hwm <<":" << newHwm<< endl;
if (rc != NO_ERROR) //Clean up is already done
return rc;
@ -1441,7 +1432,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
((totalRow - rowsLeft) > 0) &&
(rowIdArray[totalRow - rowsLeft - 1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK))
{
for (unsigned k=0; k<colStructList.size(); k++)
for (unsigned k=0; k<colStructList.size(); k++)
{
// Skip the selected column
if (k == colId)
@ -1505,7 +1496,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
return rc;
}
for (uint32_t rows = 0; rows < (totalRow - rowsLeft); rows++)
for (uint32_t rows = 0; rows < (totalRow - rowsLeft); rows++)
{
if (dctStr_iter->length() == 0)
{
@ -1560,7 +1551,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
if (rc != NO_ERROR)
return rc;
for (uint32_t rows = 0; rows < rowsLeft; rows++)
for (uint32_t rows = 0; rows < rowsLeft; rows++)
{
if (dctStr_iter->length() == 0)
{
@ -1627,13 +1618,11 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
lastRidNew = rowIdArray[totalRow - 1];
}
//cout << "rowid allocated is " << lastRid << endl;
//if a new extent is created, all the columns in this table should have their own new extent
//First column already processed
//@Bug 1701. Close the file (if uncompressed)
m_colOp[op(curCol.compressionType)]->clearColumn(curCol);
//cout << "Saving hwm info for new ext batch" << endl;
//Update hwm to set them in the end
bool succFlag = false;
unsigned colWidth = 0;
@ -1663,7 +1652,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
colWidth = colStructList[i].colWidth;
succFlag = colOp->calculateRowId(lastRid, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio);
//cout << "insertcolumnrec oid:rid:fbo:oldhwm = " << colStructList[i].dataOid << ":" << lastRid << ":" << curFbo << ":" << oldHwm << endl;
if (succFlag)
{
if ((HWM)curFbo >= oldHwm)
@ -1677,8 +1665,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
it->current = false;
}
//cout << "updated old ext info for oid " << colStructList[i].dataOid << " dbroot:part:seg:hwm:current = "
//<< it->dbRoot<<":"<<it->partNum<<":"<<it->segNum<<":"<<it->hwm<<":"<< it->current<< " and newExtent is " << newExtent << endl;
}
else
return ERR_INVALID_PARAM;
@ -1734,7 +1720,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
colNewValueList.push_back(newColTupleList);
newColTupleList.clear();
//upate the oldvalue list for the old extent
//update the oldvalue list for the old extent
for (uint64_t j = 0; j < (totalRow - rowsLeft); j++)
{
firstPartTupleList.push_back(colTupleList[j]);
@ -1749,7 +1735,6 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
#ifdef PROFILE
timer.start("writeColumnRec");
#endif
//cout << "Writing column record" << endl;
if (rc == NO_ERROR)
{
@ -3979,7 +3964,7 @@ void WriteEngineWrapper::printInputValue(const ColStructList& colStructList,
// WIP replace with a single call
char buf[utils::MAXLENGTH16BYTES];
int128_t val = boost::any_cast<int128_t>(curTuple.data);
dataconvert::DataConvert::toString(&val, 0, buf, utils::MAXLENGTH16BYTES);
dataconvert::DataConvert::decimalToString(&val, 0, buf, utils::MAXLENGTH16BYTES, curColStruct.colDataType);
curStr.assign(buf);
}
else if (curTuple.data.type() == typeid(double))