database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-07 03:22:57 +03:00
Code Activity

MCOL-901 Significanlty reduced memory consumption for group_concat().

Browse Source 
RowGroup default constructor allocates memory too generously.

    Removed commented code from groupconcat.cpp.
This commit is contained in:
Roman Nozdrin
2019-01-26 16:30:02 +03:00
parent 13398fa530
commit 6d65b13852
2 changed files with 20 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
dbcon/joblist/groupconcat.cpp
View File

@@ -344,8 +344,6 @@ void GroupConcatAgUM::merge(const rowgroup::Row& inRow, int64_t i)
joblist::GroupConcatAgUM* gccAg = *((joblist::GroupConcatAgUM**)(data + inRow.getOffset(i)));
fConcator->merge(gccAg->concator().get());
// don't reset
// gccAg->orderBy().reset();
}
@@ -393,12 +391,11 @@ GroupConcator::~GroupConcator()
void GroupConcator::initialize(const rowgroup::SP_GroupConcat& gcc)
{
// MCOL-901 This value comes from the Server and it is
// too high(3MB) to allocate it for every instance.
fGroupConcatLen = gcc->fSize;
fCurrentLength -= strlen(gcc->fSeparator.c_str());
fOutputString.reset(new uint8_t[fGroupConcatLen + 2]);
memset(fOutputString.get(), 0, fGroupConcatLen + 2);
fConstCols = gcc->fConstCols;
fConstantLen = strlen(gcc->fSeparator.c_str());
@@ -477,7 +474,6 @@ void GroupConcator::outputRow(std::ostringstream& oss, const rowgroup::Row& row)
case CalpontSystemCatalog::TEXT:
{
oss << row.getStringField(*i).c_str();
//oss << row.getStringField(*i);
break;
}
@@ -620,11 +616,6 @@ int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row)
{
int64_t colWidth = row.getStringLength(*i);
fieldLen += colWidth; // getStringLength() does the same thing as below
//assert(!row.usesStringTable());
//int64_t colWidth = row.getColumnWidth(*i);
//uint8_t* pStr = row.getData() + row.getOffset(*i);
//while ((*pStr++ > 0) && (fieldLen < colWidth))
// fieldLen++;
break;
}
@@ -752,7 +743,6 @@ void GroupConcatOrderBy::initialize(const rowgroup::SP_GroupConcat& gcc)
uint64_t GroupConcatOrderBy::getKeyLength() const
{
// only distinct the concatenated columns
//return (fRow0.getOffset(fConcatColumns.size()) - 2);
return fConcatColumns.size() - 1; // cols 0 to fConcatColumns will be conpared
}
@@ -771,8 +761,6 @@ void GroupConcatOrderBy::processRow(const rowgroup::Row& row)
if (fCurrentLength < fGroupConcatLen)
{
copyRow(row, &fRow0);
//cout << "length < GB limit: " << fRow0.toString() << endl;
//memcpy(fRow0.getData(), row.getData(), row.getSize());
// the RID is no meaning here, use it to store the estimated length.
int16_t estLen = lengthEstimate(fRow0);
fRow0.setRid(estLen);
@@ -862,7 +850,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc)
if (fDistinct)
fDistinctMap->insert(row.fData);
//fDistinctMap->insert(make_pair((row.fData+2), row.fData));
}
else if (fOrderByCond.size() > 0 && fRule.less(row.fData, fOrderByQueue.top().fData))
@@ -876,8 +863,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc)
{
fDistinctMap->erase(swapRow.fData);
fDistinctMap->insert(row.fData);
//fDistinctMap->erase(fDistinctMap->find(swapRow.fData + 2));
//fDistinctMap->insert(make_pair((row.fData+2), row.fData));
}
row1.setData(row.fData);
@@ -893,7 +878,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc)
void GroupConcatOrderBy::getResult(uint8_t* buff, const string& sep)
{
#if 1
ostringstream oss;
bool addSep = false;
@@ -919,28 +903,12 @@ void GroupConcatOrderBy::getResult(uint8_t* buff, const string& sep)
rowStack.pop();
}
strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen);
#else
ostringstream oss;
bool addSep = false;
priority_queue<OrderByRow>::reverse_iterator rit;
size_t resultSize = oss.str().size();
fOutputString.reset(new uint8_t[resultSize + 2]);
memset(fOutputString.get(), 0, resultSize + 2);
for (rit = fOrderByQueue.rbegin(); rit != fOrderByQueue.rend(); ++rit)
{
if (addSep)
oss << sep;
else
addSep = true;
const OrderByRow& topRow = *rit;
fRow0.setData(topRow.fData);
outputRow(oss, fRow0);
}
fOrderByQueue.clear();
strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen);
#endif
strncpy((char*)fOutputString.get(),
oss.str().c_str(), resultSize);
}
uint8_t* GroupConcator::getResult(const string& sep)
@@ -1010,7 +978,6 @@ void GroupConcatNoOrder::initialize(const rowgroup::SP_GroupConcat& gcc)
throw IDBExcept(fErrorCode);
}
//fData.reset(new uint8_t[fRowGroup.getDataSize(fRowsPerRG)]);
fData.reinit(fRowGroup, fRowsPerRG);
fRowGroup.setData(&fData);
fRowGroup.resetRowGroup(0);
@@ -1025,7 +992,6 @@ void GroupConcatNoOrder::processRow(const rowgroup::Row& row)
if (fCurrentLength < fGroupConcatLen && concatColIsNull(row) == false)
{
copyRow(row, &fRow);
//memcpy(fRow.getData(), row.getData(), row.getSize());
// the RID is no meaning here, use it to store the estimated length.
int16_t estLen = lengthEstimate(fRow);
@@ -1049,7 +1015,6 @@ void GroupConcatNoOrder::processRow(const rowgroup::Row& row)
fDataQueue.push(fData);
fData.reinit(fRowGroup, fRowsPerRG);
//fData.reset(new uint8_t[fRowGroup.getDataSize(fRowsPerRG)]);
fRowGroup.setData(&fData);
fRowGroup.resetRowGroup(0);
fRowGroup.getRow(0, &fRow);
@@ -1100,7 +1065,12 @@ void GroupConcatNoOrder::getResult(uint8_t* buff, const string& sep)
fDataQueue.pop();
}
strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen);
size_t resultSize = oss.str().size();
fOutputString.reset(new uint8_t[resultSize + 2]);
memset(fOutputString.get(), 0, resultSize + 2);
strncpy((char*)fOutputString.get(),
oss.str().c_str(), resultSize);
}

13
utils/rowgroup/rowgroup.cpp
View File

@@ -1211,12 +1211,13 @@ int64_t Row::getSignedNullValue(uint32_t colIndex) const
RowGroup::RowGroup() : columnCount(0), data(NULL), rgData(NULL), strings(NULL),
useStringTable(true), hasLongStringField(false), sTableThreshold(20)
{
oldOffsets.reserve(1024);
oids.reserve(1024);
keys.reserve(1024);
types.reserve(1024);
scale.reserve(1024);
precision.reserve(1024);
// 1024 is too generous to waste.
oldOffsets.reserve(10);
oids.reserve(10);
keys.reserve(10);
types.reserve(10);
scale.reserve(10);
precision.reserve(10);
}
RowGroup::RowGroup(uint32_t colCount,