MCOL-901 Significanlty reduced memory consumption for group_concat().

RowGroup default constructor allocates memory too generously.

    Removed commented code from groupconcat.cpp.

dbcon/joblist/groupconcat.cpp

@@ -344,8 +344,6 @@ void GroupConcatAgUM::merge(const rowgroup::Row& inRow, int64_t i)
     joblist::GroupConcatAgUM* gccAg = *((joblist::GroupConcatAgUM**)(data + inRow.getOffset(i)));
 
     fConcator->merge(gccAg->concator().get());
-//	don't reset
-//	gccAg->orderBy().reset();
 }
 
 
@@ -393,12 +391,11 @@ GroupConcator::~GroupConcator()
 
 void GroupConcator::initialize(const rowgroup::SP_GroupConcat& gcc)
 {
+    // MCOL-901 This value comes from the Server and it is
+    // too high(3MB) to allocate it for every instance.
     fGroupConcatLen = gcc->fSize;
     fCurrentLength -= strlen(gcc->fSeparator.c_str());
 
-    fOutputString.reset(new uint8_t[fGroupConcatLen + 2]);
-    memset(fOutputString.get(), 0, fGroupConcatLen + 2);
-
     fConstCols = gcc->fConstCols;
     fConstantLen = strlen(gcc->fSeparator.c_str());
 
@@ -477,7 +474,6 @@ void GroupConcator::outputRow(std::ostringstream& oss, const rowgroup::Row& row)
             case CalpontSystemCatalog::TEXT:
             {
                 oss << row.getStringField(*i).c_str();
-                //oss << row.getStringField(*i);
                 break;
             }
 
@@ -620,11 +616,6 @@ int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row)
             {
                 int64_t colWidth = row.getStringLength(*i);
                 fieldLen += colWidth;   // getStringLength() does the same thing as below
-                //assert(!row.usesStringTable());
-                //int64_t colWidth = row.getColumnWidth(*i);
-                //uint8_t* pStr = row.getData() + row.getOffset(*i);
-                //while ((*pStr++ > 0) && (fieldLen < colWidth))
-                //	fieldLen++;
                 break;
             }
 
@@ -752,7 +743,6 @@ void GroupConcatOrderBy::initialize(const rowgroup::SP_GroupConcat& gcc)
 uint64_t GroupConcatOrderBy::getKeyLength() const
 {
     // only distinct the concatenated columns
-    //return (fRow0.getOffset(fConcatColumns.size()) - 2);
     return fConcatColumns.size() - 1; // cols 0 to fConcatColumns will be conpared
 }
 
@@ -771,8 +761,6 @@ void GroupConcatOrderBy::processRow(const rowgroup::Row& row)
     if (fCurrentLength < fGroupConcatLen)
     {
         copyRow(row, &fRow0);
-        //cout << "length < GB limit: " << fRow0.toString() << endl;
-        //memcpy(fRow0.getData(), row.getData(), row.getSize());
         // the RID is no meaning here, use it to store the estimated length.
         int16_t estLen = lengthEstimate(fRow0);
         fRow0.setRid(estLen);
@@ -862,7 +850,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc)
             if (fDistinct)
                 fDistinctMap->insert(row.fData);
 
-            //fDistinctMap->insert(make_pair((row.fData+2), row.fData));
         }
 
         else if (fOrderByCond.size() > 0 && fRule.less(row.fData, fOrderByQueue.top().fData))
@@ -876,8 +863,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc)
             {
                 fDistinctMap->erase(swapRow.fData);
                 fDistinctMap->insert(row.fData);
-                //fDistinctMap->erase(fDistinctMap->find(swapRow.fData + 2));
-                //fDistinctMap->insert(make_pair((row.fData+2), row.fData));
             }
 
             row1.setData(row.fData);
@@ -893,7 +878,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc)
 
 void GroupConcatOrderBy::getResult(uint8_t* buff, const string& sep)
 {
-#if 1
     ostringstream oss;
     bool addSep = false;
 
@@ -919,28 +903,12 @@ void GroupConcatOrderBy::getResult(uint8_t* buff, const string& sep)
         rowStack.pop();
     }
 
-    strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen);
+    size_t resultSize = oss.str().size();
-#else
+    fOutputString.reset(new uint8_t[resultSize + 2]);
-    ostringstream oss;
+    memset(fOutputString.get(), 0, resultSize + 2);
-    bool addSep = false;
-    priority_queue<OrderByRow>::reverse_iterator rit;
 
-    for (rit = fOrderByQueue.rbegin(); rit != fOrderByQueue.rend(); ++rit)
+    strncpy((char*)fOutputString.get(),
-    {
+        oss.str().c_str(), resultSize);
-        if (addSep)
-            oss << sep;
-        else
-            addSep = true;
-
-        const OrderByRow& topRow = *rit;
-        fRow0.setData(topRow.fData);
-        outputRow(oss, fRow0);
-    }
-
-    fOrderByQueue.clear();
-
-    strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen);
-#endif
 }
 
 uint8_t* GroupConcator::getResult(const string& sep)
@@ -1010,7 +978,6 @@ void GroupConcatNoOrder::initialize(const rowgroup::SP_GroupConcat& gcc)
         throw IDBExcept(fErrorCode);
     }
 
-    //fData.reset(new uint8_t[fRowGroup.getDataSize(fRowsPerRG)]);
     fData.reinit(fRowGroup, fRowsPerRG);
     fRowGroup.setData(&fData);
     fRowGroup.resetRowGroup(0);
@@ -1025,7 +992,6 @@ void GroupConcatNoOrder::processRow(const rowgroup::Row& row)
     if (fCurrentLength < fGroupConcatLen && concatColIsNull(row) == false)
     {
         copyRow(row, &fRow);
-        //memcpy(fRow.getData(), row.getData(), row.getSize());
 
         // the RID is no meaning here, use it to store the estimated length.
         int16_t estLen = lengthEstimate(fRow);
@@ -1049,7 +1015,6 @@ void GroupConcatNoOrder::processRow(const rowgroup::Row& row)
 
             fDataQueue.push(fData);
             fData.reinit(fRowGroup, fRowsPerRG);
-            //fData.reset(new uint8_t[fRowGroup.getDataSize(fRowsPerRG)]);
             fRowGroup.setData(&fData);
             fRowGroup.resetRowGroup(0);
             fRowGroup.getRow(0, &fRow);
@@ -1100,7 +1065,12 @@ void GroupConcatNoOrder::getResult(uint8_t* buff, const string& sep)
         fDataQueue.pop();
     }
 
-    strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen);
+    size_t resultSize = oss.str().size();
+    fOutputString.reset(new uint8_t[resultSize + 2]);
+    memset(fOutputString.get(), 0, resultSize + 2);
+
+    strncpy((char*)fOutputString.get(),
+        oss.str().c_str(), resultSize);
 }
 
 

utils/rowgroup/rowgroup.cpp

@@ -1211,12 +1211,13 @@ int64_t Row::getSignedNullValue(uint32_t colIndex) const
 RowGroup::RowGroup() : columnCount(0), data(NULL), rgData(NULL), strings(NULL),
     useStringTable(true), hasLongStringField(false), sTableThreshold(20)
 {
-    oldOffsets.reserve(1024);
+    // 1024 is too generous to waste.
-    oids.reserve(1024);
+    oldOffsets.reserve(10);
-    keys.reserve(1024);
+    oids.reserve(10);
-    types.reserve(1024);
+    keys.reserve(10);
-    scale.reserve(1024);
+    types.reserve(10);
-    precision.reserve(1024);
+    scale.reserve(10);
+    precision.reserve(10);
 }
 
 RowGroup::RowGroup(uint32_t colCount,