diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index cf05c491d..b67623e9d 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -332,6 +332,7 @@ void GroupConcatAgUM::initialize() { fRowGroup = fGroupConcat->fRowGroup; fRowGroup.setUseStringTable(true); + fRowGroup.setUseOnlyLongString(true); fRowRGData.reinit(fRowGroup, 1); fRowGroup.setData(&fRowRGData); fRowGroup.resetRowGroup(0); @@ -998,6 +999,7 @@ void GroupConcatNoOrder::initialize(const rowgroup::SP_GroupConcat& gcc) GroupConcator::initialize(gcc); fRowGroup = gcc->fRowGroup; + fRowGroup.setUseOnlyLongString(true); fRowsPerRG = 128; fErrorCode = ERR_AGGREGATION_TOO_BIG; fRm = gcc->fRm; @@ -1010,7 +1012,7 @@ void GroupConcatNoOrder::initialize(const rowgroup::SP_GroupConcat& gcc) uint64_t newSize = fRowsPerRG * fRowGroup.getRowSize(); - if (!fRm->getMemory(newSize, fSessionMemLimit)) + if (fRm && !fRm->getMemory(newSize, fSessionMemLimit)) { cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) << " @" << __FILE__ << ":" << __LINE__; throw IDBExcept(fErrorCode); @@ -1020,6 +1022,7 @@ void GroupConcatNoOrder::initialize(const rowgroup::SP_GroupConcat& gcc) fData.reinit(fRowGroup, fRowsPerRG); fRowGroup.setData(&fData); + fRowGroup.setUseOnlyLongString(true); fRowGroup.resetRowGroup(0); fRowGroup.initRow(&fRow); fRowGroup.getRow(0, &fRow); diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 7a58f387a..b031fb221 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -79,7 +79,7 @@ uint64_t StringStore::storeString(const uint8_t* data, uint32_t len) if (mem.size() > 0) lastMC = (MemChunk*)mem.back().get(); - if ((len + 4) >= CHUNK_SIZE) + if ((len + 4) >= CHUNK_SIZE || fUseOnlyLongStrings) { auto allocSize = len + sizeof(MemChunk) + 4; if (alloc) @@ -316,8 +316,10 @@ RGData::RGData(const RowGroup& rg, uint32_t rowCount) RGDataSizeType s = rg.getDataSize(rowCount); rowData.reset(new uint8_t[s]); - if (rg.usesStringTable() && rowCount > 0) + if (rg.usesStringTable() && rowCount > 0) { strings.reset(new StringStore()); + strings->useOnlyLongStrings(rg.usesOnlyLongString()); + } userDataStore.reset(); columnCount = rg.getColumnCount(); @@ -329,7 +331,10 @@ RGData::RGData(const RowGroup& rg) rowData.reset(new uint8_t[rg.getMaxDataSize()]); if (rg.usesStringTable()) + { strings.reset(new StringStore()); + strings->useOnlyLongStrings(rg.usesOnlyLongString()); + } userDataStore.reset(); columnCount = rg.getColumnCount(); @@ -345,6 +350,7 @@ RGData::RGData(const RowGroup& rg, allocators::CountingAllocator& { allocators::CountingAllocator ssAlloc = _alloc; strings.reset(new StringStore(ssAlloc)); + strings->useOnlyLongStrings(rg.usesOnlyLongString()); } userDataStore.reset(); @@ -365,7 +371,7 @@ void RGData::reinit(const RowGroup& rg, uint32_t rowCount) userDataStore.reset(); -if (rg.usesStringTable()) + if (rg.usesStringTable()) { if (alloc) { @@ -386,7 +392,7 @@ if (rg.usesStringTable()) void RGData::reinit(const RowGroup& rg) { - reinit(rg, 8192); + reinit(rg, rgCommonSize); } void RGData::serialize(ByteStream& bs, RGDataSizeType amount) const @@ -1128,6 +1134,7 @@ RowGroup::RowGroup(const RowGroup& r) , rgData(r.rgData) , strings(r.strings) , useStringTable(r.useStringTable) + , useOnlyLongStrings(r.useOnlyLongStrings) , hasCollation(r.hasCollation) , hasLongStringField(r.hasLongStringField) , sTableThreshold(r.sTableThreshold) @@ -1160,6 +1167,7 @@ RowGroup& RowGroup::operator=(const RowGroup& r) rgData = r.rgData; strings = r.strings; useStringTable = r.useStringTable; + useOnlyLongStrings = r.useOnlyLongStrings; hasCollation = r.hasCollation; hasLongStringField = r.hasLongStringField; sTableThreshold = r.sTableThreshold; @@ -1212,6 +1220,7 @@ void RowGroup::serialize(ByteStream& bs) const bs << (uint8_t)hasLongStringField; bs << sTableThreshold; bs.append((uint8_t*)&forceInline[0], sizeof(bool) * columnCount); + bs << (uint8_t)useOnlyLongStrings; } void RowGroup::deserialize(ByteStream& bs) @@ -1238,6 +1247,8 @@ void RowGroup::deserialize(ByteStream& bs) forceInline.reset(new bool[columnCount]); memcpy(&forceInline[0], bs.buf(), sizeof(bool) * columnCount); bs.advance(sizeof(bool) * columnCount); + bs >> tmp8; + useOnlyLongStrings = (bool)tmp8; // offsets = (useStringTable ? &stOffsets[0] : &oldOffsets[0]); offsets = 0; diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 7ecab0eca..2401a7253 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -63,7 +63,7 @@ namespace rowgroup { -const int16_t rgCommonSize = 8192; +constexpr int16_t rgCommonSize = 8192; using RGDataSizeType = uint64_t; /* @@ -172,6 +172,8 @@ class StringStore { return fUseStoreStringMutex; } + void useOnlyLongStrings(bool b) { fUseOnlyLongStrings = b; } + bool useOnlyLongStrings() const { return fUseOnlyLongStrings; } // This is an overlay b/c the underlying data needs to be any size, // and alloc'd in one chunk. data can't be a separate dynamic chunk. @@ -193,6 +195,7 @@ class StringStore std::vector> longStrings; bool empty = true; bool fUseStoreStringMutex = false; //@bug6065, make StringStore::storeString() thread safe + bool fUseOnlyLongStrings = false; boost::mutex fMutex; std::optional> alloc {}; }; @@ -1556,6 +1559,8 @@ class RowGroup : public messageqcpp::Serializeable inline bool usesStringTable() const; inline void setUseStringTable(bool); + void setUseOnlyLongString(bool b) { useOnlyLongStrings = b; } + bool usesOnlyLongString() const { return useOnlyLongStrings ; } bool hasLongString() const { @@ -1628,6 +1633,8 @@ class RowGroup : public messageqcpp::Serializeable RGData* rgData = nullptr; StringStore* strings = nullptr; // note, strings and data belong to rgData bool useStringTable = true; + bool useOnlyLongStrings = false; + bool useAggregateDataStore = true; bool hasCollation = false; bool hasLongStringField = false; uint32_t sTableThreshold = 20;