database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-06-15 03:21:42 +03:00
Code Activity

fix(perf,allocator): test build with reduced CountingAllocator parameter values.

Browse Source
This commit is contained in:
drrtuy
2025-04-29 17:54:36 +00:00
parent 252b3ddbef
commit 01cc73d416
8 changed files with 51 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
dbcon/joblist/tupleannexstep.cpp
View File

@ -85,8 +85,9 @@ struct TAEq
// TODO: Generalize these and put them back in utils/common/hasher.h // TODO: Generalize these and put them back in utils/common/hasher.h
// using TNSDistinctMap_t = std::unordered_set<rowgroup::Row::Pointer, TAHasher, TAEq, // using TNSDistinctMap_t = std::unordered_set<rowgroup::Row::Pointer, TAHasher, TAEq,
// allocators::CountingAllocator<rowgroup::Row::Pointer> >; // allocators::CountingAllocator<rowgroup::Row::Pointer> >;
using TNSDistinctMap_t = using TNSDistinctMap_t = std::unordered_set<rowgroup::Row::Pointer, TAHasher, TAEq>;
std::unordered_set<rowgroup::Row::Pointer, TAHasher, TAEq, STLPoolAllocator<rowgroup::Row::Pointer> >; // using TNSDistinctMap_t =
// std::unordered_set<rowgroup::Row::Pointer, TAHasher, TAEq, STLPoolAllocator<rowgroup::Row::Pointer> >;
}; // namespace }; // namespace
inline uint64_t TAHasher::operator()(const Row::Pointer& p) const inline uint64_t TAHasher::operator()(const Row::Pointer& p) const
@ -462,8 +463,10 @@ void TupleAnnexStep::executeNoOrderByWithDistinct()
// auto alloc = fRm->getAllocator<rowgroup::Row::Pointer>(); // auto alloc = fRm->getAllocator<rowgroup::Row::Pointer>();
// std::unique_ptr<TNSDistinctMap_t> distinctMap(new TNSDistinctMap_t(10, TAHasher(this), TAEq(this), // std::unique_ptr<TNSDistinctMap_t> distinctMap(new TNSDistinctMap_t(10, TAHasher(this), TAEq(this),
// alloc)); // alloc));
std::unique_ptr<TNSDistinctMap_t> distinctMap( // WIP
new TNSDistinctMap_t(10, TAHasher(this), TAEq(this), STLPoolAllocator<rowgroup::Row::Pointer>(fRm))); // std::unique_ptr<TNSDistinctMap_t> distinctMap(
// new TNSDistinctMap_t(10, TAHasher(this), TAEq(this), STLPoolAllocator<rowgroup::Row::Pointer>(fRm)));
std::unique_ptr<TNSDistinctMap_t> distinctMap(new TNSDistinctMap_t(10, TAHasher(this), TAEq(this)));
rgDataOut.reinit(fRowGroupOut); rgDataOut.reinit(fRowGroupOut);
fRowGroupOut.setData(&rgDataOut); fRowGroupOut.setData(&rgDataOut);
@ -724,8 +727,9 @@ void TupleAnnexStep::finalizeParallelOrderByDistinct()
ordering::SortingPQ finalPQ(rowgroup::rgCommonSize, fRm->getAllocator<ordering::OrderByRow>()); ordering::SortingPQ finalPQ(rowgroup::rgCommonSize, fRm->getAllocator<ordering::OrderByRow>());
// ordering::SortingPQ finalPQ(rowgroup::rgCommonSize); // ordering::SortingPQ finalPQ(rowgroup::rgCommonSize);
// auto allocDistinct = fRm->getAllocator<rowgroup::Row::Pointer>(); // auto allocDistinct = fRm->getAllocator<rowgroup::Row::Pointer>();
std::unique_ptr<TNSDistinctMap_t> distinctMap( // std::unique_ptr<TNSDistinctMap_t> distinctMap(
new TNSDistinctMap_t(10, TAHasher(this), TAEq(this), STLPoolAllocator<rowgroup::Row::Pointer>(fRm))); // new TNSDistinctMap_t(10, TAHasher(this), TAEq(this), STLPoolAllocator<rowgroup::Row::Pointer>(fRm)));
std::unique_ptr<TNSDistinctMap_t> distinctMap(new TNSDistinctMap_t(10, TAHasher(this), TAEq(this)));
fRowGroupIn.initRow(&row1); fRowGroupIn.initRow(&row1);
fRowGroupIn.initRow(&row2); fRowGroupIn.initRow(&row2);

16
primitives/primproc/batchprimitiveprocessor.cpp
View File

@ -364,17 +364,17 @@ void BatchPrimitiveProcessor::initBPP(ByteStream& bs)
if (!typelessJoin[i]) if (!typelessJoin[i])
{ {
// auto allocator = exemgr::globServiceExeMgr->getRm().getAllocator<TJoiner::value_type>(); auto allocator = exemgr::globServiceExeMgr->getRm().getAllocator<TJoiner::value_type>();
bs >> joinNullValues[i]; bs >> joinNullValues[i];
bs >> largeSideKeyColumns[i]; bs >> largeSideKeyColumns[i];
for (uint j = 0; j < processorThreads; ++j) for (uint j = 0; j < processorThreads; ++j)
// tJoiners[i][j].reset(new TJoiner(10, TupleJoiner::hasher(), allocator)); tJoiners[i][j].reset(new TJoiner(10, TupleJoiner::hasher(), std::equal_to<uint64_t>(), allocator));
tJoiners[i][j].reset(new TJoiner(10, TupleJoiner::hasher(), std::equal_to<uint64_t>(), // tJoiners[i][j].reset(new TJoiner(10, TupleJoiner::hasher(), std::equal_to<uint64_t>(),
utils::STLPoolAllocator<TJoiner::value_type>(resourceManager))); // utils::STLPoolAllocator<TJoiner::value_type>(resourceManager)));
} }
else else
{ {
// auto allocator = exemgr::globServiceExeMgr->getRm().getAllocator<TLJoiner::value_type>(); auto allocator = exemgr::globServiceExeMgr->getRm().getAllocator<TLJoiner::value_type>();
deserializeVector<uint32_t>(bs, tlLargeSideKeyColumns[i]); deserializeVector<uint32_t>(bs, tlLargeSideKeyColumns[i]);
bs >> tlSmallSideKeyLengths[i]; bs >> tlSmallSideKeyLengths[i];
@ -397,9 +397,9 @@ void BatchPrimitiveProcessor::initBPP(ByteStream& bs)
mSmallSideKeyColumnsPtr, mSmallSideRGPtr); mSmallSideKeyColumnsPtr, mSmallSideRGPtr);
auto tlComparator = TupleJoiner::TypelessDataComparator(&outputRG, &tlLargeSideKeyColumns[i], auto tlComparator = TupleJoiner::TypelessDataComparator(&outputRG, &tlLargeSideKeyColumns[i],
mSmallSideKeyColumnsPtr, mSmallSideRGPtr); mSmallSideKeyColumnsPtr, mSmallSideRGPtr);
// tlJoiners[i][j].reset(new TLJoiner(10, tlHasher, tlComparator, allocator)); tlJoiners[i][j].reset(new TLJoiner(10, tlHasher, tlComparator, allocator));
tlJoiners[i][j].reset(new TLJoiner(10, tlHasher, tlComparator, // tlJoiners[i][j].reset(new TLJoiner(10, tlHasher, tlComparator,
utils::STLPoolAllocator<TLJoiner::value_type>(resourceManager))); // utils::STLPoolAllocator<TLJoiner::value_type>(resourceManager)));
} }
} }
} }

28
primitives/primproc/batchprimitiveprocessor.h
View File

@ -308,23 +308,23 @@ class BatchPrimitiveProcessor
bool hasRowGroup; bool hasRowGroup;
/* Rowgroups + join */ /* Rowgroups + join */
// using TJoiner = using TJoiner =
// std::unordered_multimap<uint64_t, uint32_t, joiner::TupleJoiner::hasher, std::equal_to<uint64_t>, std::unordered_multimap<uint64_t, uint32_t, joiner::TupleJoiner::hasher, std::equal_to<uint64_t>,
// allocators::CountingAllocator<std::pair<const uint64_t, uint32_t>>>; allocators::CountingAllocator<std::pair<const uint64_t, uint32_t>>>;
using TJoiner = std::tr1::unordered_multimap<uint64_t, uint32_t, joiner::TupleJoiner::hasher, // using TJoiner = std::tr1::unordered_multimap<uint64_t, uint32_t, joiner::TupleJoiner::hasher,
std::equal_to<uint64_t>, // std::equal_to<uint64_t>,
utils::STLPoolAllocator<std::pair<const uint64_t, uint32_t>>>; // utils::STLPoolAllocator<std::pair<const uint64_t, uint32_t>>>;
// using TLJoiner = using TLJoiner =
// std::unordered_multimap<joiner::TypelessData, uint32_t, joiner::TupleJoiner::TypelessDataHasher, std::unordered_multimap<joiner::TypelessData, uint32_t, joiner::TupleJoiner::TypelessDataHasher,
// joiner::TupleJoiner::TypelessDataComparator, joiner::TupleJoiner::TypelessDataComparator,
// allocators::CountingAllocator<std::pair<const joiner::TypelessData, uint32_t>>>; allocators::CountingAllocator<std::pair<const joiner::TypelessData, uint32_t>>>;
using TLJoiner = std::tr1::unordered_multimap< // using TLJoiner = std::tr1::unordered_multimap<
joiner::TypelessData, uint32_t, joiner::TupleJoiner::TypelessDataHasher, // joiner::TypelessData, uint32_t, joiner::TupleJoiner::TypelessDataHasher,
joiner::TupleJoiner::TypelessDataComparator, // joiner::TupleJoiner::TypelessDataComparator,
utils::STLPoolAllocator<std::pair<const joiner::TypelessData, uint32_t>>>; // utils::STLPoolAllocator<std::pair<const joiner::TypelessData, uint32_t>>>;
bool generateJoinedRowGroup(rowgroup::Row& baseRow, const uint32_t depth = 0); bool generateJoinedRowGroup(rowgroup::Row& baseRow, const uint32_t depth = 0);
/* generateJoinedRowGroup helper fcns & vars */ /* generateJoinedRowGroup helper fcns & vars */

10
utils/common/poolallocator.cpp
View File

@ -63,13 +63,13 @@ void PoolAllocator::newBlock()
{ {
if (alloc) if (alloc)
{ {
std::cout << "PoolAllocator new block with counting alloc" << std::endl; // std::cout << "PoolAllocator new block with counting alloc" << std::endl;
mem.emplace_back(boost::allocate_shared<PoolAllocatorBufType>(*alloc, allocSize)); mem.emplace_back(boost::allocate_shared<PoolAllocatorBufType>(*alloc, allocSize));
} }
else else
{ {
std::cout << "PoolAllocator new block w/o counting alloc" << std::endl; // std::cout << "PoolAllocator new block w/o counting alloc" << std::endl;
mem.emplace_back(boost::make_shared<PoolAllocatorBufType>(allocSize)); mem.emplace_back(boost::make_shared<PoolAllocatorBufType>(allocSize));
} }
nextAlloc = mem.back().get(); nextAlloc = mem.back().get();
@ -81,17 +81,17 @@ void PoolAllocator::newBlock()
void* PoolAllocator::allocOOB(uint64_t size) void* PoolAllocator::allocOOB(uint64_t size)
{ {
OOBMemInfo memInfo; OOBMemInfo memInfo;
std::cout << "PoolAllocator allocOOB" << std::endl; // std::cout << "PoolAllocator allocOOB" << std::endl;
memUsage += size; memUsage += size;
if (alloc) if (alloc)
{ {
std::cout << "PoolAllocator allocOOB with counting alloc" << std::endl; // std::cout << "PoolAllocator allocOOB with counting alloc" << std::endl;
memInfo.mem = boost::allocate_shared<PoolAllocatorBufType>(*alloc, size); memInfo.mem = boost::allocate_shared<PoolAllocatorBufType>(*alloc, size);
} }
else else
{ {
std::cout << "PoolAllocator allocOOB w/o counting alloc" << std::endl; // std::cout << "PoolAllocator allocOOB w/o counting alloc" << std::endl;
memInfo.mem = boost::make_shared<PoolAllocatorBufType>(size); memInfo.mem = boost::make_shared<PoolAllocatorBufType>(size);
} }
memInfo.size = size; memInfo.size = size;

6
utils/common/stlpoolallocator.h
View File

@ -101,13 +101,13 @@ STLPoolAllocator<T>::STLPoolAllocator(joblist::ResourceManager* rm)
{ {
if (rm) if (rm)
{ {
std::cout << "STLPoolAllocator with RM " << std::endl; // std::cout << "STLPoolAllocator with RM " << std::endl;
auto alloc = rm->getAllocator<PoolAllocatorBufType>(); auto alloc = rm->getAllocator<PoolAllocatorBufType>(1024);
pa.reset(new PoolAllocator(alloc, DEFAULT_SIZE)); pa.reset(new PoolAllocator(alloc, DEFAULT_SIZE));
} }
else else
{ {
std::cout << "STLPoolAllocator w/o RM " << std::endl; // std::cout << "STLPoolAllocator w/o RM " << std::endl;
pa.reset(new PoolAllocator(DEFAULT_SIZE)); pa.reset(new PoolAllocator(DEFAULT_SIZE));
} }
} }

2
utils/joiner/tuplejoiner.cpp
View File

@ -45,8 +45,10 @@ std::unique_ptr<HashTable> makeHashMap(size_t bucketCount, ResourceManager* reso
{ {
// auto alloc = resourceManager->getAllocator<T>(); // auto alloc = resourceManager->getAllocator<T>();
// return std::unique_ptr<T>(new T(bucketCount, TupleJoiner::hasher(), typename T::key_equal(), alloc)); // return std::unique_ptr<T>(new T(bucketCount, TupleJoiner::hasher(), typename T::key_equal(), alloc));
// std::cout << " makeHashMap " << std::endl;
return std::unique_ptr<HashTable>(new HashTable(bucketCount, TupleJoiner::hasher(), return std::unique_ptr<HashTable>(new HashTable(bucketCount, TupleJoiner::hasher(),
typename HashTable::key_equal(), typename HashTable::key_equal(),
// resourceManager->getAllocator<typename HashTable::value_type>()));
utils::STLPoolAllocator<typename HashTable::value_type>(resourceManager))); utils::STLPoolAllocator<typename HashTable::value_type>(resourceManager)));
} }

3
utils/joiner/tuplejoiner.h
View File

@ -497,6 +497,9 @@ class TupleJoiner
template<typename K, typename V> template<typename K, typename V>
using HashMapTemplate = std::unordered_multimap<K, V, hasher, std::equal_to<K>, using HashMapTemplate = std::unordered_multimap<K, V, hasher, std::equal_to<K>,
utils::STLPoolAllocator<std::pair<const K, V>>>; utils::STLPoolAllocator<std::pair<const K, V>>>;
// template<typename K, typename V>
// using HashMapTemplate = std::unordered_multimap<K, V, hasher, std::equal_to<K>,
// allocators::CountingAllocator<std::pair<const K, V>>>;
using hash_t = HashMapTemplate<int64_t, uint8_t*>; using hash_t = HashMapTemplate<int64_t, uint8_t*>;
using sthash_t = HashMapTemplate<int64_t, rowgroup::Row::Pointer>; using sthash_t = HashMapTemplate<int64_t, rowgroup::Row::Pointer>;
using typelesshash_t = HashMapTemplate<TypelessData, rowgroup::Row::Pointer>; using typelesshash_t = HashMapTemplate<TypelessData, rowgroup::Row::Pointer>;

11
utils/windowfunction/idborderby.cpp
View File

@ -776,11 +776,12 @@ void IdbOrderBy::initialize(const RowGroup& rg)
// These two blocks contain structs with memory accounting. // These two blocks contain structs with memory accounting.
fOrderByQueue.reset(new SortingPQ(rowgroup::rgCommonSize, fRm->getAllocator<OrderByRow>())); fOrderByQueue.reset(new SortingPQ(rowgroup::rgCommonSize, fRm->getAllocator<OrderByRow>()));
if (fDistinct) // WIP
{ // if (fDistinct)
fDistinctMap.reset(new DistinctMap_t(10, Hasher(this, getKeyLength()), Eq(this, getKeyLength()), // {
utils::STLPoolAllocator<rowgroup::Row::Pointer>(fRm))); // fDistinctMap.reset(new DistinctMap_t(10, Hasher(this, getKeyLength()), Eq(this, getKeyLength()),
} // utils::STLPoolAllocator<rowgroup::Row::Pointer>(fRm)));
// }
} }
bool IdbOrderBy::getData(RGData& data) bool IdbOrderBy::getData(RGData& data)