database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-01 06:46:55 +03:00
Code Activity

feat(PP,ByteStream): new counting memory allocator

Browse Source
This commit is contained in:
drrtuy
2024-11-22 00:56:26 +00:00
parent 2d69b49ba0
commit 02b8ea1331
27 changed files with 548 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
dbcon/joblist/distributedenginecomm.cpp
View File

@ -427,7 +427,7 @@ Error:
// eventually let jobstep error out.
std::unique_lock lk(fMlock);
MessageQueueMap::iterator map_tok;
sbs.reset(new ByteStream(0));
sbs.reset(new ByteStream(0U));
for (map_tok = fSessionMessages.begin(); map_tok != fSessionMessages.end(); ++map_tok)
{
@ -1103,7 +1103,7 @@ int DistributedEngineComm::writeToClient(size_t aPMIndex, const SBS& bs, uint32_
std::unique_lock lk(fMlock);
// std::cout << "WARNING: DEC WRITE BROKEN PIPE. PMS index = " << index << std::endl;
MessageQueueMap::iterator map_tok;
sbs.reset(new ByteStream(0));
sbs.reset(new ByteStream(0U));
for (map_tok = fSessionMessages.begin(); map_tok != fSessionMessages.end(); ++map_tok)
{

17
dbcon/joblist/resourcemanager.cpp
View File

@ -22,6 +22,7 @@
******************************************************************************************/
#include <unistd.h>
#include <atomic>
#include <string>
#include <stdexcept>
#include <iostream>
@ -346,23 +347,23 @@ bool ResourceManager::userPriorityEnabled() const
// If both have space, return true.
bool ResourceManager::getMemory(int64_t amount, boost::shared_ptr<int64_t>& sessionLimit, bool patience)
{
bool ret1 = (atomicops::atomicSub(&totalUmMemLimit, amount) >= 0);
bool ret1 = (totalUmMemLimit.fetch_sub(amount, std::memory_order_relaxed) >= 0);
bool ret2 = sessionLimit ? (atomicops::atomicSub(sessionLimit.get(), amount) >= 0) : ret1;
uint32_t retryCounter = 0, maxRetries = 20; // 10s delay
while (patience && !(ret1 && ret2) && retryCounter++ < maxRetries)
{
atomicops::atomicAdd(&totalUmMemLimit, amount);
totalUmMemLimit.fetch_add(amount, std::memory_order_relaxed);
sessionLimit ? atomicops::atomicAdd(sessionLimit.get(), amount) : 0;
usleep(500000);
ret1 = (atomicops::atomicSub(&totalUmMemLimit, amount) >= 0);
ret1 = (totalUmMemLimit.fetch_sub(amount, std::memory_order_relaxed) >= 0);
ret2 = sessionLimit ? (atomicops::atomicSub(sessionLimit.get(), amount) >= 0) : ret1;
}
if (!(ret1 && ret2))
{
// If we didn't get any memory, restore the counters.
atomicops::atomicAdd(&totalUmMemLimit, amount);
totalUmMemLimit.fetch_add(amount, std::memory_order_relaxed);
sessionLimit ? atomicops::atomicAdd(sessionLimit.get(), amount) : 0;
}
return (ret1 && ret2);
@ -371,20 +372,20 @@ bool ResourceManager::getMemory(int64_t amount, boost::shared_ptr<int64_t>& sess
// The amount type is unsafe if amount close to max<int64_t> that is unrealistic in 2024.
bool ResourceManager::getMemory(int64_t amount, bool patience)
{
bool ret1 = (atomicops::atomicSub(&totalUmMemLimit, amount) >= 0);
bool ret1 = (totalUmMemLimit.fetch_sub(amount, std::memory_order_relaxed) >= 0);
uint32_t retryCounter = 0, maxRetries = 20; // 10s delay
while (patience && !ret1 && retryCounter++ < maxRetries)
{
atomicops::atomicAdd(&totalUmMemLimit, amount);
totalUmMemLimit.fetch_add(amount, std::memory_order_relaxed);
usleep(500000);
ret1 = (atomicops::atomicSub(&totalUmMemLimit, amount) >= 0);
ret1 = (totalUmMemLimit.fetch_sub(amount, std::memory_order_relaxed) >= 0);
}
if (!ret1)
{
// If we didn't get any memory, restore the counters.
atomicops::atomicAdd(&totalUmMemLimit, amount);
totalUmMemLimit.fetch_add(amount, std::memory_order_relaxed);
}
return ret1;
}

16
dbcon/joblist/resourcemanager.h
View File

@ -25,6 +25,7 @@
*/
#pragma once
#include <atomic>
#include <vector>
#include <iostream>
#include <boost/thread.hpp>
@ -33,6 +34,7 @@
#include "configcpp.h"
#include "calpontselectexecutionplan.h"
#include "countingallocator.h"
#include "resourcedistributor.h"
#include "installdir.h"
#include "branchpred.h"
@ -325,16 +327,16 @@ class ResourceManager
bool getMemory(int64_t amount, bool patience = true);
inline void returnMemory(int64_t amount)
{
atomicops::atomicAdd(&totalUmMemLimit, amount);
totalUmMemLimit.fetch_add(amount, std::memory_order_relaxed);
}
inline void returnMemory(int64_t amount, boost::shared_ptr<int64_t>& sessionLimit)
{
atomicops::atomicAdd(&totalUmMemLimit, amount);
totalUmMemLimit.fetch_add(amount, std::memory_order_relaxed);
sessionLimit ? atomicops::atomicAdd(sessionLimit.get(), amount) : 0;
}
inline int64_t availableMemory() const
{
return totalUmMemLimit;
return totalUmMemLimit.load(std::memory_order_relaxed);
}
/* old HJ mem interface, used by HashJoin */
@ -454,6 +456,12 @@ class ResourceManager
return configuredUmMemLimit;
}
template<typename T>
allocators::CountingAllocator<T> getAllocator()
{
return allocators::CountingAllocator<T>(totalUmMemLimit);
}
private:
void logResourceChangeMessage(logging::LOG_TYPE logType, uint32_t sessionID, uint64_t newvalue,
uint64_t value, const std::string& source, logging::Message::MessageID mid);
@ -504,7 +512,7 @@ class ResourceManager
LockedSessionMap fHJPmMaxMemorySmallSideSessionMap;
/* new HJ/Union/Aggregation support */
volatile int64_t totalUmMemLimit; // mem limit for join, union, and aggregation on the UM
std::atomic<int64_t> totalUmMemLimit{0}; // mem limit for join, union, and aggregation on the UM
int64_t configuredUmMemLimit;
uint64_t pmJoinMemLimit; // mem limit on individual PM joins