You've already forked mariadb-columnstore-engine
mirror of
https://github.com/mariadb-corporation/mariadb-columnstore-engine.git
synced 2025-07-29 08:21:15 +03:00
MCOL-5044 This patch simplifies addJob interfaces removing extra bool that control mutex locking,
adds additional nullptr dereference check in removeJobs and fixes FairThreadPool hashmap iter invalidation issues
This commit is contained in:
committed by
Roman Nozdrin
parent
4d41a945db
commit
0907ca414f
@ -55,11 +55,6 @@ FairThreadPool::~FairThreadPool()
|
||||
}
|
||||
|
||||
void FairThreadPool::addJob(const Job& job)
|
||||
{
|
||||
addJob_(job);
|
||||
}
|
||||
|
||||
void FairThreadPool::addJob_(const Job& job, bool useLock)
|
||||
{
|
||||
boost::thread* newThread;
|
||||
std::unique_lock<std::mutex> lk(mutex, std::defer_lock_t());
|
||||
@ -72,8 +67,7 @@ void FairThreadPool::addJob_(const Job& job, bool useLock)
|
||||
threadCounts_.fetch_add(1, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
if (useLock)
|
||||
lk.lock();
|
||||
lk.lock();
|
||||
// If some threads have blocked (because of output queue full)
|
||||
// Temporarily add some extra worker threads to make up for the blocked threads.
|
||||
if (blockedThreads_ > extraThreads_)
|
||||
@ -106,21 +100,22 @@ void FairThreadPool::addJob_(const Job& job, bool useLock)
|
||||
jobsListMapIter->second->push_back(job);
|
||||
}
|
||||
|
||||
if (useLock)
|
||||
newJob.notify_one();
|
||||
newJob.notify_one();
|
||||
}
|
||||
|
||||
void FairThreadPool::removeJobs(uint32_t id)
|
||||
{
|
||||
// std::cout << "FairThreadPool::removeJobs id " << id << std::endl;
|
||||
std::unique_lock<std::mutex> lk(mutex);
|
||||
|
||||
for (auto& txnJobsMapPair : txn2JobsListMap_)
|
||||
auto txnJobsMapIter = txn2JobsListMap_.begin();
|
||||
while (txnJobsMapIter != txn2JobsListMap_.end())
|
||||
{
|
||||
auto& txnJobsMapPair = *txnJobsMapIter;
|
||||
ThreadPoolJobsList* txnJobsList = txnJobsMapPair.second;
|
||||
if (txnJobsList->empty())
|
||||
// txnJobsList must not be nullptr
|
||||
if (txnJobsList && txnJobsList->empty())
|
||||
{
|
||||
txn2JobsListMap_.erase(txnJobsMapPair.first);
|
||||
txnJobsMapIter = txn2JobsListMap_.erase(txnJobsMapIter);
|
||||
delete txnJobsList;
|
||||
continue;
|
||||
// There is no clean-up for PQ. It will happen later in threadFcn
|
||||
@ -128,21 +123,22 @@ void FairThreadPool::removeJobs(uint32_t id)
|
||||
auto job = txnJobsList->begin();
|
||||
while (job != txnJobsList->end())
|
||||
{
|
||||
// std::cout << "removeJobs() job->id_ " << job->id_ << std::endl;
|
||||
if (job->id_ == id)
|
||||
{
|
||||
job = txnJobsList->erase(job); // update the job iter
|
||||
if (txnJobsList->empty())
|
||||
{
|
||||
txn2JobsListMap_.erase(txnJobsMapPair.first);
|
||||
delete txnJobsList;
|
||||
break;
|
||||
// There is no clean-up for PQ. It will happen later in threadFcn
|
||||
}
|
||||
continue; // go-on skiping job iter increment
|
||||
continue; // go-on skiping job iter increment
|
||||
}
|
||||
++job;
|
||||
}
|
||||
|
||||
if (txnJobsList->empty())
|
||||
{
|
||||
txnJobsMapIter = txn2JobsListMap_.erase(txnJobsMapIter);
|
||||
delete txnJobsList;
|
||||
continue;
|
||||
// There is no clean-up for PQ. It will happen later in threadFcn
|
||||
}
|
||||
++txnJobsMapIter;
|
||||
}
|
||||
}
|
||||
|
||||
@ -185,6 +181,7 @@ void FairThreadPool::threadFcn(const PriorityThreadPool::Priority preferredQueue
|
||||
{
|
||||
ThreadPoolJobsList* txnJobsList = txnAndJobListPair->second;
|
||||
delete txnJobsList;
|
||||
// !txnAndJobListPair is invalidated after this!
|
||||
txn2JobsListMap_.erase(txnAndJobListPair->first);
|
||||
}
|
||||
weightedTxnsQueue_.pop();
|
||||
@ -231,16 +228,12 @@ void FairThreadPool::threadFcn(const PriorityThreadPool::Priority preferredQueue
|
||||
{
|
||||
// to avoid excessive CPU usage waiting for data from storage
|
||||
usleep(500);
|
||||
lk.lock();
|
||||
addJob_(runList[0], false);
|
||||
newJob.notify_one();
|
||||
lk.unlock();
|
||||
addJob(runList[0]);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (std::exception& ex)
|
||||
{
|
||||
// std::cout << "FairThreadPool::threadFcn(): std::exception - no reschedule but send an error" << std::endl;
|
||||
if (running)
|
||||
{
|
||||
jobsRunning_.fetch_sub(1, std::memory_order_relaxed);
|
||||
@ -270,13 +263,12 @@ void FairThreadPool::threadFcn(const PriorityThreadPool::Priority preferredQueue
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
std::cout << "FairThreadPool::threadFcn(): std::exception - double exception: failed to send an error" << std::endl;
|
||||
std::cout << "FairThreadPool::threadFcn(): std::exception - double exception: failed to send an error"
|
||||
<< std::endl;
|
||||
}
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// std::cout << "FairThreadPool::threadFcn(): ... exception - no reschedule but send an error" << std::endl;
|
||||
// Log the exception and exit this thread
|
||||
try
|
||||
{
|
||||
if (running)
|
||||
@ -302,7 +294,8 @@ void FairThreadPool::threadFcn(const PriorityThreadPool::Priority preferredQueue
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
std::cout << "FairThreadPool::threadFcn(): ... exception - double exception: failed to send an error" << std::endl;
|
||||
std::cout << "FairThreadPool::threadFcn(): ... exception - double exception: failed to send an error"
|
||||
<< std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -39,6 +39,7 @@
|
||||
|
||||
namespace threadpool
|
||||
{
|
||||
|
||||
// The idea of this thread pool is to run morsel jobs(primitive job) is to equaly distribute CPU time
|
||||
// b/w multiple parallel queries(thread maps morsel to query using txnId). Query(txnId) has its weight
|
||||
// stored in PriorityQueue that thread increases before run another morsel for the query. When query is
|
||||
@ -144,7 +145,6 @@ class FairThreadPool
|
||||
explicit FairThreadPool(const FairThreadPool&);
|
||||
FairThreadPool& operator=(const FairThreadPool&);
|
||||
|
||||
void addJob_(const Job& job, bool useLock = true);
|
||||
void threadFcn(const PriorityThreadPool::Priority preferredQueue);
|
||||
void sendErrorMsg(uint32_t id, uint32_t step, primitiveprocessor::SP_UM_IOSOCK sock);
|
||||
|
||||
|
Reference in New Issue
Block a user