database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-02 17:22:27 +03:00
Code Activity

MCOL4841 dev port run large join without OOM

Browse Source
This commit is contained in:
David Hall
2022-02-09 17:33:55 -06:00
parent d30e140dc3
commit 27dea733c5
34 changed files with 821 additions and 518 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
utils/threadpool/prioritythreadpool.cpp
View File

@ -28,6 +28,7 @@ using namespace std;
#include "messageobj.h"
#include "messagelog.h"
#include "threadnaming.h"
using namespace logging;
#include "prioritythreadpool.h"
@ -39,7 +40,7 @@ namespace threadpool
{
PriorityThreadPool::PriorityThreadPool(uint targetWeightPerRun, uint highThreads, uint midThreads,
uint lowThreads, uint ID)
: _stop(false), weightPerRun(targetWeightPerRun), id(ID)
: _stop(false), weightPerRun(targetWeightPerRun), id(ID), blockedThreads(0), extraThreads(0), stopExtra(true)
{
boost::thread* newThread;
for (uint32_t i = 0; i < highThreads; i++)
@ -98,6 +99,21 @@ void PriorityThreadPool::addJob(const Job& job, bool useLock)
threadCounts[LOW]++;
}
// If some threads have blocked (because of output queue full)
// Temporarily add some extra worker threads to make up for the blocked threads.
if (blockedThreads > extraThreads)
{
stopExtra = false;
newThread = threads.create_thread(ThreadHelper(this, EXTRA));
newThread->detach();
extraThreads++;
}
else if (blockedThreads == 0)
{
// Release the temporary threads -- some threads have become unblocked.
stopExtra = true;
}
if (job.priority > 66)
jobQueues[HIGH].push_back(job);
else if (job.priority > 33)
@ -125,7 +141,7 @@ void PriorityThreadPool::removeJobs(uint32_t id)
PriorityThreadPool::Priority PriorityThreadPool::pickAQueue(Priority preference)
{
if (!jobQueues[preference].empty())
if (preference != EXTRA && !jobQueues[preference].empty())
return preference;
else if (!jobQueues[HIGH].empty())
return HIGH;
@ -137,6 +153,10 @@ PriorityThreadPool::Priority PriorityThreadPool::pickAQueue(Priority preference)
void PriorityThreadPool::threadFcn(const Priority preferredQueue) throw()
{
if (preferredQueue == EXTRA)
utils::setThreadName("Extra");
else
utils::setThreadName("Idle");
Priority queue = LOW;
uint32_t weight, i = 0;
vector<Job> runList;
@ -155,6 +175,14 @@ void PriorityThreadPool::threadFcn(const Priority preferredQueue) throw()
if (jobQueues[queue].empty())
{
// If this is an EXTRA thread due toother threads blocking, and all blockers are unblocked,
// we don't want this one any more.
if (preferredQueue == EXTRA && stopExtra)
{
extraThreads--;
return;
}
newJob.wait(lk);
continue;
}
@ -190,6 +218,10 @@ void PriorityThreadPool::threadFcn(const Priority preferredQueue) throw()
if (reschedule[i])
rescheduleCount++;
}
if (preferredQueue == EXTRA)
utils::setThreadName("Extra (used)");
else
utils::setThreadName("Idle");
// no real work was done, prevent intensive busy waiting
if (rescheduleCount == runList.size())