database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-01 06:46:55 +03:00
Code Activity

MCOL-5001 This patch merges ExeMgr and PrimProc runtimes

Browse Source 
EM and PP are most resource-hungry runtimes.
        The merge enables to control their cummulative
        resource consumption, thread allocation + enables
        zero-copy data exchange b/w local EM and PP facilities.
This commit is contained in:
Roman Nozdrin
2022-03-02 17:13:29 +00:00
committed by Roman Nozdrin
parent 2ec502aaf3
commit e174696351
23 changed files with 2443 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
dbcon/joblist/distributedenginecomm.cpp
View File

@ -292,7 +292,7 @@ void DistributedEngineComm::Setup()
catch (std::exception& ex)
{
if (i < newPmCount)
newPmCount--;
newPmCount = newPmCount > 1 ? newPmCount-1 : 1; // We can't afford to reduce newPmCount to 0
writeToLog(__FILE__, __LINE__,
"Could not connect to PMS" + std::to_string(connectionId) + ": " + ex.what(),
@ -302,7 +302,7 @@ void DistributedEngineComm::Setup()
catch (...)
{
if (i < newPmCount)
newPmCount--;
newPmCount = newPmCount > 1 ? newPmCount-1 : 1; // We can't afford to reduce newPmCount to 0
writeToLog(__FILE__, __LINE__, "Could not connect to PMS" + std::to_string(connectionId),
LOG_TYPE_ERROR);

23
dbcon/joblist/resourcemanager.cpp
View File

@ -1,4 +1,5 @@
/* Copyright (C) 2014 InfiniDB, Inc.
Copyright (C) 2022 Mariadb Corporation.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
@ -40,39 +41,25 @@ using namespace config;
namespace joblist
{
// const string ResourceManager::fExeMgrStr("ExeMgr1");
const string ResourceManager::fHashJoinStr("HashJoin");
const string ResourceManager::fHashBucketReuseStr("HashBucketReuse");
const string ResourceManager::fJobListStr("JobList");
const string ResourceManager::fPrimitiveServersStr("PrimitiveServers");
// const string ResourceManager::fSystemConfigStr("SystemConfig");
const string ResourceManager::fTupleWSDLStr("TupleWSDL");
const string ResourceManager::fZDLStr("ZDL");
const string ResourceManager::fExtentMapStr("ExtentMap");
// const string ResourceManager::fDMLProcStr("DMLProc");
// const string ResourceManager::fBatchInsertStr("BatchInsert");
const string ResourceManager::fOrderByLimitStr("OrderByLimit");
const string ResourceManager::fRowAggregationStr("RowAggregation");
ResourceManager* ResourceManager::fInstance = NULL;
boost::mutex mx;
ResourceManager* ResourceManager::instance(bool runningInExeMgr)
ResourceManager* ResourceManager::instance(bool runningInExeMgr, config::Config* aConfig)
{
boost::mutex::scoped_lock lk(mx);
if (!fInstance)
fInstance = new ResourceManager(runningInExeMgr);
fInstance = new ResourceManager(runningInExeMgr, aConfig);
return fInstance;
}
ResourceManager::ResourceManager(bool runningInExeMgr)
ResourceManager::ResourceManager(bool runningInExeMgr, config::Config* aConfig)
: fExeMgrStr("ExeMgr1")
, fSystemConfigStr("SystemConfig")
, fDMLProcStr("DMLProc")
, fBatchInsertStr("BatchInsert")
, fConfig(Config::makeConfig())
, fConfig(aConfig == nullptr ? Config::makeConfig() : aConfig)
, fNumCores(8)
, fHjNumThreads(defaultNumThreads)
, fJlProcessorThreadsPerScan(defaultProcessorThreadsPerScan)

28
dbcon/joblist/resourcemanager.h
View File

@ -1,4 +1,5 @@
/* Copyright (C) 2014 InfiniDB, Inc.
Copyright (C) 2022 Mariadb Corporation.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
@ -138,11 +139,8 @@ class ResourceManager
/** @brief ctor
*
*/
EXPORT ResourceManager(bool runningInExeMgr = false);
static ResourceManager* instance(bool runningInExeMgr = false);
// ResourceManager(const config::Config *cf);
// ResourceManager(const std::string& config);
// passed by ExeMgr and DistributedEngineComm to MessageQueueServer or -Client
EXPORT ResourceManager(bool runningInExeMgr = false, config::Config *aConfig = nullptr);
static ResourceManager* instance(bool runningInExeMgr = false, config::Config *aConfig = nullptr);
config::Config* getConfig()
{
return fConfig;
@ -577,19 +575,19 @@ class ResourceManager
void logMessage(logging::LOG_TYPE logLevel, logging::Message::MessageID mid, uint64_t value = 0,
uint32_t sessionId = 0);
/*static const*/ std::string fExeMgrStr;
static const std::string fHashJoinStr;
static const std::string fHashBucketReuseStr;
static const std::string fJobListStr;
static const std::string fPrimitiveServersStr;
std::string fExeMgrStr;
inline static const std::string fHashJoinStr = "HashJoin";
inline static const std::string fHashBucketReuseStr = "HashBucketReuse";
inline static const std::string fJobListStr = "JobList";
inline static const std::string fPrimitiveServersStr = "PrimitiveServers";
/*static const*/ std::string fSystemConfigStr;
static const std::string fTupleWSDLStr;
static const std::string fZDLStr;
static const std::string fExtentMapStr;
inline static const std::string fTupleWSDLStr = "TupleWSDL";
inline static const std::string fZDLStr = "ZDL";
inline static const std::string fExtentMapStr = "ExtentMap";
/*static const*/ std::string fDMLProcStr;
/*static const*/ std::string fBatchInsertStr;
static const std::string fOrderByLimitStr;
static const std::string fRowAggregationStr;
inline static const std::string fOrderByLimitStr = "OrderByLimit";
inline static const std::string fRowAggregationStr = "RowAggregation";
config::Config* fConfig;
static ResourceManager* fInstance;
uint32_t fTraceFlags;