database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL 3585 update some defaults in config file

Browse Source
This commit is contained in:
jmrojas2332
2019-11-21 06:08:49 +00:00
parent dbb9b21d26
commit e712df5d0b
5 changed files with 37 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
dbcon/joblist/resourcemanager.cpp
View File

@ -87,7 +87,8 @@ ResourceManager::ResourceManager(bool runningInExeMgr) :
0), 0),
fHJPmMaxMemorySmallSideSessionMap( fHJPmMaxMemorySmallSideSessionMap(
getUintVal(fHashJoinStr, "PmMaxMemorySmallSide", defaultHJPmMaxMemorySmallSide)), getUintVal(fHashJoinStr, "PmMaxMemorySmallSide", defaultHJPmMaxMemorySmallSide)),
isExeMgr(runningInExeMgr) isExeMgr(runningInExeMgr),
fJlMaxOutstandingRequests(defaultMaxOutstandingRequests)
{ {
int temp; int temp;
int configNumCores = -1; int configNumCores = -1;
@ -127,9 +128,27 @@ ResourceManager::ResourceManager(bool runningInExeMgr) :
temp = getIntVal(fJobListStr, "ProcessorThreadsPerScan", -1); temp = getIntVal(fJobListStr, "ProcessorThreadsPerScan", -1);
if (temp > 0) if (temp > 0)
fJlProcessorThreadsPerScan = temp; fJlProcessorThreadsPerScan = temp;
temp = getIntVal(fJobListStr, "MaxOutstandingRequests", -1);
if (temp > 0)
fJlMaxOutstandingRequests = temp;
else
{
oam::Oam oam;
oam::ModuleTypeConfig moduletypeconfig;
oam.getSystemConfig("pm", moduletypeconfig);
const auto temp = moduletypeconfig.ModuleCount * fNumCores * 4 / fJlProcessorThreadsPerScan;
if (temp > defaultMaxOutstandingRequests)
{
fJlMaxOutstandingRequests = temp;
}
}
temp = getIntVal(fJobListStr, "NumScanReceiveThreads", -1); temp = getIntVal(fJobListStr, "NumScanReceiveThreads", -1);
if (temp > 0) if (temp > 0)
@ -140,7 +159,7 @@ ResourceManager::ResourceManager(bool runningInExeMgr) :
if (temp > 0) if (temp > 0)
fTwNumThreads = temp; fTwNumThreads = temp;
pmJoinMemLimit = getIntVal(fHashJoinStr, "PmMaxMemorySmallSide", pmJoinMemLimit = getUintVal(fHashJoinStr, "PmMaxMemorySmallSide",
defaultHJPmMaxMemorySmallSide); defaultHJPmMaxMemorySmallSide);
// Need to use different limits if this instance isn't running on the UM, // Need to use different limits if this instance isn't running on the UM,
@ -200,10 +219,12 @@ ResourceManager::ResourceManager(bool runningInExeMgr) :
nt = fConfig->getConfig("RowAggregation", "RowAggrThreads"); nt = fConfig->getConfig("RowAggregation", "RowAggrThreads");
if (nt.empty()) if (nt.empty())
if ( numCores() < 4 ) {
if ( numCores() > 0 )
fAggNumThreads = numCores(); fAggNumThreads = numCores();
else else
fAggNumThreads = 4; fAggNumThreads = 1;
}
else else
fAggNumThreads = fConfig->uFromText(nt); fAggNumThreads = fConfig->uFromText(nt);
@ -213,7 +234,7 @@ ResourceManager::ResourceManager(bool runningInExeMgr) :
fAggNumBuckets = fAggNumThreads * 4; fAggNumBuckets = fAggNumThreads * 4;
else else
fAggNumBuckets = fConfig->uFromText(nb); fAggNumBuckets = fConfig->uFromText(nb);
nr = fConfig->getConfig("RowAggregation", "RowAggrRowGroupsPerThread"); nr = fConfig->getConfig("RowAggregation", "RowAggrRowGroupsPerThread");
if (nr.empty()) if (nr.empty())

4
dbcon/joblist/resourcemanager.h
View File

@ -261,7 +261,8 @@ public:
} }
uint32_t getJlMaxOutstandingRequests() const uint32_t getJlMaxOutstandingRequests() const
{ {
return getUintVal(fJobListStr, "MaxOutstandingRequests", defaultMaxOutstandingRequests); return fJlMaxOutstandingRequests;
//getUintVal(fJobListStr, "MaxOutstandingRequests", defaultMaxOutstandingRequests);
} }
uint32_t getJlJoinerChunkSize() const uint32_t getJlJoinerChunkSize() const
{ {
@ -577,6 +578,7 @@ private:
uint32_t fJlProcessorThreadsPerScan; uint32_t fJlProcessorThreadsPerScan;
uint32_t fJlNumScanReceiveThreads; uint32_t fJlNumScanReceiveThreads;
uint8_t fTwNumThreads; uint8_t fTwNumThreads;
uint32_t fJlMaxOutstandingRequests;
/* old HJ support */ /* old HJ support */
ResourceDistributor fHJUmMaxMemorySmallSideDistributor; ResourceDistributor fHJUmMaxMemorySmallSideDistributor;

8
oam/etc/Columnstore.xml
View File

@ -486,8 +486,8 @@
<HashJoin> <HashJoin>
<MaxBuckets>128</MaxBuckets> <MaxBuckets>128</MaxBuckets>
<MaxElems>128K</MaxElems> <!-- 128 buckets * 128K * 16 = 256 MB --> <MaxElems>128K</MaxElems> <!-- 128 buckets * 128K * 16 = 256 MB -->
<FifoSizeLargeSide>128</FifoSizeLargeSide> <FifoSizeLargeSide>64</FifoSizeLargeSide>
<PmMaxMemorySmallSide>64M</PmMaxMemorySmallSide><!-- divide by 48 to getapproximate row count --> <PmMaxMemorySmallSide>1G</PmMaxMemorySmallSide><!-- divide by 48 to getapproximate row count -->
<TotalUmMemory>50%</TotalUmMemory> <TotalUmMemory>50%</TotalUmMemory>
<TotalPmUmMemory>10%</TotalPmUmMemory> <TotalPmUmMemory>10%</TotalPmUmMemory>
<CPUniqueLimit>100</CPUniqueLimit> <CPUniqueLimit>100</CPUniqueLimit>
@ -500,7 +500,7 @@
</HashJoin> </HashJoin>
<JobList> <JobList>
<FlushInterval>16K</FlushInterval> <FlushInterval>16K</FlushInterval>
<FifoSize>32</FifoSize> <FifoSize>16</FifoSize>
<RequestSize>1</RequestSize> <!-- Number of extents per request, should be <RequestSize>1</RequestSize> <!-- Number of extents per request, should be
less than MaxOutstandingRequests. Otherwise, default value 1 is used. --> less than MaxOutstandingRequests. Otherwise, default value 1 is used. -->
<!-- ProcessorThreadsPerScan is the number of jobs issued to process <!-- ProcessorThreadsPerScan is the number of jobs issued to process
@ -510,7 +510,7 @@
ProcessorThreadsPerScan * MaxOutstandingRequests should be at least ProcessorThreadsPerScan * MaxOutstandingRequests should be at least
as many threads are available across all PMs. --> as many threads are available across all PMs. -->
<!-- <ProcessorThreadsPerScan>16</ProcessorThreadsPerScan> --> <!-- <ProcessorThreadsPerScan>16</ProcessorThreadsPerScan> -->
<MaxOutstandingRequests>20</MaxOutstandingRequests> <!-- <MaxOutstandingRequests>20</MaxOutstandingRequests> -->
<ThreadPoolSize>100</ThreadPoolSize> <ThreadPoolSize>100</ThreadPoolSize>
</JobList> </JobList>
<TupleWSDL> <TupleWSDL>

8
oam/etc/Columnstore.xml.singleserver
View File

@ -477,8 +477,8 @@
<HashJoin> <HashJoin>
<MaxBuckets>128</MaxBuckets> <MaxBuckets>128</MaxBuckets>
<MaxElems>128K</MaxElems> <!-- 128 buckets * 128K * 16 = 256 MB --> <MaxElems>128K</MaxElems> <!-- 128 buckets * 128K * 16 = 256 MB -->
<FifoSizeLargeSide>128</FifoSizeLargeSide> <FifoSizeLargeSide>64</FifoSizeLargeSide>
<PmMaxMemorySmallSide>64M</PmMaxMemorySmallSide><!-- divide by 48 to get element count --> <PmMaxMemorySmallSide>1G</PmMaxMemorySmallSide><!-- divide by 48 to get element count -->
<TotalUmMemory>25%</TotalUmMemory> <TotalUmMemory>25%</TotalUmMemory>
<TotalPmUmMemory>10%</TotalPmUmMemory> <TotalPmUmMemory>10%</TotalPmUmMemory>
<CPUniqueLimit>100</CPUniqueLimit> <CPUniqueLimit>100</CPUniqueLimit>
@ -491,7 +491,7 @@
</HashJoin> </HashJoin>
<JobList> <JobList>
<FlushInterval>16K</FlushInterval> <FlushInterval>16K</FlushInterval>
<FifoSize>32</FifoSize> <FifoSize>16</FifoSize>
<RequestSize>1</RequestSize> <!-- Number of extents per request, should be <RequestSize>1</RequestSize> <!-- Number of extents per request, should be
less than MaxOutstandingRequests. Otherwise, default value 1 is used. --> less than MaxOutstandingRequests. Otherwise, default value 1 is used. -->
<!-- ProcessorThreadsPerScan is the number of jobs issued to process <!-- ProcessorThreadsPerScan is the number of jobs issued to process
@ -501,7 +501,7 @@
ProcessorThreadsPerScan * MaxOutstandingRequests should be at least ProcessorThreadsPerScan * MaxOutstandingRequests should be at least
as many threads are available across all PMs. --> as many threads are available across all PMs. -->
<!-- <ProcessorThreadsPerScan>16</ProcessorThreadsPerScan> --> <!-- <ProcessorThreadsPerScan>16</ProcessorThreadsPerScan> -->
<MaxOutstandingRequests>20</MaxOutstandingRequests> <!-- <MaxOutstandingRequests>20</MaxOutstandingRequests> -->
<ThreadPoolSize>100</ThreadPoolSize> <ThreadPoolSize>100</ThreadPoolSize>
</JobList> </JobList>
<TupleWSDL> <TupleWSDL>

2
tools/configMgt/autoConfigure.cpp
View File

@ -2122,7 +2122,6 @@ int main(int argc, char* argv[])
{ {
ColScanReadAheadBlocks = sysConfigOld->getConfig("PrimitiveServers", "ColScanReadAheadBlocks"); ColScanReadAheadBlocks = sysConfigOld->getConfig("PrimitiveServers", "ColScanReadAheadBlocks");
PrefetchThreshold = sysConfigOld->getConfig("PrimitiveServers", "PrefetchThreshold"); PrefetchThreshold = sysConfigOld->getConfig("PrimitiveServers", "PrefetchThreshold");
MaxOutstandingRequests = sysConfigOld->getConfig("JobList", "MaxOutstandingRequests");
PmMaxMemorySmallSide = sysConfigOld->getConfig("HashJoin", "PmMaxMemorySmallSide"); PmMaxMemorySmallSide = sysConfigOld->getConfig("HashJoin", "PmMaxMemorySmallSide");
ThreadPoolSize = sysConfigOld->getConfig("JobList", "ThreadPoolSize"); ThreadPoolSize = sysConfigOld->getConfig("JobList", "ThreadPoolSize");
} }
@ -2133,7 +2132,6 @@ int main(int argc, char* argv[])
{ {
sysConfigNew->setConfig("PrimitiveServers", "ColScanReadAheadBlocks", ColScanReadAheadBlocks); sysConfigNew->setConfig("PrimitiveServers", "ColScanReadAheadBlocks", ColScanReadAheadBlocks);
sysConfigNew->setConfig("PrimitiveServers", "PrefetchThreshold", PrefetchThreshold); sysConfigNew->setConfig("PrimitiveServers", "PrefetchThreshold", PrefetchThreshold);
sysConfigNew->setConfig("JobList", "MaxOutstandingRequests", MaxOutstandingRequests);
sysConfigNew->setConfig("HashJoin", "PmMaxMemorySmallSide", PmMaxMemorySmallSide); sysConfigNew->setConfig("HashJoin", "PmMaxMemorySmallSide", PmMaxMemorySmallSide);
sysConfigNew->setConfig("JobList", "ThreadPoolSize", ThreadPoolSize); sysConfigNew->setConfig("JobList", "ThreadPoolSize", ThreadPoolSize);
} }