database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

MCOL 3585 update some defaults in config file

Browse Source
This commit is contained in:
jmrojas2332
2019-11-21 06:08:49 +00:00
parent dbb9b21d26
commit e712df5d0b
5 changed files with 37 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
dbcon/joblist/resourcemanager.cpp
View File

@ -87,7 +87,8 @@ ResourceManager::ResourceManager(bool runningInExeMgr) :
0),
fHJPmMaxMemorySmallSideSessionMap(
getUintVal(fHashJoinStr, "PmMaxMemorySmallSide", defaultHJPmMaxMemorySmallSide)),
isExeMgr(runningInExeMgr)
isExeMgr(runningInExeMgr),
fJlMaxOutstandingRequests(defaultMaxOutstandingRequests)
{
int temp;
int configNumCores = -1;
@ -127,9 +128,27 @@ ResourceManager::ResourceManager(bool runningInExeMgr) :
temp = getIntVal(fJobListStr, "ProcessorThreadsPerScan", -1);
if (temp > 0)
fJlProcessorThreadsPerScan = temp;
temp = getIntVal(fJobListStr, "MaxOutstandingRequests", -1);
if (temp > 0)
fJlMaxOutstandingRequests = temp;
else
{
oam::Oam oam;
oam::ModuleTypeConfig moduletypeconfig;
oam.getSystemConfig("pm", moduletypeconfig);
const auto temp = moduletypeconfig.ModuleCount * fNumCores * 4 / fJlProcessorThreadsPerScan;
if (temp > defaultMaxOutstandingRequests)
{
fJlMaxOutstandingRequests = temp;
}
}
temp = getIntVal(fJobListStr, "NumScanReceiveThreads", -1);
if (temp > 0)
@ -140,7 +159,7 @@ ResourceManager::ResourceManager(bool runningInExeMgr) :
if (temp > 0)
fTwNumThreads = temp;
pmJoinMemLimit = getIntVal(fHashJoinStr, "PmMaxMemorySmallSide",
pmJoinMemLimit = getUintVal(fHashJoinStr, "PmMaxMemorySmallSide",
defaultHJPmMaxMemorySmallSide);
// Need to use different limits if this instance isn't running on the UM,
@ -200,10 +219,12 @@ ResourceManager::ResourceManager(bool runningInExeMgr) :
nt = fConfig->getConfig("RowAggregation", "RowAggrThreads");
if (nt.empty())
if ( numCores() < 4 )
{
if ( numCores() > 0 )
fAggNumThreads = numCores();
else
fAggNumThreads = 4;
fAggNumThreads = 1;
}
else
fAggNumThreads = fConfig->uFromText(nt);

4
dbcon/joblist/resourcemanager.h
View File

@ -261,7 +261,8 @@ public:
}
uint32_t getJlMaxOutstandingRequests() const
{
return getUintVal(fJobListStr, "MaxOutstandingRequests", defaultMaxOutstandingRequests);
return fJlMaxOutstandingRequests;
//getUintVal(fJobListStr, "MaxOutstandingRequests", defaultMaxOutstandingRequests);
}
uint32_t getJlJoinerChunkSize() const
{
@ -577,6 +578,7 @@ private:
uint32_t fJlProcessorThreadsPerScan;
uint32_t fJlNumScanReceiveThreads;
uint8_t fTwNumThreads;
uint32_t fJlMaxOutstandingRequests;
/* old HJ support */
ResourceDistributor fHJUmMaxMemorySmallSideDistributor;

8
oam/etc/Columnstore.xml
View File

@ -486,8 +486,8 @@
<HashJoin>
<MaxBuckets>128</MaxBuckets>
<MaxElems>128K</MaxElems> <!-- 128 buckets * 128K * 16 = 256 MB -->
<FifoSizeLargeSide>128</FifoSizeLargeSide>
<PmMaxMemorySmallSide>64M</PmMaxMemorySmallSide><!-- divide by 48 to getapproximate row count -->
<FifoSizeLargeSide>64</FifoSizeLargeSide>
<PmMaxMemorySmallSide>1G</PmMaxMemorySmallSide><!-- divide by 48 to getapproximate row count -->
<TotalUmMemory>50%</TotalUmMemory>
<TotalPmUmMemory>10%</TotalPmUmMemory>
<CPUniqueLimit>100</CPUniqueLimit>
@ -500,7 +500,7 @@
</HashJoin>
<JobList>
<FlushInterval>16K</FlushInterval>
<FifoSize>32</FifoSize>
<FifoSize>16</FifoSize>
<RequestSize>1</RequestSize> <!-- Number of extents per request, should be
less than MaxOutstandingRequests. Otherwise, default value 1 is used. -->
<!-- ProcessorThreadsPerScan is the number of jobs issued to process
@ -510,7 +510,7 @@
ProcessorThreadsPerScan * MaxOutstandingRequests should be at least
as many threads are available across all PMs. -->
<!-- <ProcessorThreadsPerScan>16</ProcessorThreadsPerScan> -->
<MaxOutstandingRequests>20</MaxOutstandingRequests>
<!-- <MaxOutstandingRequests>20</MaxOutstandingRequests> -->
<ThreadPoolSize>100</ThreadPoolSize>
</JobList>
<TupleWSDL>

8
oam/etc/Columnstore.xml.singleserver
View File

@ -477,8 +477,8 @@
<HashJoin>
<MaxBuckets>128</MaxBuckets>
<MaxElems>128K</MaxElems> <!-- 128 buckets * 128K * 16 = 256 MB -->
<FifoSizeLargeSide>128</FifoSizeLargeSide>
<PmMaxMemorySmallSide>64M</PmMaxMemorySmallSide><!-- divide by 48 to get element count -->
<FifoSizeLargeSide>64</FifoSizeLargeSide>
<PmMaxMemorySmallSide>1G</PmMaxMemorySmallSide><!-- divide by 48 to get element count -->
<TotalUmMemory>25%</TotalUmMemory>
<TotalPmUmMemory>10%</TotalPmUmMemory>
<CPUniqueLimit>100</CPUniqueLimit>
@ -491,7 +491,7 @@
</HashJoin>
<JobList>
<FlushInterval>16K</FlushInterval>
<FifoSize>32</FifoSize>
<FifoSize>16</FifoSize>
<RequestSize>1</RequestSize> <!-- Number of extents per request, should be
less than MaxOutstandingRequests. Otherwise, default value 1 is used. -->
<!-- ProcessorThreadsPerScan is the number of jobs issued to process
@ -501,7 +501,7 @@
ProcessorThreadsPerScan * MaxOutstandingRequests should be at least
as many threads are available across all PMs. -->
<!-- <ProcessorThreadsPerScan>16</ProcessorThreadsPerScan> -->
<MaxOutstandingRequests>20</MaxOutstandingRequests>
<!-- <MaxOutstandingRequests>20</MaxOutstandingRequests> -->
<ThreadPoolSize>100</ThreadPoolSize>
</JobList>
<TupleWSDL>

2
tools/configMgt/autoConfigure.cpp
View File

@ -2122,7 +2122,6 @@ int main(int argc, char* argv[])
{
ColScanReadAheadBlocks = sysConfigOld->getConfig("PrimitiveServers", "ColScanReadAheadBlocks");
PrefetchThreshold = sysConfigOld->getConfig("PrimitiveServers", "PrefetchThreshold");
MaxOutstandingRequests = sysConfigOld->getConfig("JobList", "MaxOutstandingRequests");
PmMaxMemorySmallSide = sysConfigOld->getConfig("HashJoin", "PmMaxMemorySmallSide");
ThreadPoolSize = sysConfigOld->getConfig("JobList", "ThreadPoolSize");
}
@ -2133,7 +2132,6 @@ int main(int argc, char* argv[])
{
sysConfigNew->setConfig("PrimitiveServers", "ColScanReadAheadBlocks", ColScanReadAheadBlocks);
sysConfigNew->setConfig("PrimitiveServers", "PrefetchThreshold", PrefetchThreshold);
sysConfigNew->setConfig("JobList", "MaxOutstandingRequests", MaxOutstandingRequests);
sysConfigNew->setConfig("HashJoin", "PmMaxMemorySmallSide", PmMaxMemorySmallSide);
sysConfigNew->setConfig("JobList", "ThreadPoolSize", ThreadPoolSize);
}