mirror of
https://github.com/mariadb-corporation/mariadb-columnstore-engine.git
synced 2025-04-18 21:44:02 +03:00
6445f4dff3
* feat(joblist,runtime): this is the first part of the execution model that produces a workload that can be predicted for a given query. - forces to UM join converter to use a value from a configuration - replaces a constant used to control a number of outstanding requests with a value depends on column width - modifies related Columnstore.xml values
252 lines
10 KiB
XML
252 lines
10 KiB
XML
<Columnstore Version="V1.0.0">
|
|
<!--
|
|
WARNING: Do not make changes to this file unless directed to do so by
|
|
MariaDB service engineers. Incorrect settings can render your system
|
|
unusable and will require a service call to correct.
|
|
-->
|
|
<ExeMgr1>
|
|
<IPAddr>127.0.0.1</IPAddr>
|
|
<Port>8601</Port>
|
|
<Module>unassigned</Module>
|
|
</ExeMgr1>
|
|
<pm1_WriteEngineServer>
|
|
<IPAddr>127.0.0.1</IPAddr>
|
|
<Port>8630</Port>
|
|
</pm1_WriteEngineServer>
|
|
<DDLProc>
|
|
<IPAddr>127.0.0.1</IPAddr>
|
|
<Port>8612</Port>
|
|
</DDLProc>
|
|
<DMLProc>
|
|
<IPAddr>127.0.0.1</IPAddr>
|
|
<Port>8614</Port>
|
|
</DMLProc>
|
|
<BatchInsert>
|
|
<RowsPerBatch>10000</RowsPerBatch>
|
|
</BatchInsert>
|
|
<PrimitiveServers>
|
|
<Count>1</Count>
|
|
<ConnectionsPerPrimProc>2</ConnectionsPerPrimProc>
|
|
<ProcessorThreshold>128</ProcessorThreshold>
|
|
<ProcessorQueueSize>10K</ProcessorQueueSize> <!-- minimum of extent size 8192 -->
|
|
<DebugLevel>0</DebugLevel>
|
|
<ColScanBufferSizeBlocks>512</ColScanBufferSizeBlocks>
|
|
<ColScanReadAheadBlocks>512</ColScanReadAheadBlocks> <!-- s/b factor of extent size 8192 -->
|
|
<!-- <BPPCount>16</BPPCount> --> <!-- Default num cores * 2. A cap on the number of simultaneous primitives per jobstep -->
|
|
<PrefetchThreshold>1</PrefetchThreshold>
|
|
<PTTrace>0</PTTrace>
|
|
<RotatingDestination>n</RotatingDestination> <!-- Iterate thru UM ports; set to 'n' if UM/PM on same server -->
|
|
<!-- <HighPriorityPercentage>60</HighPriorityPercentage> -->
|
|
<!-- <MediumPriorityPercentage>30</MediumPriorityPercentage> -->
|
|
<!-- <LowPriorityPercentage>10</LowPriorityPercentage> -->
|
|
<DirectIO>y</DirectIO>
|
|
<HighPriorityPercentage/>
|
|
<MediumPriorityPercentage/>
|
|
<LowPriorityPercentage/>
|
|
</PrimitiveServers>
|
|
<PMS1>
|
|
<IPAddr>127.0.0.1</IPAddr>
|
|
<Port>8620</Port>
|
|
</PMS1>
|
|
<SystemConfig>
|
|
<SystemName>columnstore-1</SystemName>
|
|
<ParentOAMModuleName>pm1</ParentOAMModuleName>
|
|
<PrimaryUMModuleName>pm1</PrimaryUMModuleName>
|
|
<!-- Warning: Do not change this value once database is built -->
|
|
<DBRootCount>1</DBRootCount>
|
|
<DBRoot1>/var/lib/columnstore/data1</DBRoot1>
|
|
<DBRMRoot>/var/lib/columnstore/data1/systemFiles/dbrm/BRM_saves</DBRMRoot>
|
|
<TableLockSaveFile>/var/lib/columnstore/data1/systemFiles/dbrm/tablelocks</TableLockSaveFile>
|
|
<DBRMTimeOut>15</DBRMTimeOut> <!-- in seconds -->
|
|
<DBRMSnapshotInterval>100000</DBRMSnapshotInterval>
|
|
<WaitPeriod>10</WaitPeriod> <!-- in seconds -->
|
|
<MemoryCheckPercent>95</MemoryCheckPercent> <!-- Max real memory to limit growth of buffers to -->
|
|
<DataFileLog>OFF</DataFileLog>
|
|
<!-- enable if you want to limit how much memory may be used for hdfs read/write memory buffers.
|
|
<hdfsRdwrBufferMaxSize>8G</hdfsRdwrBufferMaxSize>
|
|
-->
|
|
<hdfsRdwrScratch>/rdwrscratch</hdfsRdwrScratch> <!-- Do not set to an hdfs file path -->
|
|
<!-- Be careful modifying SystemTempFileDir! On start, ExeMgr deletes
|
|
the entire subdirectories "joins" & "aggregates" and recreates it to make sure no
|
|
files are left behind. -->
|
|
<SystemTempFileDir>/tmp/columnstore_tmp_files</SystemTempFileDir>
|
|
</SystemConfig>
|
|
<SystemModuleConfig>
|
|
<ModuleType2>um</ModuleType2>
|
|
<ModuleDesc2>User Module</ModuleDesc2>
|
|
<ModuleCount2>0</ModuleCount2>
|
|
<ModuleIPAddr1-1-2>0.0.0.0</ModuleIPAddr1-1-2>
|
|
<ModuleHostName1-1-2>unassigned</ModuleHostName1-1-2>
|
|
<ModuleDisableState1-2>ENABLED</ModuleDisableState1-2>
|
|
<ModuleCPUCriticalThreshold2>0</ModuleCPUCriticalThreshold2>
|
|
<ModuleCPUMajorThreshold2>0</ModuleCPUMajorThreshold2>
|
|
<ModuleCPUMinorThreshold2>0</ModuleCPUMinorThreshold2>
|
|
<ModuleCPUMinorClearThreshold2>0</ModuleCPUMinorClearThreshold2>
|
|
<ModuleDiskCriticalThreshold2>90</ModuleDiskCriticalThreshold2>
|
|
<ModuleDiskMajorThreshold2>80</ModuleDiskMajorThreshold2>
|
|
<ModuleDiskMinorThreshold2>70</ModuleDiskMinorThreshold2>
|
|
<ModuleMemCriticalThreshold2>90</ModuleMemCriticalThreshold2>
|
|
<ModuleMemMajorThreshold2>0</ModuleMemMajorThreshold2>
|
|
<ModuleMemMinorThreshold2>0</ModuleMemMinorThreshold2>
|
|
<ModuleSwapCriticalThreshold2>90</ModuleSwapCriticalThreshold2>
|
|
<ModuleSwapMajorThreshold2>80</ModuleSwapMajorThreshold2>
|
|
<ModuleSwapMinorThreshold2>70</ModuleSwapMinorThreshold2>
|
|
<ModuleDiskMonitorFileSystem1-2>/</ModuleDiskMonitorFileSystem1-2>
|
|
<ModuleDBRootCount1-2>unassigned</ModuleDBRootCount1-2>
|
|
<ModuleDBRootID1-1-2>unassigned</ModuleDBRootID1-1-2>
|
|
<ModuleType3>pm</ModuleType3>
|
|
<ModuleDesc3>Performance Module</ModuleDesc3>
|
|
<ModuleCount3>1</ModuleCount3>
|
|
<ModuleIPAddr1-1-3>127.0.0.1</ModuleIPAddr1-1-3>
|
|
<ModuleHostName1-1-3>localhost</ModuleHostName1-1-3>
|
|
<ModuleDisableState1-3>ENABLED</ModuleDisableState1-3>
|
|
<ModuleCPUCriticalThreshold3>0</ModuleCPUCriticalThreshold3>
|
|
<ModuleCPUMajorThreshold3>0</ModuleCPUMajorThreshold3>
|
|
<ModuleCPUMinorThreshold3>0</ModuleCPUMinorThreshold3>
|
|
<ModuleCPUMinorClearThreshold3>0</ModuleCPUMinorClearThreshold3>
|
|
<ModuleDiskCriticalThreshold3>90</ModuleDiskCriticalThreshold3>
|
|
<ModuleDiskMajorThreshold3>80</ModuleDiskMajorThreshold3>
|
|
<ModuleDiskMinorThreshold3>70</ModuleDiskMinorThreshold3>
|
|
<ModuleMemCriticalThreshold3>90</ModuleMemCriticalThreshold3>
|
|
<ModuleMemMajorThreshold3>0</ModuleMemMajorThreshold3>
|
|
<ModuleMemMinorThreshold3>0</ModuleMemMinorThreshold3>
|
|
<ModuleSwapCriticalThreshold3>90</ModuleSwapCriticalThreshold3>
|
|
<ModuleSwapMajorThreshold3>80</ModuleSwapMajorThreshold3>
|
|
<ModuleSwapMinorThreshold3>70</ModuleSwapMinorThreshold3>
|
|
<ModuleDiskMonitorFileSystem1-3>/</ModuleDiskMonitorFileSystem1-3>
|
|
<ModuleDBRootCount1-3>1</ModuleDBRootCount1-3>
|
|
<ModuleDBRootID1-1-3>1</ModuleDBRootID1-1-3>
|
|
</SystemModuleConfig>
|
|
<SessionManager>
|
|
<MaxConcurrentTransactions>1000</MaxConcurrentTransactions>
|
|
<TxnIDFile>/var/lib/columnstore/data1/systemFiles/dbrm/SMTxnID</TxnIDFile>
|
|
</SessionManager>
|
|
<VersionBuffer>
|
|
<!-- VersionBufferFileSize must be a multiple of 8192.
|
|
One version buffer file will be put on each DB root. -->
|
|
<VersionBufferFileSize>1GB</VersionBufferFileSize>
|
|
</VersionBuffer>
|
|
<OIDManager>
|
|
<!-- Do not change this file after database built -->
|
|
<OIDBitmapFile>/var/lib/columnstore/data1/systemFiles/dbrm/oidbitmap</OIDBitmapFile>
|
|
<!-- Do not change this value after database built -->
|
|
<FirstOID>3000</FirstOID>
|
|
</OIDManager>
|
|
<WriteEngine>
|
|
<BulkRoot>/var/log/mariadb/columnstore/data/bulk</BulkRoot>
|
|
<BulkRollbackDir>/var/lib/columnstore/data1/systemFiles/bulkRollback</BulkRollbackDir>
|
|
<MaxFileSystemDiskUsagePct>98</MaxFileSystemDiskUsagePct>
|
|
<CompressedPaddingBlocks>1</CompressedPaddingBlocks> <!-- Number of blocks used to pad compressed chunks -->
|
|
<FastDelete>n</FastDelete>
|
|
</WriteEngine>
|
|
<DBRM_Controller>
|
|
<NumWorkers>1</NumWorkers>
|
|
<IPAddr>127.0.0.1</IPAddr>
|
|
<Port>8616</Port>
|
|
</DBRM_Controller>
|
|
<!-- Worker Port: 8700 - 8720 is reserved to support External Modules-->
|
|
<DBRM_Worker1>
|
|
<IPAddr>127.0.0.1</IPAddr>
|
|
<Port>8700</Port>
|
|
<Module>pm1</Module>
|
|
</DBRM_Worker1>
|
|
<DBBC>
|
|
<!-- The percentage of RAM to use for the disk block cache. Defaults to 70% -->
|
|
<!-- Alternatively, this can be specified in absolute terms using
|
|
the suffixes 'm' or 'g' to denote size in megabytes or gigabytes.-->
|
|
<!-- <NumBlocksPct>95</NumBlocksPct> -->
|
|
<!-- <NumThreads>16</NumThreads> --> <!-- 1-256. Default is 16. -->
|
|
<NumCaches>1</NumCaches><!-- # of parallel caches to instantiate -->
|
|
<IOMTracing>0</IOMTracing>
|
|
<BRPTracing>0</BRPTracing>
|
|
<ReportFrequency>65536</ReportFrequency>
|
|
<MaxOpenFiles>2K</MaxOpenFiles>
|
|
<DecreaseOpenFilesCount>200</DecreaseOpenFilesCount>
|
|
<FDCacheTrace>0</FDCacheTrace>
|
|
<NumBlocksPct>50</NumBlocksPct>
|
|
</DBBC>
|
|
<Installation>
|
|
<ServerTypeInstall>2</ServerTypeInstall>
|
|
<PMwithUM>y</PMwithUM>
|
|
<MySQLRep>n</MySQLRep>
|
|
<DBRootStorageType>internal</DBRootStorageType>
|
|
<UMStorageType>internal</UMStorageType>
|
|
<ProfileFile>/etc/profile.d/columnstoreAlias.sh</ProfileFile>
|
|
<DataRedundancyNetworkType/>
|
|
</Installation>
|
|
<ExtentMap>
|
|
<!--
|
|
WARNING: these can only be changed on an empty system. Once any object has been allocated
|
|
it cannot be changed!. Extent size is 8M rows.
|
|
-->
|
|
<FilesPerColumnPartition>4</FilesPerColumnPartition> <!-- should be multiple of DBRootCount -->
|
|
<BRM_UID>0x0</BRM_UID>
|
|
</ExtentMap>
|
|
<HashJoin>
|
|
<MaxBuckets>128</MaxBuckets>
|
|
<MaxElems>128K</MaxElems> <!-- 128 buckets * 128K * 16 = 256 MB -->
|
|
<PmMaxMemorySmallSide>1G</PmMaxMemorySmallSide>
|
|
<TotalUmMemory>25%</TotalUmMemory>
|
|
<CPUniqueLimit>100</CPUniqueLimit>
|
|
<AllowDiskBasedJoin>N</AllowDiskBasedJoin>
|
|
<TempFileCompression>Y</TempFileCompression>
|
|
<TempFileCompressionType>Snappy</TempFileCompressionType> <!-- LZ4, Snappy -->
|
|
</HashJoin>
|
|
<JobList>
|
|
<FlushInterval>16K</FlushInterval>
|
|
<FifoSize>16</FifoSize>
|
|
<RequestSize>1</RequestSize> <!-- Number of extents per request, should be
|
|
less than MaxOutstandingRequests. Otherwise, default value 1 is used. -->
|
|
<!-- ProcessorThreadsPerScan is the number of jobs issued to process
|
|
each extent. The default is 16. MaxOutstandingRequests is the size of
|
|
the window of work in terms of extents. A value of 20 means there
|
|
is 20 extents worth of work for the PMs to process at any given time.
|
|
ProcessorThreadsPerScan * MaxOutstandingRequests should be at least
|
|
as many threads are available across all PMs. -->
|
|
<!-- <ProcessorThreadsPerScan>16</ProcessorThreadsPerScan> -->
|
|
<!-- MaxOutstandingRequests is a windows size for outstanding PrimitiveJobs messages per query step.
|
|
A bigger number increases PrimProc workload. -->
|
|
<MaxOutstandingRequests>20000</MaxOutstandingRequests>
|
|
<ThreadPoolSize>1000</ThreadPoolSize>
|
|
</JobList>
|
|
<RowAggregation>
|
|
<!-- <RowAggrThreads>4</RowAggrThreads> --> <!-- Default value is the number of cores -->
|
|
<!-- <RowAggrBuckets>32</RowAggrBuckets> --> <!-- Default value is number of cores * 4 -->
|
|
<!-- <RowAggrRowGroupsPerThread>20</RowAggrRowGroupsPerThread> --> <!-- Default value is 20 -->
|
|
<AllowDiskBasedAggregation>N</AllowDiskBasedAggregation>
|
|
</RowAggregation>
|
|
<CrossEngineSupport>
|
|
<Host>127.0.0.1</Host>
|
|
<Port>3306</Port>
|
|
<User>root</User>
|
|
<Password/>
|
|
<TLSCA/>
|
|
<TLSClientCert/>
|
|
<TLSClientKey/>
|
|
</CrossEngineSupport>
|
|
<QueryStats>
|
|
<Enabled>N</Enabled>
|
|
</QueryStats>
|
|
<UserPriority>
|
|
<Enabled>N</Enabled>
|
|
</UserPriority>
|
|
<NetworkCompression>
|
|
<Enabled>Y</Enabled>
|
|
<NetworkCompressionType>Snappy</NetworkCompressionType> <!-- LZ4, Snappy -->
|
|
</NetworkCompression>
|
|
<QueryTele>
|
|
<Host>127.0.0.1</Host>
|
|
<Port>0</Port>
|
|
</QueryTele>
|
|
<StorageManager>
|
|
<MaxSockets>30</MaxSockets>
|
|
<Enabled>N</Enabled>
|
|
</StorageManager>
|
|
<DataRedundancyConfig>
|
|
<DBRoot1PMs/>
|
|
</DataRedundancyConfig>
|
|
<Rewrites>
|
|
<CommonLeafConjunctionsToTop>Y</CommonLeafConjunctionsToTop>
|
|
</Rewrites>
|
|
</Columnstore>
|