database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

1. Set 1M as the threshold on the number of records to flush the cache.

Browse Source 
2. Set 100k as the batch size when flushing records into ColumnStore, i.e.,
a flush of 1M records will be performed in 10 batches, each being 100k.

3. For INSERT ... SELECT on the cache, use the default insertion method of cpimport.
This commit is contained in:
Gagan Goel
2020-07-01 20:13:04 -04:00
parent f5a8d228a0
commit 86fb66365c
4 changed files with 25 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
dbcon/mysql/ha_mcs_dml.cpp
View File

@ -77,6 +77,7 @@ using namespace joblist;
namespace
{
#define BATCH_INSERT_GROUP_ROWS_FOR_CACHE 100000
uint64_t fBatchInsertGroupRows = 0; // ResourceManager::instance()->getRowsPerBatch();
// HDFS is never used nowadays, so don't bother
bool useHdfs = false; // ResourceManager::instance()->useHdfs();
@ -594,13 +595,17 @@ int ha_mcs_impl_write_row_(const uchar* buf, TABLE* table, cal_connection_info&
}
if (fBatchInsertGroupRows == 0)
{
fBatchInsertGroupRows = ResourceManager::instance()->getRowsPerBatch();
}
//timer.stop( "buildValueList");
if ( ci.singleInsert // Single insert
|| (( ci.bulkInsertRows > 0 ) && (( ( ci.rowsHaveInserted + size) >= ci.bulkInsertRows ) || ( size >= fBatchInsertGroupRows )) )
//Insert with mutilple value case: processed batch by batch. Last batch is sent also.
|| (( ci.bulkInsertRows == 0 ) && ( size >= fBatchInsertGroupRows ) ) ) // Load data in file is processed batch by batch
|| (( ci.bulkInsertRows > 0 ) && (( ( ci.rowsHaveInserted + size) >= ci.bulkInsertRows )
|| ( (!ci.isCacheInsert && size >= fBatchInsertGroupRows) || (ci.isCacheInsert && size >= BATCH_INSERT_GROUP_ROWS_FOR_CACHE) )) )
//Insert with mutilple value case: processed batch by batch. Last batch is sent also.
|| (( ci.bulkInsertRows == 0 ) && ( (!ci.isCacheInsert && size >= fBatchInsertGroupRows)
|| (ci.isCacheInsert && size >= BATCH_INSERT_GROUP_ROWS_FOR_CACHE) ) ) ) // Load data in file is processed batch by batch
{
//timer.start( "DMLProc takes");
//cout <<" sending a batch to DMLProc ... The size is " << size << " the current bulkInsertRows = " << ci.bulkInsertRows << endl;