database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

Improve batch inserts.

Browse Source 
1) Instead of making dbrm calls to writeVBEntry() per block,
     we make these calls per batch. This can have non-trivial
     reductions in the overhead of these calls if the batch size
     is large.

  2) In dmlproc, do not deserialize the whole insertpackage, which
     consists of the complete record set per column, which would be
     wasteful as we only need some metadata fields from insertpackage
     here. This is only done for batch inserts at the moment, this
     should also be applied to single inserts.
This commit is contained in:
Gagan Goel
2020-05-10 19:38:06 -04:00
parent 04fdacb927
commit d4d0ebdf5d
13 changed files with 306 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
dbcon/dmlpackage/dmltable.cpp
View File

@ -72,6 +72,28 @@ int DMLTable::read(messageqcpp::ByteStream& bytestream)
return retval;
}
void DMLTable::readMetaData(messageqcpp::ByteStream& bytestream)
{
// read the table name
bytestream >> fName;
// read the schema name
bytestream >> fSchema;
}
void DMLTable::readRowData(messageqcpp::ByteStream& bytestream)
{
messageqcpp::ByteStream::quadbyte rowNum;
bytestream >> rowNum;
for (unsigned int i = 0; i < rowNum; i++)
{
Row* aRow = new Row();
aRow->read(bytestream);
fRows.push_back(aRow);
}
}
int DMLTable::write(messageqcpp::ByteStream& bytestream)
{
int retval = 1;

14
dbcon/dmlpackage/dmltable.h
View File

@ -91,6 +91,20 @@ public:
int read(messageqcpp::ByteStream& bytestream);
/** @brief read a DMLTable metadata from a ByteStream
*
* @param bytestream the ByteStream to read from
*/
void readMetaData(messageqcpp::ByteStream& bytestream);
/** @brief read a DMLTable row data from a ByteStream
*
* @param bytestream the ByteStream to read from
*/
void readRowData(messageqcpp::ByteStream& bytestream);
/** @brief write a DMLTable to a ByteStream
*
* @param bytestream the ByteStream to write to

53
dbcon/dmlpackage/insertdmlpackage.cpp
View File

@ -66,16 +66,16 @@ int InsertDMLPackage::write(messageqcpp::ByteStream& bytestream)
bytestream << (uint8_t)fLogging;
bytestream << (uint8_t)fLogending;
if (fTable != 0)
{
retval = fTable->write(bytestream);
}
bytestream << fTableOid;
bytestream << static_cast<const messageqcpp::ByteStream::byte>(fIsInsertSelect);
bytestream << static_cast<const messageqcpp::ByteStream::byte>(fIsBatchInsert);
bytestream << static_cast<const messageqcpp::ByteStream::byte>(fIsAutocommitOn);
if (fTable != 0)
{
retval = fTable->write(bytestream);
}
return retval;
}
@ -100,15 +100,50 @@ int InsertDMLPackage::read(messageqcpp::ByteStream& bytestream)
bytestream >> logending;
fLogending = (logending != 0);
bytestream >> fTableOid;
bytestream >> reinterpret_cast<messageqcpp::ByteStream::byte&>(fIsInsertSelect);
bytestream >> reinterpret_cast<messageqcpp::ByteStream::byte&>(fIsBatchInsert);
bytestream >> reinterpret_cast<messageqcpp::ByteStream::byte&>(fIsAutocommitOn);
fTable = new DMLTable();
retval = fTable->read(bytestream);
bytestream >> fTableOid;
bytestream >> reinterpret_cast< messageqcpp::ByteStream::byte&>(fIsInsertSelect);
bytestream >> reinterpret_cast< messageqcpp::ByteStream::byte&>(fIsBatchInsert);
bytestream >> reinterpret_cast< messageqcpp::ByteStream::byte&>(fIsAutocommitOn);
return retval;
}
void InsertDMLPackage::readMetaData(messageqcpp::ByteStream& bytestream)
{
messageqcpp::ByteStream::quadbyte session_id;
bytestream >> session_id;
fSessionID = session_id;
bytestream >> fUuid;
std::string dmlStatement;
bytestream >> fDMLStatement;
bytestream >> fSQLStatement;
bytestream >> fSchemaName;
bytestream >> fTimeZone;
uint8_t logging;
bytestream >> logging;
fLogging = (logging != 0);
uint8_t logending;
bytestream >> logending;
fLogending = (logending != 0);
bytestream >> fTableOid;
bytestream >> reinterpret_cast<messageqcpp::ByteStream::byte&>(fIsInsertSelect);
bytestream >> reinterpret_cast<messageqcpp::ByteStream::byte&>(fIsBatchInsert);
bytestream >> reinterpret_cast<messageqcpp::ByteStream::byte&>(fIsAutocommitOn);
fTable = new DMLTable();
fTable->readMetaData(bytestream);
}
// Has to be called after InsertDMLPackage::readMetaData()
void InsertDMLPackage::readRowData(messageqcpp::ByteStream& bytestream)
{
fTable->readRowData(bytestream);
}
int InsertDMLPackage::buildFromBuffer(std::string& buffer, int columns, int rows)
{
#ifdef DML_PACKAGE_DEBUG

12
dbcon/dmlpackage/insertdmlpackage.h
View File

@ -73,6 +73,18 @@ public:
*/
EXPORT int read(messageqcpp::ByteStream& bytestream);
/** @brief read InsertDMLPackage metadata from bytestream
*
* @param bytestream the ByteStream to read from
*/
EXPORT void readMetaData(messageqcpp::ByteStream& bytestream);
/** @brief read InsertDMLPackage row data from bytestream
*
* @param bytestream the ByteStream to read from
*/
EXPORT void readRowData(messageqcpp::ByteStream& bytestream);
/** @brief build a InsertDMLPackage from a string buffer
*
* @param buffer