database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

MCOL-5021 Code changes based on review feedback.

Browse Source
This commit is contained in:
Gagan Goel
2022-08-01 19:35:55 -04:00
parent 11b7ee2f11
commit cbfdae3481
20 changed files with 407 additions and 435 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
writeengine/bulk/we_bulkload.cpp
View File

@ -265,7 +265,7 @@ int BulkLoad::loadJobInfo(const string& fullName, bool bUseTempJobFile, int argc
return rc;
}
Job& curJob = const_cast<Job&>(fJobInfo.getJob());
Job& curJob = fJobInfo.getJob();
string logFile, errlogFile;
logFile = std::string(MCSLOGDIR) + "/cpimport/" + "Job_" + Convertor::int2Str(curJob.id) + LOG_SUFFIX;
errlogFile =
@ -319,6 +319,8 @@ int BulkLoad::loadJobInfo(const string& fullName, bool bUseTempJobFile, int argc
execplan::CalpontSystemCatalog::OID tableAUXColOid;
std::string tblName;
std::string curTblName = curJob.jobTableList[i].tblName;
// Parse out <tablename> from [<schemaname>.]<tablename> string
string::size_type startName = curTblName.rfind('.');
if (startName == std::string::npos)
@ -376,18 +378,10 @@ int BulkLoad::loadJobInfo(const string& fullName, bool bUseTempJobFile, int argc
// tableAUXColOid = 0
if (tableAUXColOid > 3000)
{
JobColumn curColumn;
curColumn.colName = "aux";
curColumn.mapOid = tableAUXColOid;
curColumn.typeName = execplan::AUX_COL_DATATYPE_STRING;
curColumn.width = execplan::AUX_COL_WIDTH;
curColumn.definedWidth = execplan::AUX_COL_WIDTH;
curColumn.compressionType = execplan::AUX_COL_COMPRESSION_TYPE;
curColumn.dctnry.fCompressionType = execplan::AUX_COL_COMPRESSION_TYPE;
curColumn.fMinIntSat = execplan::AUX_COL_MINVALUE;
curColumn.fMaxIntSat = execplan::AUX_COL_MAXVALUE;
curColumn.fWithDefault = true;
curColumn.fDefaultUInt = 1;
JobColumn curColumn("aux", tableAUXColOid, execplan::AUX_COL_DATATYPE_STRING,
execplan::AUX_COL_WIDTH, execplan::AUX_COL_WIDTH,
execplan::AUX_COL_COMPRESSION_TYPE, execplan::AUX_COL_COMPRESSION_TYPE,
execplan::AUX_COL_MINVALUE, execplan::AUX_COL_MAXVALUE, true, 1);
curJob.jobTableList[i].colList.push_back(curColumn);
JobFieldRef fieldRef(BULK_FLDCOL_COLUMN_DEFAULT, curJob.jobTableList[i].colList.size() - 1);
curJob.jobTableList[i].fFldRefs.push_back(fieldRef);

62
writeengine/server/we_dmlcommandproc.cpp
View File

@ -91,6 +91,27 @@ WE_DMLCommandProc::~WE_DMLCommandProc()
dbRootExtTrackerVec.clear();
}
void WE_DMLCommandProc::processAuxCol(const std::vector<std::string>& origVals,
WriteEngine::ColValueList& colValuesList,
WriteEngine::DictStrList& dicStringList)
{
WriteEngine::ColTupleList auxColTuples;
WriteEngine::dictStr auxDicStrings;
for (uint32_t j = 0; j < origVals.size(); j++)
{
WriteEngine::ColTuple auxColTuple;
auxColTuple.data = (uint8_t)1;
auxColTuples.push_back(auxColTuple);
//@Bug 2515. Only pass string values to write engine
auxDicStrings.push_back("");
}
colValuesList.push_back(auxColTuples);
//@Bug 2515. Only pass string values to write engine
dicStringList.push_back(auxDicStrings);
}
uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std::string& err)
{
uint8_t rc = 0;
@ -443,22 +464,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
// MCOL-5021
if ((i == numcols - 1) && (tableAUXColOid > 3000))
{
WriteEngine::ColTupleList auxColTuples;
WriteEngine::dictStr auxDicStrings;
for (uint32_t j = 0; j < origVals.size(); j++)
{
WriteEngine::ColTuple auxColTuple;
auxColTuple.data = (uint8_t)1;
auxColTuples.push_back(auxColTuple);
//@Bug 2515. Only pass string values to write engine
auxDicStrings.push_back("");
}
colValuesList.push_back(auxColTuples);
//@Bug 2515. Only pass string values to write engine
dicStringList.push_back(auxDicStrings);
processAuxCol(origVals, colValuesList, dicStringList);
}
++row_iterator;
@ -1385,22 +1391,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
// MCOL-5021
if ((i == numcols - 1) && (tableAUXColOid > 3000))
{
WriteEngine::ColTupleList auxColTuples;
WriteEngine::dictStr auxDicStrings;
for (uint32_t j = 0; j < origVals.size(); j++)
{
WriteEngine::ColTuple auxColTuple;
auxColTuple.data = (uint8_t)1;
auxColTuples.push_back(auxColTuple);
//@Bug 2515. Only pass string values to write engine
auxDicStrings.push_back("");
}
colValuesList.push_back(auxColTuples);
//@Bug 2515. Only pass string values to write engine
dicStringList.push_back(auxDicStrings);
processAuxCol(origVals, colValuesList, dicStringList);
}
++row_iterator;
@ -1502,7 +1493,6 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
return rc;
}
#if 0
uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, std::string& err,
ByteStream::quadbyte& PMId)
{
@ -1665,8 +1655,9 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs,
return rc;
}
//@Bug 5996 validate hwm before starts
rc = validateColumnHWMs(ridList, systemCatalogPtr, colDBRootExtentInfo, "Starting");
// @Bug 5996 validate hwm before starts
// TODO MCOL-5021 hasAuxCol is hardcoded to false; add support here.
rc = validateColumnHWMs(ridList, systemCatalogPtr, colDBRootExtentInfo, "Starting", false);
if (rc != 0)
{
@ -2233,7 +2224,6 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs,
CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId | 0x80000000);
return rc;
}
#endif
uint8_t WE_DMLCommandProc::commitBatchAutoOn(messageqcpp::ByteStream& bs, std::string& err)
{

8
writeengine/server/we_dmlcommandproc.h
View File

@ -78,8 +78,8 @@ class WE_DMLCommandProc
EXPORT uint8_t rollbackVersion(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processBatchInsert(messageqcpp::ByteStream& bs, std::string& err,
ByteStream::quadbyte& PMId);
//EXPORT uint8_t processBatchInsertBinary(messageqcpp::ByteStream& bs, std::string& err,
// ByteStream::quadbyte& PMId);
EXPORT uint8_t processBatchInsertBinary(messageqcpp::ByteStream& bs, std::string& err,
ByteStream::quadbyte& PMId);
EXPORT uint8_t commitBatchAutoOn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t commitBatchAutoOff(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t rollbackBatchAutoOn(messageqcpp::ByteStream& bs, std::string& err);
@ -127,6 +127,10 @@ class WE_DMLCommandProc
const std::vector<BRM::FileInfo>& files,
const std::vector<BRM::OID_t>& oidsToFlush, std::string& err);
void processAuxCol(const std::vector<std::string>& origVals,
WriteEngine::ColValueList& colValuesList,
WriteEngine::DictStrList& dicStringList);
bool fIsFirstBatchPm;
std::map<uint32_t, rowgroup::RowGroup*> rowGroups;
std::map<uint32_t, dmlpackage::UpdateDMLPackage> cpackages;

2
writeengine/server/we_readthread.cpp
View File

@ -156,13 +156,11 @@ void DmlReadThread::operator()()
break;
}
#if 0
case WE_SVR_BATCH_INSERT_BINARY:
{
rc = fWeDMLprocessor->processBatchInsertBinary(ibs, errMsg, PMId);
break;
}
#endif
case WE_SVR_GET_WRITTEN_LBIDS:
{

3
writeengine/shared/we_config.cpp
View File

@ -188,7 +188,8 @@ void Config::checkReload()
const std::string fastDeleteTemp = cf->getConfig("WriteEngine", "FastDelete");
if (fastDeleteTemp.length() != 0 && boost::iequals(fastDeleteTemp, "true"))
if (fastDeleteTemp.length() != 0 &&
(fastDeleteTemp == "y" || fastDeleteTemp == "Y"))
{
m_FastDelete = true;
}

32
writeengine/shared/we_type.h
View File

@ -436,6 +436,38 @@ struct JobColumn /** @brief Job Column Structure */
, fDefaultWideDecimal(0)
{
}
JobColumn(const std::string& colName_, OID mapOid_, const std::string& typeName_,
int width_, int definedWidth_, int compressionType_, int dctnryCompressionType_,
int64_t minIntSat_, uint64_t maxIntSat_, bool withDefault_,
unsigned long long defaultUInt_)
: colName(colName_)
, mapOid(mapOid_)
, dataType(execplan::CalpontSystemCatalog::INT)
, weType(WR_INT)
, typeName(typeName_)
, emptyVal(nullptr)
, width(width_)
, definedWidth(definedWidth_)
, dctnryWidth(0)
, precision(0)
, scale(0)
, fNotNull(false)
, fFldColRelation(BULK_FLDCOL_COLUMN_FIELD)
, colType(' ')
, compressionType(compressionType_)
, autoIncFlag(false)
, fMinIntSat(minIntSat_)
, fMaxIntSat(maxIntSat_)
, fMinDblSat(0)
, fMaxDblSat(0)
, fWithDefault(withDefault_)
, fDefaultInt(0)
, fDefaultUInt(defaultUInt_)
, fDefaultDbl(0.0)
, fDefaultWideDecimal(0)
{
dctnry.fCompressionType = dctnryCompressionType_;
}
};
typedef std::vector<JobColumn> JobColList; /** @brief column value list */

14
writeengine/wrapper/we_colop.cpp
View File

@ -1412,12 +1412,12 @@ int ColumnOp::openColumnFile(Column& column, std::string& segFile, bool useTmpSu
if (!isValid(column))
return ERR_INVALID_PARAM;
std::string mode;
std::string mode = "r";
if (isReadOnly)
mode = "r";
else
if (!isReadOnly)
{
mode = "r+b";
}
// open column data file
column.dataFile.pFile =
@ -1817,10 +1817,9 @@ int ColumnOp::writeRowsReadOnly(Column& curCol, uint64_t totalRow, const RIDList
uint64_t i = 0, curRowId;
int dataFbo, dataBio, curDataFbo = -1;
unsigned char dataBuf[BYTE_PER_BLOCK];
bool bExit = false;
int rc = NO_ERROR;
while (!bExit)
while (i < totalRow)
{
curRowId = ridList[i];
@ -1845,9 +1844,6 @@ int ColumnOp::writeRowsReadOnly(Column& curCol, uint64_t totalRow, const RIDList
}
i++;
if (i >= totalRow)
bExit = true;
}
return rc;

8
writeengine/xml/we_xmljob.h
View File

@ -84,6 +84,14 @@ class XMLJob : public XMLOp
return fJob;
}
/**
* @brief Get reference to job structure
*/
Job& getJob()
{
return fJob;
}
/**
* @brief Load job information
* @param fileName Name of Job XML file to be read