You've already forked mariadb-columnstore-engine
mirror of
https://github.com/mariadb-corporation/mariadb-columnstore-engine.git
synced 2025-07-04 04:42:30 +03:00
Merge branch 'develop-1.1' into 1.1-merge-up-2018-10-05
This commit is contained in:
Andrew Hutchings
@ -187,6 +187,37 @@ int WriteEngineWrapper::checkValid(const TxnID& txnid, const ColStructList& colS
|
||||
return NO_ERROR;
|
||||
}
|
||||
|
||||
/*@brief findSmallestColumn --Find the smallest column for this table
|
||||
*/
|
||||
/***********************************************************
|
||||
* DESCRIPTION:
|
||||
* Find the smallest column for this table
|
||||
* PARAMETERS:
|
||||
* lowColLen - returns smallest column width
|
||||
* colId - returns smallest column id
|
||||
* colStructList - column struct list
|
||||
* RETURN:
|
||||
* void
|
||||
***********************************************************/
|
||||
void WriteEngineWrapper::findSmallestColumn(uint32_t& colId, ColStructList colStructList)
|
||||
// MCOL-1675: find the smallest column width to calculate the RowID from so
|
||||
// that all HWMs will be incremented by this operation
|
||||
{
|
||||
int32_t lowColLen = 8192;
|
||||
for (uint32_t colIt = 0; colIt < colStructList.size(); colIt++)
|
||||
{
|
||||
if (colStructList[colIt].colWidth < lowColLen)
|
||||
{
|
||||
colId = colIt;
|
||||
lowColLen = colStructList[colId].colWidth;
|
||||
if ( lowColLen == 1 )
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*@convertValArray - Convert interface values to internal values
|
||||
*/
|
||||
/***********************************************************
|
||||
@ -953,6 +984,11 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
|
||||
for (i = 0; i < colStructList.size(); i++)
|
||||
Convertor::convertColType(&colStructList[i]);
|
||||
|
||||
uint32_t colId = 0;
|
||||
// MCOL-1675: find the smallest column width to calculate the RowID from so
|
||||
// that all HWMs will be incremented by this operation
|
||||
findSmallestColumn(colId, colStructList);
|
||||
|
||||
// rc = checkValid(txnid, colStructList, colValueList, ridList);
|
||||
// if (rc != NO_ERROR)
|
||||
// return rc;
|
||||
@ -979,8 +1015,8 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
|
||||
//--------------------------------------------------------------------------
|
||||
if (isFirstBatchPm)
|
||||
{
|
||||
currentDBrootIdx = dbRootExtentTrackers[0]->getCurrentDBRootIdx();
|
||||
extentInfo = dbRootExtentTrackers[0]->getDBRootExtentList();
|
||||
currentDBrootIdx = dbRootExtentTrackers[colId]->getCurrentDBRootIdx();
|
||||
extentInfo = dbRootExtentTrackers[colId]->getDBRootExtentList();
|
||||
dbRoot = extentInfo[currentDBrootIdx].fDbRoot;
|
||||
partitionNum = extentInfo[currentDBrootIdx].fPartition;
|
||||
|
||||
@ -1027,7 +1063,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
|
||||
{
|
||||
colOp = m_colOp[op(colStructList[i].fCompressionType)];
|
||||
colOp->initColumn(curCol);
|
||||
colOp->setColParam(curCol, 0, colStructList[i].colWidth, colStructList[i].colDataType,
|
||||
colOp->setColParam(curCol, colId, colStructList[i].colWidth, colStructList[i].colDataType,
|
||||
colStructList[i].colType, colStructList[i].dataOid, colStructList[i].fCompressionType,
|
||||
dbRoot, partitionNum, segmentNum);
|
||||
rc = colOp->extendColumn(curCol, false, extents[i].startBlkOffset, extents[i].startLbid, extents[i].allocSize, dbRoot,
|
||||
@ -1165,7 +1201,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
|
||||
} // if (isFirstBatchPm)
|
||||
else //get the extent info from tableMetaData
|
||||
{
|
||||
ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[0].dataOid);
|
||||
ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[colId].dataOid);
|
||||
ColExtsInfo::iterator it = aColExtsInfo.begin();
|
||||
|
||||
while (it != aColExtsInfo.end())
|
||||
@ -1201,7 +1237,7 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
|
||||
//--------------------------------------------------------------------------
|
||||
// allocate row id(s)
|
||||
//--------------------------------------------------------------------------
|
||||
curColStruct = colStructList[0];
|
||||
curColStruct = colStructList[colId];
|
||||
colOp = m_colOp[op(curColStruct.fCompressionType)];
|
||||
|
||||
colOp->initColumn(curCol);
|
||||
@ -1212,26 +1248,29 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
|
||||
vector<ExtentInfo> fileInfo;
|
||||
dbRoot = curColStruct.fColDbRoot;
|
||||
//use the first column to calculate row id
|
||||
ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[0].dataOid);
|
||||
ColExtsInfo aColExtsInfo = tableMetaData->getColExtsInfo(colStructList[colId].dataOid);
|
||||
ColExtsInfo::iterator it = aColExtsInfo.begin();
|
||||
|
||||
while (it != aColExtsInfo.end())
|
||||
{
|
||||
if ((it->dbRoot == colStructList[0].fColDbRoot) && (it->partNum == colStructList[0].fColPartition) && (it->segNum == colStructList[0].fColSegment) && it->current )
|
||||
if ((it->dbRoot == colStructList[colId].fColDbRoot) &&
|
||||
(it->partNum == colStructList[colId].fColPartition) &&
|
||||
(it->segNum == colStructList[colId].fColSegment) && it->current )
|
||||
{
|
||||
break;
|
||||
|
||||
}
|
||||
it++;
|
||||
}
|
||||
|
||||
if (it != aColExtsInfo.end())
|
||||
{
|
||||
hwm = it->hwm;
|
||||
//cout << "Got from colextinfo hwm for oid " << colStructList[0].dataOid << " is " << hwm << " and seg is " << colStructList[0].fColSegment << endl;
|
||||
//cout << "Got from colextinfo hwm for oid " << colStructList[colId].dataOid << " is " << hwm << " and seg is " << colStructList[0].fColSegment << endl;
|
||||
}
|
||||
|
||||
oldHwm = hwm; //Save this info for rollback
|
||||
//need to pass real dbRoot, partition, and segment to setColParam
|
||||
colOp->setColParam(curCol, 0, curColStruct.colWidth, curColStruct.colDataType,
|
||||
colOp->setColParam(curCol, colId, curColStruct.colWidth, curColStruct.colDataType,
|
||||
curColStruct.colType, curColStruct.dataOid, curColStruct.fCompressionType,
|
||||
curColStruct.fColDbRoot, curColStruct.fColPartition, curColStruct.fColSegment);
|
||||
rc = colOp->openColumnFile(curCol, segFile, useTmpSuffix); // @bug 5572 HDFS tmp file
|
||||
@ -1261,8 +1300,8 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
|
||||
curCol, (uint64_t)totalRow, rowIdArray, hwm, newExtent, rowsLeft, newHwm, newFile,
|
||||
newColStructList, newDctnryStructList, dbRootExtentTrackers, insertSelect, true, tableOid, isFirstBatchPm);
|
||||
|
||||
//cout << "after allocrowid, total row = " <<totalRow << " newExtent is " << newExtent << endl;
|
||||
// cout << "column oid " << curColStruct.dataOid << " has hwm:newHwm = " << hwm <<":" << newHwm<< endl;
|
||||
//cout << "after allocrowid, total row = " <<totalRow << " newExtent is " << newExtent << endl;
|
||||
//cout << "column oid " << curColStruct.dataOid << " has hwm:newHwm = " << hwm <<":" << newHwm<< endl;
|
||||
if (rc != NO_ERROR) //Clean up is already done
|
||||
return rc;
|
||||
|
||||
@ -1274,14 +1313,17 @@ int WriteEngineWrapper::insertColumnRecs(const TxnID& txnid,
|
||||
// Expand initial abbreviated extent if any RID in 1st extent is > 256K.
|
||||
// if totalRow == rowsLeft, then not adding rows to 1st extent, so skip it.
|
||||
//--------------------------------------------------------------------------
|
||||
// DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated?
|
||||
// DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated?
|
||||
if ((curCol.dataFile.fPartition == 0) &&
|
||||
(curCol.dataFile.fSegment == 0) &&
|
||||
((totalRow - rowsLeft) > 0) &&
|
||||
(rowIdArray[totalRow - rowsLeft - 1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK))
|
||||
{
|
||||
for (size_t k = 1; k < colStructList.size(); k++)
|
||||
for (unsigned k=0; k<colStructList.size(); k++)
|
||||
{
|
||||
// Skip the selected column
|
||||
if (k == colId)
|
||||
continue;
|
||||
Column expandCol;
|
||||
colOp = m_colOp[op(colStructList[k].fCompressionType)];
|
||||
colOp->setColParam(expandCol, 0,
|
||||
@ -1683,19 +1725,11 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid,
|
||||
for (i = 0; i < colStructList.size(); i++)
|
||||
Convertor::convertColType(&colStructList[i]);
|
||||
|
||||
// MCOL-984: find the smallest column width to calculate the RowID from so
|
||||
uint32_t colId = 0;
|
||||
// MCOL-1675: find the smallest column width to calculate the RowID from so
|
||||
// that all HWMs will be incremented by this operation
|
||||
int32_t lowColLen = 8192;
|
||||
int32_t colId = 0;
|
||||
findSmallestColumn(colId, colStructList);
|
||||
|
||||
for (uint32_t colIt = 0; colIt < colStructList.size(); colIt++)
|
||||
{
|
||||
if (colStructList[colIt].colWidth < lowColLen)
|
||||
{
|
||||
colId = colIt;
|
||||
lowColLen = colStructList[colId].colWidth;
|
||||
}
|
||||
}
|
||||
|
||||
// rc = checkValid(txnid, colStructList, colValueList, ridList);
|
||||
// if (rc != NO_ERROR)
|
||||
@ -2019,7 +2053,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid,
|
||||
// Expand initial abbreviated extent if any RID in 1st extent is > 256K.
|
||||
// if totalRow == rowsLeft, then not adding rows to 1st extent, so skip it.
|
||||
//--------------------------------------------------------------------------
|
||||
// DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated?
|
||||
// DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated?
|
||||
if ((curCol.dataFile.fPartition == 0) &&
|
||||
(curCol.dataFile.fSegment == 0) &&
|
||||
((totalRow - rowsLeft) > 0) &&
|
||||
@ -2032,7 +2066,8 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid,
|
||||
continue;
|
||||
|
||||
Column expandCol;
|
||||
colOp = m_colOp[op(colStructList[k].fCompressionType)];
|
||||
colOp = m_colOp[op(colStructList[k].fCompressionType)];
|
||||
// Shouldn't we change 0 to colId here?
|
||||
colOp->setColParam(expandCol, 0,
|
||||
colStructList[k].colWidth,
|
||||
colStructList[k].colDataType,
|
||||
@ -3106,6 +3141,11 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid,
|
||||
for (i = 0; i < colStructList.size(); i++)
|
||||
Convertor::convertColType(&colStructList[i]);
|
||||
|
||||
uint32_t colId = 0;
|
||||
// MCOL-1675: find the smallest column width to calculate the RowID from so
|
||||
// that all HWMs will be incremented by this operation
|
||||
findSmallestColumn(colId, colStructList);
|
||||
|
||||
rc = checkValid(txnid, colStructList, colValueList, ridList);
|
||||
|
||||
if (rc != NO_ERROR)
|
||||
@ -3124,7 +3164,7 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid,
|
||||
//--------------------------------------------------------------------------
|
||||
// allocate row id(s)
|
||||
//--------------------------------------------------------------------------
|
||||
curColStruct = colStructList[0];
|
||||
curColStruct = colStructList[colId];
|
||||
colOp = m_colOp[op(curColStruct.fCompressionType)];
|
||||
|
||||
colOp->initColumn(curCol);
|
||||
@ -3161,7 +3201,7 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid,
|
||||
|
||||
oldHwm = hwm; //Save this info for rollback
|
||||
//need to pass real dbRoot, partition, and segment to setColParam
|
||||
colOp->setColParam(curCol, 0, curColStruct.colWidth, curColStruct.colDataType,
|
||||
colOp->setColParam(curCol, colId, curColStruct.colWidth, curColStruct.colDataType,
|
||||
curColStruct.colType, curColStruct.dataOid, curColStruct.fCompressionType,
|
||||
dbRoot, partitionNum, segmentNum);
|
||||
|
||||
@ -3279,13 +3319,15 @@ int WriteEngineWrapper::insertColumnRec_Single(const TxnID& txnid,
|
||||
// if totalRow == rowsLeft, then not adding rows to 1st extent, so skip it.
|
||||
//--------------------------------------------------------------------------
|
||||
// DMC-SHARED_NOTHING_NOTE: Is it safe to assume only part0 seg0 is abbreviated?
|
||||
if ((colStructList[0].fColPartition == 0) &&
|
||||
(colStructList[0].fColSegment == 0) &&
|
||||
if ((colStructList[colId].fColPartition == 0) &&
|
||||
(colStructList[colId].fColSegment == 0) &&
|
||||
((totalRow - rowsLeft) > 0) &&
|
||||
(rowIdArray[totalRow - rowsLeft - 1] >= (RID)INITIAL_EXTENT_ROWS_TO_DISK))
|
||||
{
|
||||
for (size_t k = 1; k < colStructList.size(); k++)
|
||||
for (unsigned k=0; k<colStructList.size(); k++)
|
||||
{
|
||||
if (k == colId)
|
||||
continue;
|
||||
Column expandCol;
|
||||
colOp = m_colOp[op(colStructList[k].fCompressionType)];
|
||||
colOp->setColParam(expandCol, 0,
|
||||
|
||||
Reference in New Issue
Block a user