database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

Merge pull request #1771 from mariadb-SergeyZefirov/MCOL-2044-update-ranges-during-DML

Browse Source 
Mcol 2044 update ranges during dml
This commit is contained in:
Roman Nozdrin
2021-04-06 12:46:02 +03:00
committed by GitHub
parent 47772cff10 1143e89fd0
commit c1138c4793
198 changed files with 2949 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
writeengine/server/we_ddlcommandproc.cpp
View File

@ -4998,7 +4998,6 @@ void WE_DDLCommandProc::purgeFDCache()
aFile.compType = aIt->compType;
files.push_back(aFile);
fDbrm.lookupLocalStartLbid(aFile.oid, aFile.partitionNum, aFile.segmentNum, aIt->hwm, startLbid);
lbidList.push_back(startLbid);
//cout <<"Added to files oid:dbroot:part:seg:compType = " << aFile.oid<<":"<<aFile.dbRoot<<":"<<aFile.partitionNum<<":"<<aFile.segmentNum
//<<":"<<aFile.compType <<endl;
aIt++;

6
writeengine/shared/we_brm.cpp
View File

@ -1803,6 +1803,12 @@ void BRMWrapper::writeVBEnd(const VER_t transID, std::vector<LBIDRange>& rangeLi
blockRsltnMgrPtr->endVBCopy(transID, rangeList);
}
int BRMWrapper::getExtentCPMaxMin(const BRM::LBID_t lbid, BRM::CPMaxMin& cpMaxMin)
{
int rc = blockRsltnMgrPtr->getExtentCPMaxMin(lbid, cpMaxMin);
return getRC(rc, ERR_BRM_GET_EXTENT_CP);
}
} //end of namespace
// vim:ts=4 sw=4:

51
writeengine/shared/we_brm.h
View File

@ -33,6 +33,7 @@
#include "we_obj.h"
#include<sys/time.h>
#include "brmtypes.h"
#include "mcs_datatype.h"
#include "IDBDataFile.h"
#include "IDBPolicy.h"
@ -48,6 +49,42 @@ namespace WriteEngine
// forward reference
class DbFileOp;
/** @brief Extended CPInfo - with type handler for all type-related information */
struct ExtCPInfo
{
execplan::CalpontSystemCatalog::ColDataType fColType;
int fColWidth;
BRM::CPInfo fCPInfo;
ExtCPInfo(execplan::CalpontSystemCatalog::ColDataType colType, int colWidth)
: fColType(colType), fColWidth(colWidth)
{
fCPInfo.isBinaryColumn = colWidth > datatypes::MAXLEGACYWIDTH;
}
void toInvalid()
{
auto mm = datatypes::MinMaxInfo::invalidRange(fColType);
fCPInfo.max = mm.max;
fCPInfo.min = mm.min;
fCPInfo.bigMax = mm.int128Max;
fCPInfo.bigMin = mm.int128Min;
}
bool isInvalid()
{
datatypes::MinMaxInfo mm;
mm.max = fCPInfo.max;
mm.min = fCPInfo.min;
mm.int128Max = fCPInfo.bigMax;
mm.int128Min = fCPInfo.bigMin;
return datatypes::MinMaxInfo::isRangeInvalid(mm, fColType, fColWidth);
}
bool isBinaryColumn()
{
return fCPInfo.isBinaryColumn;
}
};
typedef std::vector<ExtCPInfo> ExtCPInfoList;
/** Class BRMWrapper */
class BRMWrapper : public WEObj
{
@ -262,7 +299,7 @@ public:
/**
* @brief set extents CP min/max info into extent map
*/
int setExtentsMaxMin(const BRM::CPInfoList_t& cpinfoList);
int setExtentsMaxMin(const ExtCPInfoList& cpinfoList);
/**
* @brief Perform bulk rollback of any column extents that logically follow
@ -473,6 +510,8 @@ public:
m_useVb = val;
}
int getExtentCPMaxMin(const BRM::LBID_t lbid, BRM::CPMaxMin& cpMaxMin);
private:
//--------------------------------------------------------------------------
// Private methods
@ -647,9 +686,15 @@ inline int BRMWrapper::bulkSetHWMAndCP(
return getRC( rc, ERR_BRM_BULK_UPDATE );
}
inline int BRMWrapper::setExtentsMaxMin(const BRM::CPInfoList_t& cpinfoList)
inline int BRMWrapper::setExtentsMaxMin(const ExtCPInfoList& extCPInfoList)
{
int rc = blockRsltnMgrPtr->setExtentsMaxMin(cpinfoList);
BRM::CPInfoList_t toSet;
toSet.reserve(extCPInfoList.size());
for (const auto& extCPInfo : extCPInfoList)
{
toSet.push_back(extCPInfo.fCPInfo);
}
int rc = blockRsltnMgrPtr->setExtentsMaxMin(toSet);
return getRC( rc, ERR_BRM_SET_EXTENTS_CP );
}

1
writeengine/shared/we_define.cpp
View File

@ -206,6 +206,7 @@ WErrorCodes::WErrorCodes() : fErrorCodes()
fErrorCodes[ERR_BRM_GET_SUSPEND] = " BRM error get the system suspend flag ";
fErrorCodes[ERR_BRM_BAD_STRIPE_CNT] = " Incorrect number of column extents allocated in stripe";
fErrorCodes[ERR_BRM_UNSUPP_WIDTH] = " Unsupported non-dictionary column width";
fErrorCodes[ERR_BRM_GET_EXTENT_CP] = " BRM error getting extent max/min";
// DM error
fErrorCodes[ERR_DM_CONVERT_OID] = " a DM Conversion error";

1
writeengine/shared/we_define.h
View File

@ -311,6 +311,7 @@ const int ERR_BRM_SUSPEND = ERR_BRMBASE + 44;// BRM is set to Suspend
const int ERR_BRM_GET_SUSPEND = ERR_BRMBASE + 45;// error getting BRM Suspend flag
const int ERR_BRM_BAD_STRIPE_CNT = ERR_BRMBASE + 46;// Incorrect num of cols allocated in stripe
const int ERR_BRM_UNSUPP_WIDTH = ERR_BRMBASE + 47;// Non-dict column Width > allowed MAX.
const int ERR_BRM_GET_EXTENT_CP = ERR_BRMBASE + 48;// Error getting extent's CPInfo
//--------------------------------------------------------------------------
// DM error

1
writeengine/shared/we_type.h
View File

@ -34,6 +34,7 @@
#include <cstring>
#include <stdexcept>
#include "brm.h"
#include "we_define.h"
#include "we_typeext.h"
#include "calpontsystemcatalog.h"

123
writeengine/wrapper/we_colop.cpp
View File

@ -82,7 +82,8 @@ ColumnOp::~ColumnOp()
* tableFid - the file id for table bitmap file
* totalRow - the total number of rows need to be allocated
* useStartingExtent - Indicates whether rows can be added to an existing
* starting extent
* starting extent
* newExtents - where to write extents allocated for newColStructList, etc, structures.
* RETURN:
* NO_ERROR if success
* rowIdArray - allocation of the row id left here
@ -90,7 +91,7 @@ ColumnOp::~ColumnOp()
int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent,
Column& column, uint64_t totalRow, RID* rowIdArray, HWM& hwm, bool& newExtent, uint64_t& rowsLeft, HWM& newHwm,
bool& newFile, ColStructList& newColStructList, DctnryStructList& newDctnryStructList, std::vector<boost::shared_ptr<DBRootExtentTracker> >& dbRootExtentTrackers,
bool insertSelect, bool isBatchInsert, OID tableOid, bool isFirstBatchPm)
bool insertSelect, bool isBatchInsert, OID tableOid, bool isFirstBatchPm, std::vector<BRM::LBID_t>* newExtents)
{
//MultiFiles per OID: always append the rows to the end for now.
// See if the current HWM block might be in an abbreviated extent that
@ -379,6 +380,10 @@ int ColumnOp::allocRowId(const TxnID& txnid, bool useStartingExtent,
newDctnryStructList[i].fColSegment = segment;
newDctnryStructList[i].fColDbRoot = dbRoot;
lbids.push_back(extents[i].startLbid);
if (newExtents)
{
(*newExtents).push_back(extents[i].startLbid);
}
colDataTypes.push_back(newColStructList[i].colDataType);
}
@ -963,7 +968,7 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi
refBufOffset = BYTE_PER_BLOCK;
colBufOffset = BYTE_PER_BLOCK;
dirty = false;
BRM::CPInfo cpInfo;
ExtCPInfo cpInfo(column.colDataType, column.colWidth);
if (autoincrement)
{
@ -1048,19 +1053,17 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi
}
}
cpInfo.isBinaryColumn = column.colWidth > 8;
if (!cpInfo.isBinaryColumn)
if (!cpInfo.isBinaryColumn())
{
cpInfo.max = nextValStart + nexValNeeded - 1;
cpInfo.min = nextValStart;
cpInfo.fCPInfo.max = nextValStart + nexValNeeded - 1;
cpInfo.fCPInfo.min = nextValStart;
}
else
{
cpInfo.bigMax = nextValStart + nexValNeeded - 1;
cpInfo.bigMin = nextValStart;
cpInfo.fCPInfo.bigMax = nextValStart + nexValNeeded - 1;
cpInfo.fCPInfo.bigMin = nextValStart;
}
cpInfo.seqNum = 0;
cpInfo.fCPInfo.seqNum = 0;
}
else
@ -1113,36 +1116,9 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi
}
}
cpInfo.isBinaryColumn = column.colWidth > 8;
cpInfo.toInvalid();
if (!cpInfo.isBinaryColumn)
{
if (isUnsigned(column.colDataType))
{
cpInfo.max = 0;
cpInfo.min = static_cast<int64_t>(numeric_limits<uint64_t>::max());
}
else
{
cpInfo.max = numeric_limits<int64_t>::min();
cpInfo.min = numeric_limits<int64_t>::max();
}
}
else
{
if (isUnsigned(column.colDataType))
{
cpInfo.bigMax = 0;
cpInfo.min = -1;
}
else
{
utils::int128Min(cpInfo.bigMax);
utils::int128Max(cpInfo.bigMin);
}
}
cpInfo.seqNum = -1;
cpInfo.fCPInfo.seqNum = SEQNUM_MARK_INVALID;
}
if (dirty)
@ -1165,39 +1141,11 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi
if (autoincrement) //@Bug 4074. Mark it invalid first to set later
{
BRM::CPInfo cpInfo1;
cpInfo1.isBinaryColumn = column.colWidth > 8;
if (!cpInfo1.isBinaryColumn)
{
if (isUnsigned(column.colDataType))
{
cpInfo1.max = 0;
cpInfo1.min = static_cast<int64_t>(numeric_limits<uint64_t>::max());
}
else
{
cpInfo1.max = numeric_limits<int64_t>::min();
cpInfo1.min = numeric_limits<int64_t>::max();
}
}
else
{
if (isUnsigned(column.colDataType))
{
cpInfo1.bigMax = 0;
cpInfo1.bigMin = -1;
}
else
{
utils::int128Min(cpInfo1.bigMax);
utils::int128Max(cpInfo1.bigMin);
}
}
cpInfo1.seqNum = -1;
cpInfo1.firstLbid = startLbid;
BRM::CPInfoList_t cpinfoList1;
ExtCPInfo cpInfo1(column.colDataType, column.colWidth);
cpInfo1.toInvalid();
cpInfo1.fCPInfo.seqNum = SEQNUM_MARK_INVALID;
cpInfo1.fCPInfo.firstLbid = startLbid;
ExtCPInfoList cpinfoList1;
cpinfoList1.push_back(cpInfo1);
rc = BRMWrapper::getInstance()->setExtentsMaxMin(cpinfoList1);
@ -1205,8 +1153,8 @@ int ColumnOp::fillColumn(const TxnID& txnid, Column& column, Column& refCol, voi
return rc;
}
BRM::CPInfoList_t cpinfoList;
cpInfo.firstLbid = startLbid;
ExtCPInfoList cpinfoList;
cpInfo.fCPInfo.firstLbid = startLbid;
cpinfoList.push_back(cpInfo);
rc = BRMWrapper::getInstance()->setExtentsMaxMin(cpinfoList);
@ -1627,7 +1575,7 @@ void ColumnOp::setColParam(Column& column,
* RETURN:
* NO_ERROR if success, other number otherwise
***********************************************************/
int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, const void* valArray, bool bDelete )
int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, const void* valArray, void* oldValArray, bool bDelete )
{
uint64_t i = 0, curRowId;
int dataFbo, dataBio, curDataFbo = -1;
@ -1748,6 +1696,12 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray,
}
// This is the write stuff
if (oldValArray)
{
uint8_t* p = static_cast<uint8_t*>(oldValArray);
memcpy(p + curCol.colWidth * i, dataBuf + dataBio, curCol.colWidth);
}
writeBufValue(dataBuf + dataBio, pVal, curCol.colWidth);
i++;
@ -1780,7 +1734,7 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray,
* RETURN:
* NO_ERROR if success, other number otherwise
***********************************************************/
int ColumnOp::writeRows(Column& curCol, uint64_t totalRow, const RIDList& ridList, const void* valArray, const void* oldValArray, bool bDelete )
int ColumnOp::writeRows(Column& curCol, uint64_t totalRow, const RIDList& ridList, const void* valArray, void* oldValArray, bool bDelete )
{
uint64_t i = 0, curRowId;
int dataFbo, dataBio, curDataFbo = -1;
@ -1895,6 +1849,13 @@ int ColumnOp::writeRows(Column& curCol, uint64_t totalRow, const RIDList& ridLis
curCol.colWidth);
}
// This is the write stuff
if (oldValArray)
{
uint8_t* p = static_cast<uint8_t*>(oldValArray);
memcpy(p + i * curCol.colWidth, dataBuf + dataBio, curCol.colWidth);
}
writeBufValue(dataBuf + dataBio, pVal, curCol.colWidth);
i++;
@ -1927,7 +1888,7 @@ int ColumnOp::writeRows(Column& curCol, uint64_t totalRow, const RIDList& ridLis
* RETURN:
* NO_ERROR if success, other number otherwise
***********************************************************/
int ColumnOp::writeRowsValues(Column& curCol, uint64_t totalRow, const RIDList& ridList, const void* valArray )
int ColumnOp::writeRowsValues(Column& curCol, uint64_t totalRow, const RIDList& ridList, const void* valArray, void* oldValArray)
{
uint64_t i = 0, curRowId;
int dataFbo, dataBio, curDataFbo = -1;
@ -2035,6 +1996,12 @@ int ColumnOp::writeRowsValues(Column& curCol, uint64_t totalRow, const RIDList&
}
// This is the write stuff
if (oldValArray)
{
uint8_t* p = static_cast<uint8_t*>(oldValArray);
memcpy(p + curCol.colWidth * i, dataBuf + dataBio, curCol.colWidth);
}
writeBufValue(dataBuf + dataBio, pVal, curCol.colWidth);
i++;

13
writeengine/wrapper/we_colop.h
View File

@ -73,9 +73,10 @@ public:
ColStructList& newColStructList,
DctnryStructList& newDctnryStructList,
std::vector<boost::shared_ptr<DBRootExtentTracker> >& dbRootExtentTrackers,
bool insertSelect = false,
bool isBatchInsert = false,
OID tableOid = 0, bool isFirstBatchPm = false);
bool insertSelect,
bool isBatchInsert,
OID tableOid, bool isFirstBatchPm,
std::vector<BRM::LBID_t>* newEtents);
/**
* @brief Create column file(s)
@ -266,6 +267,7 @@ public:
uint64_t totalRow,
const RID* rowIdArray,
const void* valArray,
void* oldValArray =0,
bool bDelete = false);
/**
@ -275,7 +277,7 @@ public:
uint64_t totalRow,
const RIDList& ridList,
const void* valArray,
const void* oldValArray = 0,
void* oldValArray = 0,
bool bDelete = false);
/**
@ -284,7 +286,8 @@ public:
EXPORT virtual int writeRowsValues(Column& curCol,
uint64_t totalRow,
const RIDList& ridList,
const void* valArray);
const void* valArray,
void* oldValArray = 0);
/**
* @brief Test if the pFile is an abbreviated extent.

773
writeengine/wrapper/writeengine.cpp
View File

File diff suppressed because it is too large Load Diff

46
writeengine/wrapper/writeengine.h
View File

@ -83,7 +83,7 @@ struct TxnLBIDRec
if ( m_LBIDSet.insert(lbid).second)
{
m_LBIDs.push_back(lbid);
m_ColDataTypes.push_back(colDataType);
m_ColDataTypes.push_back(colDataType);
}
}
};
@ -91,6 +91,20 @@ struct TxnLBIDRec
typedef boost::shared_ptr<TxnLBIDRec> SP_TxnLBIDRec_t;
typedef std::set<BRM::LBID_t> dictLBIDRec_t;
/** @brief Range information for 1 or 2 extents changed by DML operation. */
struct ColSplitMaxMinInfo {
ExtCPInfo fSplitMaxMinInfo[2]; /** @brief internal to write engine: min/max ranges for data in one and, possible, second extent. */
ExtCPInfo* fSplitMaxMinInfoPtrs[2]; /** @brief pointers to CPInfos in fSplitMaxMinInfo above */
ColSplitMaxMinInfo(execplan::CalpontSystemCatalog::ColDataType colDataType, int colWidth)
: fSplitMaxMinInfo { ExtCPInfo(colDataType, colWidth), ExtCPInfo(colDataType, colWidth) }
{
fSplitMaxMinInfoPtrs[0] = fSplitMaxMinInfoPtrs[1] = NULL; // disable by default.
}
};
typedef std::vector<ColSplitMaxMinInfo> ColSplitMaxMinInfoList;
/** Class WriteEngineWrapper */
class WriteEngineWrapper : public WEObj
{
@ -166,6 +180,15 @@ public:
const ColType colType,
ColTupleList& curTupleList, void* valArray,
bool bFromList = true) ;
/**
* @brief Updates range information given old range information, old values, new values and column information.
*/
EXPORT void updateMaxMinRange(const size_t totalNewRow, const size_t totalOldRow,
const execplan::CalpontSystemCatalog::ColType& cscColType,
const ColType colType,
const void* valArray, const void* oldValArray,
ExtCPInfo* maxMin, bool canStartWithInvalidRange);
/**
* @brief Create a column, include object ids for column data and bitmap files
@ -705,7 +728,8 @@ private:
ColValueList& colValueList,
RID* rowIdArray, const ColStructList& newColStructList,
ColValueList& newColValueList, const int32_t tableOid,
bool useTmpSuffix, bool versioning = true);
bool useTmpSuffix, bool versioning = true,
ColSplitMaxMinInfoList* maxMins = NULL);
int writeColumnRecBinary(const TxnID& txnid, const ColStructList& colStructList,
std::vector<uint64_t>& colValueList,
@ -715,18 +739,18 @@ private:
bool useTmpSuffix, bool versioning = true);
//@Bug 1886,2870 pass the address of ridList vector
int writeColumnRec(const TxnID& txnid,
int writeColumnRecUpdate(const TxnID& txnid,
const CSCTypesList& cscColTypeList,
const ColStructList& colStructList,
const ColValueList& colValueList, std::vector<void*>& colOldValueList,
const RIDList& ridList, const int32_t tableOid,
bool convertStructFlag = true, ColTupleList::size_type nRows = 0);
bool convertStructFlag = true, ColTupleList::size_type nRows = 0, std::vector<ExtCPInfo*>* cpInfos = NULL);
//For update column from column to use
int writeColumnRecords(const TxnID& txnid, const CSCTypesList& cscColTypeList,
std::vector<ColStruct>& colStructList,
ColValueList& colValueList, const RIDList& ridLists,
const int32_t tableOid, bool versioning = true);
const int32_t tableOid, bool versioning = true, std::vector<ExtCPInfo*>* cpInfos = NULL);
/**
* @brief util method to convert rowid to a column file
@ -739,18 +763,24 @@ private:
void AddDictToList(const TxnID txnid, std::vector<BRM::LBID_t>& lbids);
void RemoveTxnFromDictMap(const TxnID txnid);
// Bug 4312: We use a hash set to hold the set of starting LBIDS for a given
// Bug 4312: We use a hash map to hold the set of starting LBIDS for a given
// txn so that we don't waste time marking the same extent as invalid. This
// list should be trimmed if it gets too big.
int AddLBIDtoList(const TxnID txnid,
const ColStruct& colStruct,
const int fbo,
const BRM::CPInfo& cpInfo // there is dummy value for you to use
);
ExtCPInfo* cpInfo = NULL // provide CPInfo pointer if you want max/min updated.
);
// Get CPInfo for given starting LBID and column description structure.
int GetLBIDRange(const BRM::LBID_t startingLBID, const ColStruct& colStruct, ExtCPInfo& cpInfo);
// mark extents of the transaction as invalid. erase transaction from txn->lbidsrec map if requested.
int markTxnExtentsAsInvalid(const TxnID txnid, bool erase = false);
// write LBID's new ranges.
int setExtentsNewMaxMins(const ColSplitMaxMinInfoList& maxMins, bool haveSplit);
int RemoveTxnFromLBIDMap(const TxnID txnid);
int op(int compressionType)