database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

MCOL-641 atoi128 now correctly processes decimal point and - signs.

Browse Source 
There are multiple overloaded version of the low level DML write methods to
push down CSC column type. WE needs the type to convert values correctly.

Replaced WE_INT128 with CSC data type that is more informative.

Removed commented and obsolete code.

Replaced switch-case blocks with oneliners.
This commit is contained in:
drrtuy
2020-01-12 15:11:31 +03:00
committed by Roman Nozdrin
parent 49a5573418
commit 0c67b6ab50
8 changed files with 1475 additions and 362 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
writeengine/server/we_dmlcommandproc.cpp
View File

@ -1,5 +1,5 @@
/* Copyright (C) 2014 InfiniDB, Inc.
Copyright (C) 2016 MariaDB Corporation
Copyright (C) 2016-2019 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
@ -116,6 +116,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
RowList rows = tablePtr->get_RowList();
WriteEngine::ColStructList colStructs;
WriteEngine::CSCTypesList cscColTypes;
WriteEngine::DctnryStructList dctnryStructList;
WriteEngine::DctnryValueList dctnryValueList;
WriteEngine::ColValueList colValuesList;
@ -139,12 +140,18 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
{
Row* rowPtr = rows.at(0);
ColumnList columns = rowPtr->get_ColumnList();
unsigned int numcols = rowPtr->get_NumberOfColumns();
cscColTypes.reserve(numcols);
// WIP
// We presume that DictCols number is low
colStructs.reserve(numcols);
ColumnList::const_iterator column_iterator = columns.begin();
while (column_iterator != columns.end())
{
DMLColumn* columnPtr = *column_iterator;
tableColName.column = columnPtr->get_Name();
// WIP MCOL-641 replace with getColRidsOidsTypes()
CalpontSystemCatalog::ROPair roPair = systemCatalogPtr->columnRID(tableColName);
CalpontSystemCatalog::OID oid = systemCatalogPtr->lookupOID(tableColName);
@ -152,6 +159,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
CalpontSystemCatalog::ColType colType;
colType = systemCatalogPtr->colType(oid);
cscColTypes.push_back(colType);
WriteEngine::ColStruct colStruct;
colStruct.fColDbRoot = dbroot;
WriteEngine::DctnryStruct dctnryStruct;
@ -163,6 +171,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
// Token
if ( isDictCol(colType) )
{
// WIP Hardcoded value
colStruct.colWidth = 8;
colStruct.tokenFlag = true;
}
@ -194,7 +203,6 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
++column_iterator;
}
unsigned int numcols = rowPtr->get_NumberOfColumns();
std::string tmpStr("");
for (unsigned int i = 0; i < numcols; i++)
@ -210,6 +218,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
const DMLColumn* columnPtr = rowPtr->get_ColumnAt(i);
tableColName.column = columnPtr->get_Name();
// WIP MCOL-641 remove these calls
CalpontSystemCatalog::OID oid = systemCatalogPtr->lookupOID(tableColName);
CalpontSystemCatalog::ColType colType;
@ -303,6 +312,8 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
{
try
{
// WIP What if we combine this and previous loop and fail
// after get nextAIValue ?
nextVal = systemCatalogPtr->nextAutoIncrValue(tableName);
fDbrm.startAISequence(oid, nextVal, colType.colWidth, colType.colDataType);
}
@ -359,6 +370,8 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
try
{
// WIP
// make convertColumnData a template
datavalue = DataConvert::convertColumnData(colType, indata, pushWarning, insertPkg.get_TimeZone(), isNULL, false, false);
}
catch (exception&)
@ -412,6 +425,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
// call the write engine to write the rows
int error = NO_ERROR;
// WIP
fWEWrapper.setDebugLevel(WriteEngine::DEBUG_3);
cout << "inserting a row with transaction id " << txnid.id << endl;
fWEWrapper.setIsInsert(true);
@ -420,6 +434,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
//For hdfs use only
uint32_t tblOid = tableRoPair.objnum;
// WIP are we saving HDFS?
if (idbdatafile::IDBPolicy::useHdfs())
{
@ -523,7 +538,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
if (colValuesList[0].size() > 0)
{
if (NO_ERROR !=
(error = fWEWrapper.insertColumnRec_Single(txnid.id, colStructs, colValuesList, dctnryStructList, dicStringList, tableRoPair.objnum)))
(error = fWEWrapper.insertColumnRec_Single(txnid.id, cscColTypes, colStructs, colValuesList, dctnryStructList, dicStringList, tableRoPair.objnum)))
{
if (error == ERR_BRM_DEAD_LOCK)
{

3
writeengine/shared/we_convertor.cpp
View File

@ -639,7 +639,8 @@ void Convertor::convertColType(ColStruct* curStruct)
default:
// WIP replace with BINARY
*internalType = WriteEngine::WR_INT128;
//*internalType = WriteEngine::WR_INT128;
*internalType = WriteEngine::WR_BINARY;
break;
}

7
writeengine/shared/we_type.h
View File

@ -111,9 +111,9 @@ enum ColType /** @brief Column type enumeration*/
WR_TEXT = 17, /** @brief TEXT */
WR_MEDINT = 18, /** @brief Medium Int */
WR_UMEDINT = 19, /** @brief Unsigned Medium Int */
WR_BINARY = 20, /** @brief BINARY */
// WIP We might be good using WR_BINARY
WR_INT128 = 21 /** @brief __int128 */
WR_BINARY = 20 /** @brief BINARY */
// WIP
//WR_INT128
};
// Describes relation of field to column for a bulk load
@ -302,6 +302,7 @@ struct ColStruct /** @brief Column Interface Struct*/
typedef std::vector<ColStruct> ColStructList; /** @brief column struct list */
typedef std::vector<ColTupleList> ColValueList; /** @brief column value list */
typedef std::vector<RID> RIDList; /** @brief RID list */
typedef std::vector<execplan::CalpontSystemCatalog::ColType> CSCTypesList; /** @brief CSC column types list */
typedef std::vector<std::string> dictStr;
typedef std::vector<dictStr> DictStrList;

9
writeengine/wrapper/we_colop.cpp
View File

@ -1684,12 +1684,13 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray,
if (!bDelete) pVal = &((uint64_t*) valArray)[i];
break;
case WriteEngine::WR_INT128:
pVal = &((uint128_t*) valArray)[i];
break;
// WIP
//case WriteEngine::WR_INT128:
case WriteEngine::WR_BINARY:
if (!bDelete) pVal = (uint8_t*) valArray + i * curCol.colWidth;
// WIP CSCCol type
pVal = &((uint128_t*) valArray)[i];
//pVal = (uint8_t*) valArray + i * curCol.colWidth;
break;
default :

1522
writeengine/wrapper/writeengine.cpp
View File

File diff suppressed because it is too large Load Diff

39
writeengine/wrapper/writeengine.h
View File

@ -157,7 +157,15 @@ public:
/**
* @brief Convert interface value list to internal value array
*/
EXPORT void convertValArray(size_t totalRow, const ColType colType,
EXPORT void convertValArray(const size_t totalRow,
const execplan::CalpontSystemCatalog::ColType& cscColType,
const ColType colType,
ColTupleList& curTupleList, void* valArray,
bool bFromList = true) ;
// WIP legacy
EXPORT void convertValArray(const size_t totalRow,
const ColType colType,
ColTupleList& curTupleList, void* valArray,
bool bFromList = true) ;
@ -364,6 +372,7 @@ public:
* @param dicStringListt dictionary values list
*/
EXPORT int insertColumnRec_Single(const TxnID& txnid,
CSCTypesList& cscColTypesList,
ColStructList& colStructList,
ColValueList& colValueList,
DctnryStructList& dctnryStructList,
@ -650,10 +659,8 @@ private:
/**
* @brief Convert interface column type to a internal column type
*/
// void convertColType(void* curStruct, const FuncType curType = FUNC_WRITE_ENGINE) const;
void convertValue(const execplan::CalpontSystemCatalog::ColType &fullColType, ColType colType, void* valArray, size_t pos, boost::any& data, bool fromList = true);
void convertValue(const ColType colType, void* valArray, size_t pos, boost::any& data, bool fromList = true);
/**
* @brief Convert column value to its internal representation
*
@ -661,6 +668,7 @@ private:
* @param value Memory pointer for storing output value. Should be pre-allocated
* @param data Column data
*/
void convertValue(const execplan::CalpontSystemCatalog::ColType &fullColType, const ColType colType, void* value, boost::any& data);
void convertValue(const ColType colType, void* value, boost::any& data);
/**
@ -690,11 +698,21 @@ private:
/**
* @brief Common methods to write values to a column
*/
int writeColumnRec(const TxnID& txnid, const ColStructList& colStructList,
int writeColumnRec(const TxnID& txnid,
const CSCTypesList& cscColTypes,
const ColStructList& colStructList,
ColValueList& colValueList,
RID* rowIdArray, const ColStructList& newColStructList,
ColValueList& newColValueList, const int32_t tableOid,
bool useTmpSuffix, bool versioning = true);
// WIP
int writeColumnRec(const TxnID& txnid,
const ColStructList& colStructList,
ColValueList& colValueList,
RID* rowIdArray, const ColStructList& newColStructList,
ColValueList& newColValueList, const int32_t tableOid,
bool useTmpSuffix, bool versioning = true);
int writeColumnRecBinary(const TxnID& txnid, const ColStructList& colStructList,
std::vector<uint64_t>& colValueList,
@ -705,10 +723,19 @@ private:
//@Bug 1886,2870 pass the address of ridList vector
int writeColumnRec(const TxnID& txnid, const ColStructList& colStructList,
int writeColumnRec(const TxnID& txnid,
const CSCTypesList& cscColTypes,
const ColStructList& colStructList,
const ColValueList& colValueList, std::vector<void*>& colOldValueList,
const RIDList& ridList, const int32_t tableOid,
bool convertStructFlag = true, ColTupleList::size_type nRows = 0);
// WIP legacy
int writeColumnRec(const TxnID& txnid,
const ColStructList& colStructList,
const ColValueList& colValueList, std::vector<void*>& colOldValueList,
const RIDList& ridList, const int32_t tableOid,
bool convertStructFlag = true, ColTupleList::size_type nRows = 0);
//For update column from column to use
int writeColumnRecords(const TxnID& txnid, std::vector<ColStruct>& colStructList,