From c9f42fb5cc204b3d1a496d0149d580847c2edf21 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Tue, 29 Oct 2019 01:21:17 -0500 Subject: [PATCH] MCOL-641 PoC version for DECIMAL(38) using BINARY as a basis. --- dbcon/ddlpackage/ddlpkg.cpp | 12 +++++++----- dbcon/joblist/passthrucommand-jl.cpp | 2 +- dbcon/joblist/pcolscan.cpp | 10 ++++------ dbcon/joblist/pcolstep.cpp | 5 ++++- dbcon/mysql/ha_mcs_impl.cpp | 9 ++++++++- primitives/linux-port/column.cpp | 6 +++++- utils/dataconvert/dataconvert.cpp | 10 ++++++++-- utils/rowgroup/rowgroup.h | 1 + writeengine/server/we_ddlcommandproc.cpp | 10 ++++++---- writeengine/server/we_dmlcommandproc.cpp | 4 ++-- writeengine/server/we_dmlcommandproc.h | 6 +++--- writeengine/shared/we_convertor.cpp | 6 +++++- writeengine/shared/we_type.h | 3 ++- writeengine/wrapper/we_colop.cpp | 2 +- writeengine/wrapper/writeengine.cpp | 2 ++ writeengine/wrapper/writeengine.h | 22 +++++++++++----------- 16 files changed, 70 insertions(+), 40 deletions(-) diff --git a/dbcon/ddlpackage/ddlpkg.cpp b/dbcon/ddlpackage/ddlpkg.cpp index 24404c52b..d048fff89 100644 --- a/dbcon/ddlpackage/ddlpkg.cpp +++ b/dbcon/ddlpackage/ddlpkg.cpp @@ -199,14 +199,12 @@ void ColumnDef::convertDecimal() } else if ((fType->fPrecision > 0) && (fType->fPrecision < 3)) { - //dataType = CalpontSystemCatalog::TINYINT; fType->fType = DDL_TINYINT; fType->fLength = 1; } else if (fType->fPrecision < 5 && (fType->fPrecision > 2)) { - //dataType = CalpontSystemCatalog::SMALLINT; fType->fType = DDL_SMALLINT; fType->fLength = 2; } @@ -217,15 +215,19 @@ void ColumnDef::convertDecimal() } else if (fType->fPrecision > 6 && fType->fPrecision < 10) { - //dataType = CalpontSystemCatalog::INT; fType->fType = DDL_INT; fType->fLength = 4; } else if (fType->fPrecision > 9 && fType->fPrecision < 19) { - //dataType = CalpontSystemCatalog::BIGINT; fType->fType = DDL_BIGINT; fType->fLength = 8; } + else if (fType->fPrecision > 19 && fType->fPrecision <39) + { + fType->fType = DDL_BINARY; + fType->fLength = 16; + + } } -} +} // end of namespace diff --git a/dbcon/joblist/passthrucommand-jl.cpp b/dbcon/joblist/passthrucommand-jl.cpp index 222f9e5bd..9fcf79131 100644 --- a/dbcon/joblist/passthrucommand-jl.cpp +++ b/dbcon/joblist/passthrucommand-jl.cpp @@ -76,7 +76,7 @@ PassThruCommandJL::PassThruCommandJL(const PassThruStep& p) case 16: case 32: - tableColumnType = TableColumn::STRING; + tableColumnType = TableColumn::STRING; break; default: diff --git a/dbcon/joblist/pcolscan.cpp b/dbcon/joblist/pcolscan.cpp index 3a9384353..d22fbf8e8 100644 --- a/dbcon/joblist/pcolscan.cpp +++ b/dbcon/joblist/pcolscan.cpp @@ -148,11 +148,6 @@ pColScanStep::pColScanStep( int err, i, mask; BRM::LBIDRange_v::iterator it; - //pthread_mutex_init(&mutex, NULL); - //pthread_mutex_init(&dlMutex, NULL); - //pthread_mutex_init(&cpMutex, NULL); - //pthread_cond_init(&condvar, NULL); - //pthread_cond_init(&condvarWakeupProducer, NULL); finishedSending = false; recvWaiting = 0; recvExited = 0; @@ -178,7 +173,10 @@ pColScanStep::pColScanStep( fColType.colWidth = 8; fIsDict = true; } - else if (fColType.colWidth > 8 && fColType.colDataType != CalpontSystemCatalog::BINARY) + // MCOL-641 WIP + else if (fColType.colWidth > 8 + && fColType.colDataType != CalpontSystemCatalog::BINARY + && fColType.colDataType != CalpontSystemCatalog::DECIMAL) { fColType.colWidth = 8; fIsDict = true; diff --git a/dbcon/joblist/pcolstep.cpp b/dbcon/joblist/pcolstep.cpp index 5b07d9374..c4f09ce4b 100644 --- a/dbcon/joblist/pcolstep.cpp +++ b/dbcon/joblist/pcolstep.cpp @@ -177,7 +177,10 @@ pColStep::pColStep( fColType.colWidth = 8; fIsDict = true; } - else if (fColType.colWidth > 8 && fColType.colDataType != CalpontSystemCatalog::BINARY ) + // WIP MCOL-641 + else if (fColType.colWidth > 8 + && fColType.colDataType != CalpontSystemCatalog::BINARY + && fColType.colDataType != CalpontSystemCatalog::DECIMAL) { fColType.colWidth = 8; fIsDict = true; diff --git a/dbcon/mysql/ha_mcs_impl.cpp b/dbcon/mysql/ha_mcs_impl.cpp index b3018be57..81df0c31e 100644 --- a/dbcon/mysql/ha_mcs_impl.cpp +++ b/dbcon/mysql/ha_mcs_impl.cpp @@ -801,7 +801,14 @@ int fetchNextRow(uchar* buf, cal_table_info& ti, cal_connection_info* ci, bool h case CalpontSystemCatalog::DECIMAL: case CalpontSystemCatalog::UDECIMAL: { - intColVal = row.getIntField(s); + if (row.getPrecision(s) > 18) + { + sscanf(row.getBinaryField(s).c_str(), "%ld",&intColVal); + } + else + { + intColVal = row.getIntField(s); + } storeNumericField(f, intColVal, colType); break; } diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 243858157..130707e09 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -285,7 +285,11 @@ template<> inline bool isEmptyVal<16>(uint8_t type, const uint8_t* ival) // For BINARY { const uint64_t* val = reinterpret_cast(ival); - return ((val[0] == joblist::BINARYEMPTYROW) && (val[1] == joblist::BINARYEMPTYROW)); + // WIP ugly speed hack + return ((val[0] == joblist::BINARYEMPTYROW) && (val[1] == joblist::BINARYEMPTYROW) + || (val[0] == joblist::BIGINTEMPTYROW) && (val[1] == joblist::BIGINTEMPTYROW)) +; + } template<> diff --git a/utils/dataconvert/dataconvert.cpp b/utils/dataconvert/dataconvert.cpp index f89c73d16..af0446616 100644 --- a/utils/dataconvert/dataconvert.cpp +++ b/utils/dataconvert/dataconvert.cpp @@ -1224,8 +1224,12 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, value = (long long) number_int_value(data, colType, pushWarning, noRoundup); break; + // MCOL-641 WIP + // use string2BCD conversion here + set sign case CalpontSystemCatalog::DECIMAL: - if (colType.colWidth == 1) + if (colType.colWidth == 16) + value = data; + else if (colType.colWidth == 1) value = (char) number_int_value(data, colType, pushWarning, noRoundup); else if (colType.colWidth == 2) value = (short) number_int_value(data, colType, pushWarning, noRoundup); @@ -1233,9 +1237,11 @@ DataConvert::convertColumnData(const CalpontSystemCatalog::ColType& colType, value = (int) number_int_value(data, colType, pushWarning, noRoundup); else if (colType.colWidth == 8) value = (long long) number_int_value(data, colType, pushWarning, noRoundup); + else if (colType.colWidth == 32) + value = data; break; - + // MCOL-641 Implement UDECIMAL case CalpontSystemCatalog::UDECIMAL: // UDECIMAL numbers may not be negative diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index c1789dc18..8d89521bd 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -746,6 +746,7 @@ inline int64_t Row::getIntField(uint32_t colIndex) const return *((int64_t*) &data[offsets[colIndex]]); default: + std::cout << "Row::getIntField getColumnWidth(colIndex) " << getColumnWidth(colIndex) << std::endl; idbassert(0); throw std::logic_error("Row::getIntField(): bad length."); } diff --git a/writeengine/server/we_ddlcommandproc.cpp b/writeengine/server/we_ddlcommandproc.cpp index fd5fe2a5d..30b50ff4c 100644 --- a/writeengine/server/we_ddlcommandproc.cpp +++ b/writeengine/server/we_ddlcommandproc.cpp @@ -458,9 +458,11 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err { if (colDefPtr->fType->fPrecision > 18) //@Bug 5717 precision cannot be over 18. { - ostringstream os; - os << "Syntax error: The maximum precision (total number of digits) that can be specified is 18"; - throw std::runtime_error(os.str()); + // WIP MCOL-641 + //ostringstream os; + //os << "Syntax error: The maximum precision (total number of digits) that can be specified is 18"; + //throw std::runtime_error(os.str()); + colDefPtr->convertDecimal(); } else if (colDefPtr->fType->fPrecision < colDefPtr->fType->fScale) { @@ -507,7 +509,7 @@ uint8_t WE_DDLCommandProc::writeCreateSyscolumn(ByteStream& bs, std::string& err } else if (dataType == CalpontSystemCatalog::BINARY - && ! (colDefPtr->fType->fLength == 16 + && ! (colDefPtr->fType->fLength == 16 || colDefPtr->fType->fLength == 32)) { ostringstream os; diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 2592e58c9..15c898bc3 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -412,8 +412,8 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: // call the write engine to write the rows int error = NO_ERROR; - //fWriteEngine.setDebugLevel(WriteEngine::DEBUG_3); - //cout << "inserting a row with transaction id " << txnid.id << endl; + fWEWrapper.setDebugLevel(WriteEngine::DEBUG_3); + cout << "inserting a row with transaction id " << txnid.id << endl; fWEWrapper.setIsInsert(true); fWEWrapper.setBulkFlag(true); fWEWrapper.setTransId(txnid.id); diff --git a/writeengine/server/we_dmlcommandproc.h b/writeengine/server/we_dmlcommandproc.h index 8eb7d2fe2..3786052b1 100644 --- a/writeengine/server/we_dmlcommandproc.h +++ b/writeengine/server/we_dmlcommandproc.h @@ -106,12 +106,12 @@ private: WriteEngineWrapper fWEWrapper; boost::scoped_ptr fRBMetaWriter; std::vector > dbRootExtTrackerVec; - inline bool isDictCol ( execplan::CalpontSystemCatalog::ColType colType ) + inline bool isDictCol ( execplan::CalpontSystemCatalog::ColType &colType ) { if (((colType.colDataType == execplan::CalpontSystemCatalog::CHAR) && (colType.colWidth > 8)) || ((colType.colDataType == execplan::CalpontSystemCatalog::VARCHAR) && (colType.colWidth > 7)) - || ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 18)) - || ((colType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && (colType.precision > 18)) + || ((colType.colDataType == execplan::CalpontSystemCatalog::DECIMAL) && (colType.precision > 65)) + || ((colType.colDataType == execplan::CalpontSystemCatalog::UDECIMAL) && (colType.precision > 65)) || (colType.colDataType == execplan::CalpontSystemCatalog::VARBINARY) || (colType.colDataType == execplan::CalpontSystemCatalog::BLOB) || (colType.colDataType == execplan::CalpontSystemCatalog::TEXT)) diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index 2af0df827..dbe51b1d7 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -633,9 +633,13 @@ void Convertor::convertColType(ColStruct* curStruct) *internalType = WriteEngine::WR_INT; break; - default: + case 8: *internalType = WriteEngine::WR_LONGLONG; break; + + default: + *internalType = WriteEngine::WR_BCDECIMAL; + break; } break; diff --git a/writeengine/shared/we_type.h b/writeengine/shared/we_type.h index 6732b48b1..64cb1d6c3 100644 --- a/writeengine/shared/we_type.h +++ b/writeengine/shared/we_type.h @@ -111,7 +111,8 @@ enum ColType /** @brief Column type enumeration*/ WR_TEXT = 17, /** @brief TEXT */ WR_MEDINT = 18, /** @brief Medium Int */ WR_UMEDINT = 19, /** @brief Unsigned Medium Int */ - WR_BINARY = 20 /** @brief BINARY */ + WR_BINARY = 20, /** @brief BINARY */ + WR_BCDECIMAL = 21 /** @brief BINARY CODED DECIMAL */ }; // Describes relation of field to column for a bulk load diff --git a/writeengine/wrapper/we_colop.cpp b/writeengine/wrapper/we_colop.cpp index 6e4e8c91f..8d7bc3d4d 100644 --- a/writeengine/wrapper/we_colop.cpp +++ b/writeengine/wrapper/we_colop.cpp @@ -1687,9 +1687,9 @@ int ColumnOp::writeRow(Column& curCol, uint64_t totalRow, const RID* rowIdArray, break; case WriteEngine::WR_BINARY: + case WriteEngine::WR_BCDECIMAL: if (!bDelete) pVal = (uint8_t*) valArray + i * curCol.colWidth; - //pOldVal = (uint8_t*) oldValArray + i * curCol.colWidth; break; default : diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index 93a5c1200..620428303 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -503,6 +503,7 @@ void WriteEngineWrapper::convertValue(const ColType colType, void* valArray, con break; case WriteEngine::WR_BINARY: + case WriteEngine::WR_BCDECIMAL: curStr = boost::any_cast(data); memcpy((char*)valArray + pos * curStr.length(), curStr.c_str(), curStr.length()); break; @@ -5157,6 +5158,7 @@ int WriteEngineWrapper::writeColumnRec(const TxnID& txnid, break; case WriteEngine::WR_BINARY: + case WriteEngine::WR_BCDECIMAL: valArray = calloc(colStructList[i].colWidth, totalRow1); break; } diff --git a/writeengine/wrapper/writeengine.h b/writeengine/wrapper/writeengine.h index 864d064b1..229d8c76e 100644 --- a/writeengine/wrapper/writeengine.h +++ b/writeengine/wrapper/writeengine.h @@ -621,7 +621,17 @@ public: */ // todo: add implementation when we work on version control // int endTran(const TransID transOid) { return NO_ERROR; } + // WIP + void setDebugLevel(const DebugLevel level) + { + WEObj::setDebugLevel(level); + for (int i = 0; i < TOTAL_COMPRESS_OP; i++) + { + m_colOp[i]->setDebugLevel(level); + m_dctnry[i]->setDebugLevel(level); + } + } // todo: cleanup /************************************************************************ * Internal use definitions @@ -676,17 +686,7 @@ private: std::vector& freeList, std::vector >& fboLists, std::vector >& rangeLists, std::vector& rangeListTot); - void setDebugLevel(const DebugLevel level) - { - WEObj::setDebugLevel(level); - - for (int i = 0; i < TOTAL_COMPRESS_OP; i++) - { - m_colOp[i]->setDebugLevel(level); - m_dctnry[i]->setDebugLevel(level); - } - } // todo: cleanup - + /** * @brief Common methods to write values to a column */