From 262cd5c501a8696f6fb3d9553e5ee52b8626c5a6 Mon Sep 17 00:00:00 2001 From: Gagan Goel Date: Fri, 3 Jun 2022 13:18:25 -0400 Subject: [PATCH] MCOL-5021 Remove hard-coded values for data type, column width and compression type for the AUX column, and replace them with constants defined in the execplan namespace. --- dbcon/ddlpackageproc/createtableprocessor.cpp | 9 ++- dbcon/execplan/calpontsystemcatalog.h | 8 +++ dbcon/joblist/batchprimitiveprocessor-jl.cpp | 5 +- dbcon/joblist/batchprimitiveprocessor-jl.h | 3 +- dbcon/joblist/columncommand-jl.cpp | 22 ++++++- dbcon/joblist/columncommand-jl.h | 17 +++++- dbcon/joblist/tuple-bps.cpp | 60 +++++++++++++++++-- primitives/linux-port/column.cpp | 2 +- primitives/primproc/batchprimitiveprocessor.h | 2 +- writeengine/bulk/we_bulkload.cpp | 14 ++--- writeengine/server/we_dmlcommandproc.cpp | 24 ++++---- 11 files changed, 129 insertions(+), 37 deletions(-) diff --git a/dbcon/ddlpackageproc/createtableprocessor.cpp b/dbcon/ddlpackageproc/createtableprocessor.cpp index 5677cb569..ee56d20a4 100644 --- a/dbcon/ddlpackageproc/createtableprocessor.cpp +++ b/dbcon/ddlpackageproc/createtableprocessor.cpp @@ -607,14 +607,13 @@ keepGoing: ++iter; } - // MCOL-5021 - // TODO compressionType is hardcoded to 2 (SNAPPY) + // TODO MCOL-5021 compressionType is hardcoded to 2 (SNAPPY) bytestream << (fStartingColOID + size); - bytestream << (uint8_t)datatypes::SystemCatalog::UTINYINT; + bytestream << (uint8_t)execplan::AUX_COL_DATATYPE; bytestream << (uint8_t) false; - bytestream << (uint32_t)1; + bytestream << (uint32_t)execplan::AUX_COL_WIDTH; bytestream << (uint16_t)useDBRoot; - bytestream << (uint32_t)2; + bytestream << (uint32_t)execplan::AUX_COL_COMPRESSION_TYPE; //@Bug 4176. save oids to a log file for cleanup after fail over. std::vector oidList; diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index eabce30a4..a7abf31ef 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -918,6 +918,14 @@ class CalpontSystemCatalog : public datatypes::SystemCatalog static uint32_t fModuleID; }; +// MCOL-5021 +const datatypes::SystemCatalog::ColDataType AUX_COL_DATATYPE = datatypes::SystemCatalog::UTINYINT; +const int32_t AUX_COL_WIDTH = 1; +const CalpontSystemCatalog::CompressionType AUX_COL_COMPRESSION_TYPE = CalpontSystemCatalog::COMPRESSION2; +const std::string AUX_COL_DATATYPE_STRING = "unsigned-tinyint"; +const uint64_t AUX_COL_MINVALUE = MIN_UTINYINT; +const uint64_t AUX_COL_MAXVALUE = MAX_UTINYINT; + /** convenience function to make a TableColName from 3 strings */ const CalpontSystemCatalog::TableColName make_tcn(const std::string& s, const std::string& t, diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.cpp b/dbcon/joblist/batchprimitiveprocessor-jl.cpp index 1faccda6e..6596eb8c5 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.cpp +++ b/dbcon/joblist/batchprimitiveprocessor-jl.cpp @@ -93,12 +93,13 @@ BatchPrimitiveProcessorJL::~BatchPrimitiveProcessorJL() void BatchPrimitiveProcessorJL::addFilterStep(const pColScanStep& scan, vector lastScannedLBID, bool hasAuxCol, - const std::vector& extentsAux) + const std::vector& extentsAux, + execplan::CalpontSystemCatalog::OID oidAux) { SCommand cc; tableOID = scan.tableOid(); - cc.reset(new ColumnCommandJL(scan, lastScannedLBID, hasAuxCol, extentsAux)); + cc.reset(new ColumnCommandJL(scan, lastScannedLBID, hasAuxCol, extentsAux, oidAux)); cc->setBatchPrimitiveProcessor(this); cc->setQueryUuid(scan.queryUuid()); cc->setStepUuid(uuid); diff --git a/dbcon/joblist/batchprimitiveprocessor-jl.h b/dbcon/joblist/batchprimitiveprocessor-jl.h index 6a76332e2..8c3dfd6d3 100644 --- a/dbcon/joblist/batchprimitiveprocessor-jl.h +++ b/dbcon/joblist/batchprimitiveprocessor-jl.h @@ -116,7 +116,8 @@ class BatchPrimitiveProcessorJL } void addFilterStep(const pColScanStep&, std::vector lastScannedLBID, - bool hasAuxCol, const std::vector& extentsAux); + bool hasAuxCol, const std::vector& extentsAux, + execplan::CalpontSystemCatalog::OID oidAux); void addFilterStep(const PseudoColStep&); void addFilterStep(const pColStep&); void addFilterStep(const pDictionaryStep&); diff --git a/dbcon/joblist/columncommand-jl.cpp b/dbcon/joblist/columncommand-jl.cpp index 4b56bda67..2418554b6 100644 --- a/dbcon/joblist/columncommand-jl.cpp +++ b/dbcon/joblist/columncommand-jl.cpp @@ -44,8 +44,10 @@ using namespace messageqcpp; namespace joblist { ColumnCommandJL::ColumnCommandJL(const pColScanStep& scan, vector lastLBID, - bool hasAuxCol_, const std::vector& extentsAux_) : - extentsAux(extentsAux_), hasAuxCol(hasAuxCol_) + bool hasAuxCol_, const std::vector& extentsAux_, + execplan::CalpontSystemCatalog::OID oidAux) : + extentsAux(extentsAux_), hasAuxCol(hasAuxCol_), + fOidAux(oidAux) { BRM::DBRM dbrm; isScan = true; @@ -163,6 +165,8 @@ ColumnCommandJL::ColumnCommandJL(const ColumnCommandJL& prevCmd, const DictStepJ BOP = prevCmd.BOP; } isScan = prevCmd.isScan; + hasAuxCol = prevCmd.hasAuxCol; + extentsAux = prevCmd.extentsAux; colType = prevCmd.colType; extents = prevCmd.extents; OID = prevCmd.OID; @@ -375,6 +379,20 @@ void ColumnCommandJL::reloadExtents() } sort(extents.begin(), extents.end(), BRM::ExtentSorter()); + + if (hasAuxCol) + { + err = dbrm.getExtents(fOidAux, extentsAux); + + if (err) + { + ostringstream os; + os << "BRM lookup error. Could not get extents for Aux OID " << fOidAux; + throw runtime_error(os.str()); + } + + sort(extentsAux.begin(), extentsAux.end(), BRM::ExtentSorter()); + } } bool ColumnCommandJL::getIsDict() diff --git a/dbcon/joblist/columncommand-jl.h b/dbcon/joblist/columncommand-jl.h index f72fff1de..037022581 100644 --- a/dbcon/joblist/columncommand-jl.h +++ b/dbcon/joblist/columncommand-jl.h @@ -41,7 +41,8 @@ class ColumnCommandJL : public CommandJL { public: ColumnCommandJL(const pColScanStep&, std::vector lastLBID, - bool hasAuxCol_, const std::vector& extentsAux_); + bool hasAuxCol_, const std::vector& extentsAux_, + execplan::CalpontSystemCatalog::OID oidAux); ColumnCommandJL(const pColStep&); ColumnCommandJL(const ColumnCommandJL&, const DictStepJL&); virtual ~ColumnCommandJL(); @@ -73,6 +74,10 @@ class ColumnCommandJL : public CommandJL { return extents; } + const std::vector& getExtentsAux() + { + return extentsAux; + } const execplan::CalpontSystemCatalog::ColType& getColType() const { return colType; @@ -90,6 +95,15 @@ class ColumnCommandJL : public CommandJL { return isScan; } + bool auxCol() const + { + return hasAuxCol; + } + + execplan::CalpontSystemCatalog::OID getOIDAux() const + { + return fOidAux; + } void reloadExtents(); @@ -127,6 +141,7 @@ class ColumnCommandJL : public CommandJL std::vector extentsAux; bool hasAuxCol; uint64_t lbidAux; + execplan::CalpontSystemCatalog::OID fOidAux; static const unsigned DEFAULT_FILES_PER_COLUMN_PARTITION = 32; diff --git a/dbcon/joblist/tuple-bps.cpp b/dbcon/joblist/tuple-bps.cpp index c2384df79..f2b173b77 100644 --- a/dbcon/joblist/tuple-bps.cpp +++ b/dbcon/joblist/tuple-bps.cpp @@ -530,12 +530,45 @@ TupleBPS::TupleBPS(const pColScanStep& rhs, const JobInfo& jobInfo) : BatchPrimi extentSize = rhs.extentSize; lbidRanges = rhs.lbidRanges; hasAuxCol = false; + execplan::CalpontSystemCatalog::TableName tableName; - // TODO MCOL-5021 Add try-catch block if (fTableOid >= 3000) { - execplan::CalpontSystemCatalog::TableName tableName = jobInfo.csc->tableName(fTableOid); - fOidAux = jobInfo.csc->tableAUXColumnOID(tableName); + try + { + tableName = jobInfo.csc->tableName(fTableOid); + fOidAux = jobInfo.csc->tableAUXColumnOID(tableName); + } + catch (logging::IDBExcept& ie) + { + std::ostringstream oss; + + if (ie.errorCode() == logging::ERR_TABLE_NOT_IN_CATALOG) + { + oss << "Table " << tableName.toString(); + oss << " does not exist in the system catalog."; + } + else + { + oss << "Error getting AUX column OID for table " << tableName.toString(); + oss << " due to: " << ie.what(); + } + + throw runtime_error(oss.str()); + } + catch(std::exception& ex) + { + std::ostringstream oss; + oss << "Error getting AUX column OID for table " << tableName.toString(); + oss << " due to: " << ex.what(); + throw runtime_error(oss.str()); + } + catch(...) + { + std::ostringstream oss; + oss << "Error getting AUX column OID for table " << tableName.toString(); + throw runtime_error(oss.str()); + } if (fOidAux > 3000) { @@ -546,6 +579,12 @@ TupleBPS::TupleBPS(const pColScanStep& rhs, const JobInfo& jobInfo) : BatchPrimi idbassert(!extentsAux.empty()); sort(extentsAux.begin(), extentsAux.end(), BRM::ExtentSorter()); + + extentsMap[fOidAux] = tr1::unordered_map(); + tr1::unordered_map& refAux = extentsMap[fOidAux]; + + for (uint32_t z = 0; z < extentsAux.size(); z++) + refAux[extentsAux[z].range.start] = extentsAux[z]; } } @@ -849,7 +888,7 @@ void TupleBPS::setBPP(JobStep* jobStep) if (pcss != 0) { - fBPP->addFilterStep(*pcss, lastScannedLBID, hasAuxCol, extentsAux); + fBPP->addFilterStep(*pcss, lastScannedLBID, hasAuxCol, extentsAux, fOidAux); extentsMap[pcss->fOid] = tr1::unordered_map(); tr1::unordered_map& ref = extentsMap[pcss->fOid]; @@ -1279,7 +1318,6 @@ void TupleBPS::initExtentMarkers() } } -// TODO MCOL-5021 Add support here void TupleBPS::reloadExtentLists() { /* @@ -1333,6 +1371,18 @@ void TupleBPS::reloadExtentLists() for (j = 0; j < extents.size(); j++) mref[extents[j].range.start] = extents[j]; + + if (cc->auxCol()) + { + const vector& extentsAux = cc->getExtentsAux(); + oid = cc->getOIDAux(); + + extentsMap[oid] = tr1::unordered_map(); + tr1::unordered_map& mrefAux = extentsMap[oid]; + + for (j = 0; j < extentsAux.size(); j++) + mrefAux[extentsAux[j].range.start] = extentsAux[j]; + } } for (i = 0; i < projections.size(); i++) diff --git a/primitives/linux-port/column.cpp b/primitives/linux-port/column.cpp index 8b4ee5d56..72d36a0bf 100644 --- a/primitives/linux-port/column.cpp +++ b/primitives/linux-port/column.cpp @@ -1959,7 +1959,7 @@ void filterColumnData(NewColRequestHeader* in, ColResultHeader* out, uint16_t* r // Bit patterns in srcArray[i] representing EMPTY and NULL values T emptyValue = getEmptyValue(dataType); T nullValue = getNullValue(dataType); - uint8_t emptyValueAux = getEmptyValue(datatypes::SystemCatalog::UTINYINT); + uint8_t emptyValueAux = getEmptyValue(execplan::AUX_COL_DATATYPE); // Precompute filter results for NULL values bool isNullValueMatches = diff --git a/primitives/primproc/batchprimitiveprocessor.h b/primitives/primproc/batchprimitiveprocessor.h index 78eabe761..59b54818c 100644 --- a/primitives/primproc/batchprimitiveprocessor.h +++ b/primitives/primproc/batchprimitiveprocessor.h @@ -235,7 +235,7 @@ class BatchPrimitiveProcessor /* Common space for primitive data */ alignas(utils::MAXCOLUMNWIDTH) uint8_t blockData[BLOCK_SIZE * utils::MAXCOLUMNWIDTH]; - uint8_t blockDataAux[BLOCK_SIZE]; + uint8_t blockDataAux[BLOCK_SIZE * execplan::AUX_COL_WIDTH]; boost::scoped_array outputMsg; uint32_t outMsgSize; diff --git a/writeengine/bulk/we_bulkload.cpp b/writeengine/bulk/we_bulkload.cpp index e316b21a3..9b8905d6e 100644 --- a/writeengine/bulk/we_bulkload.cpp +++ b/writeengine/bulk/we_bulkload.cpp @@ -379,13 +379,13 @@ int BulkLoad::loadJobInfo(const string& fullName, bool bUseTempJobFile, int argc JobColumn curColumn; curColumn.colName = "aux"; curColumn.mapOid = tableAUXColOid; - curColumn.typeName = "unsigned-tinyint"; - curColumn.width = 1; - curColumn.definedWidth = 1; - curColumn.compressionType = 2; - curColumn.dctnry.fCompressionType = 2; - curColumn.fMinIntSat = MIN_UTINYINT; - curColumn.fMaxIntSat = MAX_UTINYINT; + curColumn.typeName = execplan::AUX_COL_DATATYPE_STRING; + curColumn.width = execplan::AUX_COL_WIDTH; + curColumn.definedWidth = execplan::AUX_COL_WIDTH; + curColumn.compressionType = execplan::AUX_COL_COMPRESSION_TYPE; + curColumn.dctnry.fCompressionType = execplan::AUX_COL_COMPRESSION_TYPE; + curColumn.fMinIntSat = execplan::AUX_COL_MINVALUE; + curColumn.fMaxIntSat = execplan::AUX_COL_MAXVALUE; curColumn.fWithDefault = true; curColumn.fDefaultUInt = 1; curJob.jobTableList[i].colList.push_back(curColumn); diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 92e47a7ff..bedac4e81 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -213,9 +213,9 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std: if (tableAUXColOid > 3000) { CalpontSystemCatalog::ColType colType; - colType.compressionType = 2; - colType.colWidth = 1; - colType.colDataType = datatypes::SystemCatalog::UTINYINT; + colType.compressionType = execplan::AUX_COL_COMPRESSION_TYPE; + colType.colWidth = execplan::AUX_COL_WIDTH; + colType.colDataType = execplan::AUX_COL_DATATYPE; WriteEngine::ColStruct colStruct; colStruct.fColDbRoot = dbroot; WriteEngine::DctnryStruct dctnryStruct; @@ -996,10 +996,10 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std:: if ((i == ridList.size() - 1) && (tableAUXColOid > 3000)) // AUX column { - aColumn.colWidth = 1; - aColumn.colDataType = datatypes::SystemCatalog::UTINYINT; + aColumn.colWidth = execplan::AUX_COL_WIDTH; + aColumn.colDataType = execplan::AUX_COL_DATATYPE; // TODO MCOL-5021 compressionType for the AUX column is hard-coded to 2 - aColumn.compressionType = 2; + aColumn.compressionType = execplan::AUX_COL_COMPRESSION_TYPE; } else { @@ -1153,9 +1153,9 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std:: if (tableAUXColOid > 3000) { CalpontSystemCatalog::ColType colType; - colType.compressionType = 2; - colType.colWidth = 1; - colType.colDataType = datatypes::SystemCatalog::UTINYINT; + colType.compressionType = execplan::AUX_COL_COMPRESSION_TYPE; + colType.colWidth = execplan::AUX_COL_WIDTH; + colType.colDataType = execplan::AUX_COL_DATATYPE; WriteEngine::ColStruct colStruct; WriteEngine::DctnryStruct dctnryStruct; colStruct.dataOid = tableAUXColOid; @@ -4305,9 +4305,9 @@ uint8_t WE_DMLCommandProc::processDelete(messageqcpp::ByteStream& bs, std::strin if (hasAUXCol) { CalpontSystemCatalog::ColType colType; - colType.compressionType = 2; - colType.colWidth = 1; - colType.colDataType = datatypes::SystemCatalog::UTINYINT; + colType.compressionType = execplan::AUX_COL_COMPRESSION_TYPE; + colType.colWidth = execplan::AUX_COL_WIDTH; + colType.colDataType = execplan::AUX_COL_DATATYPE; colStruct.dataOid = tableAUXColOid; colStruct.tokenFlag = false; colStruct.fCompressionType = colType.compressionType;