From 558034b4d6eb9b6dc2a9225aaac4fb285113302f Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 27 Nov 2018 09:50:08 -0600 Subject: [PATCH 1/7] MCOL-1797 - fix issue with dual ddl/dml active states on resumedb --- oamapps/mcsadmin/mcsadmin.cpp | 36 ++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 29d53b35f..459044330 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -3988,21 +3988,27 @@ int processCommand(string* arguments) dbrm.setSystemSuspended(false); oam.getProcessStatus(systemprocessstatus); - for( unsigned int i = 0 ; i < systemprocessstatus.processstatus.size(); i++) - { - if (systemprocessstatus.processstatus[i].ProcessName == "DMLProc") - { - oam.setProcessStatus(systemprocessstatus.processstatus[i].ProcessName, systemprocessstatus.processstatus[i].Module, ACTIVE, 1); - } - if (systemprocessstatus.processstatus[i].ProcessName == "DDLProc") - { - oam.setProcessStatus(systemprocessstatus.processstatus[i].ProcessName, systemprocessstatus.processstatus[i].Module, ACTIVE, 1); - } - if (systemprocessstatus.processstatus[i].ProcessName == "WriteEngineServer") - { - oam.setProcessStatus(systemprocessstatus.processstatus[i].ProcessName, systemprocessstatus.processstatus[i].Module, ACTIVE, 1); - } - } + + for ( unsigned int i = 0 ; i < systemprocessstatus.processstatus.size(); i++) + { + if ( (systemprocessstatus.processstatus[i].ProcessName == "DDLProc") && + (systemprocessstatus.processstatus[i].ProcessOpState == oam::ACTIVE) ) + { + oam.setProcessStatus(systemprocessstatus.processstatus[i].ProcessName, systemprocessstatus.processstatus[i].Module, oam::ACTIVE, 1); + } + + if ( (systemprocessstatus.processstatus[i].ProcessName == "DMLProc") && + (systemprocessstatus.processstatus[i].ProcessOpState == oam::ACTIVE) ) + { + oam.setProcessStatus(systemprocessstatus.processstatus[i].ProcessName, systemprocessstatus.processstatus[i].Module, oam::ACTIVE, 1); + } + + if (systemprocessstatus.processstatus[i].ProcessName == "WriteEngineServer") + { + oam.setProcessStatus(systemprocessstatus.processstatus[i].ProcessName, systemprocessstatus.processstatus[i].Module, oam::ACTIVE, 1); + } + } + oam.setSystemStatus(ACTIVE); cout << endl << "Resume MariaDB ColumnStore Database Writes Request successfully completed" << endl; } From 9e0a72ccd8f9228f85c06c7552208426104eac50 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 27 Nov 2018 10:24:01 -0600 Subject: [PATCH 2/7] MCOL-1797 - change calpont to mariadb columnstore --- oam/oamcpp/liboamcpp.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index baa8dbe94..e793b5efe 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -3068,7 +3068,7 @@ namespace oam switch (returnStatus) { case API_SUCCESS: - cout << endl << "Suspend Calpont Database Writes Request successfully completed" << endl; + cout << endl << "Suspend MariaDB Columnstore Database Writes Request successfully completed" << endl; break; case API_FAILURE_DB_ERROR: cout << endl << "**** stopDatabaseWrites Failed: save_brm Failed" << endl; @@ -10096,7 +10096,7 @@ namespace oam } catch (...) { - writeLog("Communication with calpont console failed while waiting for transactions", LOG_TYPE_ERROR); + writeLog("Communication with MariaDB ColumnStore Admin console failed while waiting for transactions", LOG_TYPE_ERROR); } // writeLog("Returning from wait with value " + itoa(ret), LOG_TYPE_INFO ); return ret; From ba6204179570a4430e538b783b0590cadc1ef5d6 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 27 Nov 2018 13:16:57 -0600 Subject: [PATCH 3/7] MCOL-1797 - fix issue with dual ddl/dml active states on resumedb --- oamapps/mcsadmin/mcsadmin.cpp | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 459044330..f2ea97596 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -3982,33 +3982,10 @@ int processCommand(string* arguments) // resume writes to MariaDB ColumnStore Database try{ - SystemProcessStatus systemprocessstatus; BRM::DBRM dbrm; dbrm.setSystemSuspended(false); - oam.getProcessStatus(systemprocessstatus); - - for ( unsigned int i = 0 ; i < systemprocessstatus.processstatus.size(); i++) - { - if ( (systemprocessstatus.processstatus[i].ProcessName == "DDLProc") && - (systemprocessstatus.processstatus[i].ProcessOpState == oam::ACTIVE) ) - { - oam.setProcessStatus(systemprocessstatus.processstatus[i].ProcessName, systemprocessstatus.processstatus[i].Module, oam::ACTIVE, 1); - } - - if ( (systemprocessstatus.processstatus[i].ProcessName == "DMLProc") && - (systemprocessstatus.processstatus[i].ProcessOpState == oam::ACTIVE) ) - { - oam.setProcessStatus(systemprocessstatus.processstatus[i].ProcessName, systemprocessstatus.processstatus[i].Module, oam::ACTIVE, 1); - } - - if (systemprocessstatus.processstatus[i].ProcessName == "WriteEngineServer") - { - oam.setProcessStatus(systemprocessstatus.processstatus[i].ProcessName, systemprocessstatus.processstatus[i].Module, oam::ACTIVE, 1); - } - } - oam.setSystemStatus(ACTIVE); cout << endl << "Resume MariaDB ColumnStore Database Writes Request successfully completed" << endl; } From 3fc1893ce9026722b16b8eb791df7c9bc9c8e637 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Wed, 5 Dec 2018 13:39:21 -0800 Subject: [PATCH 4/7] MCOL-1347 BRM_INFO is fixed and could be used for BRM debugging. --- versioning/BRM/dbrm.cpp | 12 ++++++------ versioning/BRM/extentmap.cpp | 25 ++++++++++++------------- versioning/BRM/tracer.cpp | 2 ++ 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/versioning/BRM/dbrm.cpp b/versioning/BRM/dbrm.cpp index aa452d39f..742f01089 100644 --- a/versioning/BRM/dbrm.cpp +++ b/versioning/BRM/dbrm.cpp @@ -1647,10 +1647,10 @@ int DBRM::deletePartition(const std::vector& oids, { TRACER_WRITENOW("deletePartition"); std::ostringstream oss; - oss << "partitionNum: " + oss << "partitionNum: "; std::set::const_iterator partIt; for (partIt = partitionNums.begin(); partIt != partitionNums.end(); ++partIt) - oss << (*it) << " " + oss << (*partIt) << " "; oss << "; OIDS: "; std::vector::const_iterator it; for (it=oids.begin(); it!=oids.end(); ++it) @@ -1696,10 +1696,10 @@ int DBRM::markPartitionForDeletion(const std::vector& oids, { TRACER_WRITENOW("markPartitionForDeletion"); std::ostringstream oss; - oss << "partitionNum: " + oss << "partitionNum: "; std::set::const_iterator partIt; for (partIt = partitionNums.begin(); partIt != partitionNums.end(); ++partIt) - oss << (*it) << " " + oss << (*partIt) << " "; oss << "; OIDS: "; std::vector::const_iterator it; for (it=oids.begin(); it!=oids.end(); ++it) @@ -1784,10 +1784,10 @@ int DBRM::restorePartition(const std::vector& oids, { TRACER_WRITENOW("restorePartition"); std::ostringstream oss; - oss << "partitionNum: " + oss << "partitionNum: "; std::set::const_iterator partIt; for (partIt = partitionNums.begin(); partIt != partitionNums.end(); ++partIt) - oss << (*it) << " " + oss << (*partIt) << " "; oss << "; OIDS: "; std::vector::const_iterator it; for (it=oids.begin(); it!=oids.end(); ++it) diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index b464cbeef..8792b20dd 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -59,7 +59,6 @@ namespace bi=boost::interprocess; #include "IDBDataFile.h" #include "IDBPolicy.h" #ifdef BRM_INFO - #error BRM_INFO is broken right now #include "tracer.h" #include "configcpp.h" #endif @@ -4113,12 +4112,12 @@ void ExtentMap::setLocalHWM(int OID, uint32_t partitionNum, if ((oldHWMExtentIndex != -1) && (oldHWMExtentIndex != lastExtentIndex)) { makeUndoRecord(&fExtentMap[oldHWMExtentIndex], sizeof(EMEntry)); fExtentMap[oldHWMExtentIndex].HWM = 0; -#ifdef BRM_DEBUG +#ifdef BRM_INFO addedAnExtent = true; #endif } -#ifdef BRM_DEBUG +#ifdef BRM_INFO if (firstNode) { ostringstream os; os << "ExtentMap::setLocalHWM(): firstLBID=" << fExtentMap[lastExtentIndex].range.start << @@ -4381,9 +4380,9 @@ void ExtentMap::deletePartition(const set& oids, TRACER_WRITENOW("deletePartition"); ostringstream oss; set::const_iterator partIt; - oss << "partitionNums: " - for (partIt=partitionNums.begin(); it!=partitionNums.end(); ++it) - oss << (*it) << " "; + oss << "partitionNums: "; + for (partIt=partitionNums.begin(); partIt!=partitionNums.end(); ++partIt) + oss << (*partIt) << " "; oss << endl; oss << "OIDS: "; @@ -4474,9 +4473,9 @@ void ExtentMap::markPartitionForDeletion(const set& oids, TRACER_WRITENOW("markPartitionForDeletion"); ostringstream oss; set::const_iterator partIt; - oss << "partitionNums: " - for (partIt=partitionNums.begin(); it!=partitionNums.end(); ++it) - oss << (*it) << " "; + oss << "partitionNums: "; + for (partIt=partitionNums.begin(); partIt!=partitionNums.end(); ++partIt) + oss << (*partIt) << " "; oss << endl; oss << "OIDS: "; @@ -4618,9 +4617,9 @@ void ExtentMap::restorePartition(const set& oids, TRACER_WRITENOW("restorePartition"); ostringstream oss; set::const_iterator partIt; - oss << "partitionNums: " - for (partIt=partitionNums.begin(); it!=partitionNums.end(); ++it) - oss << (*it) << " "; + oss << "partitionNums: "; + for (partIt=partitionNums.begin(); partIt!=partitionNums.end(); ++partIt) + oss << (*partIt) << " "; oss << endl; oss << "OIDS: "; set::const_iterator it; @@ -4768,7 +4767,7 @@ bool ExtentMap::isDBRootEmpty(uint16_t dbroot) if (fDebug) { TRACER_WRITELATER("isDBRootEmpty"); - TRACER_ADDINPUT(OID); + TRACER_ADDINPUT(dbroot); TRACER_WRITE; } #endif diff --git a/versioning/BRM/tracer.cpp b/versioning/BRM/tracer.cpp index 33693f62f..66ab2e969 100644 --- a/versioning/BRM/tracer.cpp +++ b/versioning/BRM/tracer.cpp @@ -30,6 +30,8 @@ #include #endif +#include + #define TRACER_DLLEXPORT #include "tracer.h" #undef TRACER_DLLEXPORT From d807aaee0a7de36020c53997464c38cdf36d0d65 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Wed, 5 Dec 2018 13:47:49 -0800 Subject: [PATCH 5/7] MCOL-1347 ALTER TABLE ADD COLUMN now creates a column with correct width for a varchar columns. --- writeengine/wrapper/writeengine.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/writeengine/wrapper/writeengine.cpp b/writeengine/wrapper/writeengine.cpp index d163d1f42..368585d63 100644 --- a/writeengine/wrapper/writeengine.cpp +++ b/writeengine/wrapper/writeengine.cpp @@ -562,6 +562,7 @@ int WriteEngineWrapper::fillColumn(const TxnID& txnid, const OID& dataOid, // refColOp.reset(colOpRefCol); // dctnry.reset(dctOp); uint16_t dbRoot = 1; //not to be used + int newDataWidth = dataWidth; //Convert HWM of the reference column for the new column //Bug 1703,1705 bool isToken = false; @@ -584,10 +585,17 @@ int WriteEngineWrapper::fillColumn(const TxnID& txnid, const OID& dataOid, isToken = true; } + newDataWidth = colOpNewCol->getCorrectRowWidth(dataType, dataWidth); + // MCOL-1347 CS doubles the width for ALTER TABLE..ADD COLUMN + if ( dataWidth < 4 && dataType == CalpontSystemCatalog::VARCHAR ) + { + newDataWidth >>= 1; + } + Convertor::convertColType(refColDataType, refColType, isToken); refColOp->setColParam(refCol, 0, refColOp->getCorrectRowWidth(refColDataType, refColWidth), refColDataType, refColType, (FID)refColOID, refCompressionType, dbRoot); - colOpNewCol->setColParam(newCol, 0, colOpNewCol->getCorrectRowWidth(dataType, dataWidth), + colOpNewCol->setColParam(newCol, 0, newDataWidth, dataType, newColType, (FID)dataOid, compressionType, dbRoot); int size = sizeof(Token); From 7989ccefe0052fa87893e1d72e3075d90ad010f5 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 12 Dec 2018 18:06:09 +0000 Subject: [PATCH 6/7] MCOL-2018 Fix array bounds issue StringStore NULL check includes a check for _CpNuLl_ in the StringStore. This is a case should never happen but we keep it just in case. Unfortunately this check was skipping 4*8 bytes instead of just 4 bytes. This is definitely bad behaviour but it could cause an out-of-bounds read based crash. --- utils/rowgroup/rowgroup.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index 7aca0c93f..37ca2c403 100755 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -1617,7 +1617,7 @@ inline bool StringStore::isNullValue(uint64_t off) const return true; if (mc->data[offset+4] == 0) // "" = NULL string for some reason... return true; - return (*((uint64_t *) &mc->data[offset]+4) == *((uint64_t *) joblist::CPNULLSTRMARK.c_str())); + return (memcmp(&mc->data[offset+4], joblist::CPNULLSTRMARK.c_str(), 8) == 0); } inline bool StringStore::equals(const std::string &str, uint64_t off) const From 177b472ad4b9c726c52e9c4a5f93425bd7780cce Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Tue, 18 Dec 2018 15:23:55 -0600 Subject: [PATCH 7/7] MCOL-2007: add gitversionEngine file to builds/packages --- CMakeLists.txt | 8 ++++++++ gitversionEngine.in | 1 + 2 files changed, 9 insertions(+) create mode 100644 gitversionEngine.in diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f1d2b646..2792aae79 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,6 +83,14 @@ INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum DESTINATION ${INSTALL CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h.in ${CMAKE_CURRENT_SOURCE_DIR}/columnstoreversion.h) CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/config.h) +exec_program("git" + ${CMAKE_CURRENT_SOURCE_DIR} + ARGS "describe --match=NeVeRmAtCh --always --dirty" + OUTPUT_VARIABLE GIT_VERSION) + +CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/gitversionEngine.in ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine IMMEDIATE) +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/gitversionEngine DESTINATION ${INSTALL_ENGINE} COMPONENT platform) + INCLUDE(bison.cmake) FIND_PROGRAM(LEX_EXECUTABLE flex DOC "path to the flex executable") diff --git a/gitversionEngine.in b/gitversionEngine.in new file mode 100644 index 000000000..b6089ee10 --- /dev/null +++ b/gitversionEngine.in @@ -0,0 +1 @@ +@GIT_VERSION@ \ No newline at end of file