From 3ba6c5e8047bbaffbaa6eaf7ade44930d6cd5f8e Mon Sep 17 00:00:00 2001 From: david hill Date: Tue, 22 Aug 2017 13:07:45 -0500 Subject: [PATCH 01/36] change to look for packages versus gz file --- oamapps/postConfigure/postConfigure.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 4390db2f5..464d8336f 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -2766,15 +2766,13 @@ int main(int argc, char *argv[]) } //check if pkgs are located in $HOME directory - string version = systemsoftware.Version + "-" + systemsoftware.Release; if ( EEPackageType == "rpm") - columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.rpm.tar.gz"; + columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.rpm"; else - if ( EEPackageType == "deb") - columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.deb.tar.gz"; + if ( EEPackageType == "deb") + columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.deb"; else - columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.bin.tar.gz"; - + columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.bin.tar.gz"; if( !pkgCheck(columnstorePackage) ) exit(1); From 37d1a1683778d76833be39f063f3ff0d759d0466 Mon Sep 17 00:00:00 2001 From: david hill Date: Tue, 22 Aug 2017 16:59:18 -0500 Subject: [PATCH 02/36] MCOL-765 - fix debian 9 nonroot issue with rc.local --- oam/install_scripts/post-install | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install index bc19e3677..027eaef1f 100755 --- a/oam/install_scripts/post-install +++ b/oam/install_scripts/post-install @@ -171,6 +171,17 @@ rm -rf $installdir/mysql/db/columnstore_log_archive > /dev/null 2>&1 # delete Columnstore shared memory segments $installdir/bin/clearShm > /dev/null 2>&1 +#check and create rc.local file if missing +if [ ! -f /etc/rc.local ]; then + printf '%s\n' '#!/bin/bash' "#" | $SUDO tee -a /etc/rc.local > /dev/null 2>&1 + $SUDO chmod +x /etc/rc.local + + if [ -n "$systemctl" ]; then + systemctl restart columnstore >/dev/null 2>&1 + systemctl enable columnstore >/dev/null 2>&1 + fi +fi + #setup the columnstore service script rm -f /etc/init.d/columnstore >/dev/null 2>&1 rm -f /etc/init.d/mysql-Columnstore >/dev/null 2>&1 From 80aff1477a0f4ec1c144c96039e917270d29cb82 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 23 Aug 2017 13:57:13 -0500 Subject: [PATCH 03/36] MCOL-837 - removed change for partiion number --- oamapps/postConfigure/postConfigure.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 464d8336f..04cc66e30 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -648,8 +648,6 @@ int main(int argc, char *argv[]) exit(1); } - checkFilesPerPartion(DBRootCount, sysConfig); - //check if dbrm data resides in older directory path and inform user if it does dbrmDirCheck(); From 6f66b0e0ebf9df2db57d2a1b1c1dff283ecd39ad Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 25 Aug 2017 13:15:03 -0500 Subject: [PATCH 04/36] fix build issue, version variable is missing --- oamapps/postConfigure/postConfigure.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 04cc66e30..35196edf8 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -2764,6 +2764,7 @@ int main(int argc, char *argv[]) } //check if pkgs are located in $HOME directory + string version = systemsoftware.Version + "-" + systemsoftware.Release; if ( EEPackageType == "rpm") columnstorePackage = HOME + "/" + "mariadb-columnstore-" + version + "*.rpm"; else From 7e15f1c296a9dd2096819640d433efc2fd6135a3 Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 28 Aug 2017 13:40:19 -0500 Subject: [PATCH 05/36] MCOL-892 - changed from using the flock command --- VERSION | 2 +- oam/oamcpp/liboamcpp.cpp | 69 ++++-- oamapps/alarmmanager/alarmmanager.cpp | 343 ++++++++++++++++---------- procmon/processmonitor.cpp | 63 +++-- 4 files changed, 291 insertions(+), 186 deletions(-) diff --git a/VERSION b/VERSION index a405d8f12..dbe1148f8 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=0 COLUMNSTORE_VERSION_PATCH=11 -COLUMNSTORE_VERSION_RELEASE=1 +COLUMNSTORE_VERSION_RELEASE=2 diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index aa8dd4804..6e9175e9a 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -2523,6 +2523,9 @@ namespace oam string Section = "AlarmConfig"; int returnValue; + struct flock fl; + int fd; + // validate Alarm ID if( alarmid > MAX_ALARM_ID ) @@ -2543,41 +2546,55 @@ namespace oam exceptionControl("setAlarmConfig", API_READONLY_PARAMETER); string fileName = AlarmConfigFile; - - int fd = open(fileName.c_str(), O_RDWR|O_CREAT, 0644); - // Aquire an exclusive lock - if (flock(fd,LOCK_EX) == -1) { - throw runtime_error ("Lock file error: " + fileName); - } + memset(&fl, 0, sizeof(fl)); + fl.l_type = F_RDLCK; // read lock + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; //lock whole file - // write parameter to disk - - Config* alaConfig = Config::makeConfig(AlarmConfigFile.c_str()); - alaConfig->setConfig(Section, name, value); - - try - { - alaConfig->write(); - } - catch(...) - { - // Release lock - if (flock(fd,LOCK_UN)==-1) + // open config file + if ((fd = open(fileName.c_str(), O_RDWR)) >= 0) + { // lock file + if (fcntl(fd, F_SETLKW, &fl) != 0) { - throw runtime_error ("Release lock file error: " + fileName); + ostringstream oss; + oss << "Oam::setAlarmConfig: error locking file " << + fileName << + ": " << + strerror(errno) << + ", proceding anyway."; + cerr << oss.str() << endl; } + + // write parameter to disk - exceptionControl("setAlarmConfig", API_FAILURE); - } + Config* alaConfig = Config::makeConfig(AlarmConfigFile.c_str()); + alaConfig->setConfig(Section, name, value); + + try + { + alaConfig->write(); + } + catch(...) + {} - // Release lock - if (flock(fd,LOCK_UN)==-1) + fl.l_type = F_UNLCK; //unlock + fcntl(fd, F_SETLK, &fl); + + close(fd); + } + else { - throw runtime_error ("Release lock file error: " + fileName); + ostringstream oss; + oss << "Oam::setAlarmConfig: error opening file " << + fileName << + ": " << + strerror(errno); + throw runtime_error(oss.str()); } - close(fd); + return; } /******************************************************************** diff --git a/oamapps/alarmmanager/alarmmanager.cpp b/oamapps/alarmmanager/alarmmanager.cpp index 5d3990b2d..49f6c8638 100644 --- a/oamapps/alarmmanager/alarmmanager.cpp +++ b/oamapps/alarmmanager/alarmmanager.cpp @@ -97,6 +97,9 @@ ALARMManager::~ALARMManager() *****************************************************************************************/ void rewriteActiveLog (const AlarmList& alarmList) { + struct flock fl; + int fd; + if (ALARM_DEBUG) { LoggingID lid(11); MessageLog ml(lid); @@ -110,30 +113,52 @@ void rewriteActiveLog (const AlarmList& alarmList) // delete the old file unlink (ACTIVE_ALARM_FILE.c_str()); + memset(&fl, 0, sizeof(fl)); + fl.l_type = F_RDLCK; // read lock + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; //lock whole file + // create new file - int fd = open(ACTIVE_ALARM_FILE.c_str(), O_RDWR|O_CREAT, 0644); - - // Aquire an exclusive lock - if (flock(fd,LOCK_EX) == -1) { - throw runtime_error ("Lock active alarm log file error"); - } + if ((fd = open(ACTIVE_ALARM_FILE.c_str(), O_RDWR|O_CREAT, 0664)) >= 0) + { // lock file + if (fcntl(fd, F_SETLKW, &fl) != 0) + { + ostringstream oss; + oss << "rewriteActiveLog: error locking file " << + ACTIVE_ALARM_FILE << + ": " << + strerror(errno) << + ", proceding anyway."; + cerr << oss.str() << endl; + } + + ofstream activeAlarmFile (ACTIVE_ALARM_FILE.c_str()); + + AlarmList::const_iterator i; + for (i = alarmList.begin(); i != alarmList.end(); ++i) + { + activeAlarmFile << i->second; + } - ofstream activeAlarmFile (ACTIVE_ALARM_FILE.c_str()); - - AlarmList::const_iterator i; - for (i = alarmList.begin(); i != alarmList.end(); ++i) - { - activeAlarmFile << i->second; - } + activeAlarmFile.close(); - activeAlarmFile.close(); + fl.l_type = F_UNLCK; //unlock + fcntl(fd, F_SETLK, &fl); - // Release lock - if (flock(fd,LOCK_UN)==-1) - { - throw runtime_error ("Release lock active alarm log file error"); + close(fd); } - close(fd); + else + { + ostringstream oss; + oss << "rewriteActiveLog: error opening file " << + ACTIVE_ALARM_FILE << + ": " << + strerror(errno); + throw runtime_error(oss.str()); + } + + return; } /***************************************************************************************** @@ -144,6 +169,9 @@ void rewriteActiveLog (const AlarmList& alarmList) *****************************************************************************************/ void logAlarm (const Alarm& calAlarm, const string& fileName) { + struct flock fl; + int fd; + if (ALARM_DEBUG) { LoggingID lid(11); MessageLog ml(lid); @@ -154,24 +182,48 @@ void logAlarm (const Alarm& calAlarm, const string& fileName) ml.logDebugMessage(msg); } - int fd = open(fileName.c_str(), O_RDWR|O_CREAT, 0644); - ofstream AlarmFile (fileName.c_str(), ios::app); + memset(&fl, 0, sizeof(fl)); + fl.l_type = F_RDLCK; // read lock + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; //lock whole file - // Aquire an exclusive lock - if (flock(fd,LOCK_EX) == -1) { - throw runtime_error ("Lock file error: " + fileName); - } + // create new file + if ((fd = open(fileName.c_str(), O_RDWR|O_CREAT, 0664)) >= 0) + { // lock file + if (fcntl(fd, F_SETLKW, &fl) != 0) + { + ostringstream oss; + oss << "logAlarm: error locking file " << + fileName << + ": " << + strerror(errno) << + ", proceding anyway."; + cerr << oss.str() << endl; + } + + ofstream AlarmFile (fileName.c_str(), ios::app); - AlarmFile << calAlarm; - AlarmFile.close(); - - // Release lock - if (flock(fd,LOCK_UN)==-1) - { - throw runtime_error ("Release lock file error: " + fileName); + AlarmFile << calAlarm; + AlarmFile.close(); + + fl.l_type = F_UNLCK; //unlock + fcntl(fd, F_SETLK, &fl); + + close(fd); } + else + { + ostringstream oss; + oss << "logAlarm: error opening file " << + fileName << + ": " << + strerror(errno); + throw runtime_error(oss.str()); + } + + return; - close(fd); } /***************************************************************************************** @@ -323,7 +375,6 @@ void configAlarm (Alarm& calAlarm) oam.setAlarmConfig (alarmID, "LastIssueTime", now); oam.setAlarmConfig (alarmID, "Occurrences", 1); } - else { // increment counter and check the ctnThreshold @@ -377,7 +428,7 @@ void configAlarm (Alarm& calAlarm) * *****************************************************************************************/ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int state, - std::string repModuleName, std::string repProcessName) + std::string repModuleName, std::string repProcessName) { #ifdef SKIP_ALARM @@ -475,58 +526,62 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st *****************************************************************************************/ void ALARMManager::getActiveAlarm(AlarmList& alarmList) const { + struct flock fl; + int fd; + //add-on to fileName with mount name if on non Parent Module Oam oam; string fileName = ACTIVE_ALARM_FILE; - int fd = open(fileName.c_str(),O_RDONLY); + memset(&fl, 0, sizeof(fl)); + fl.l_type = F_RDLCK; // read lock + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; //lock whole file - if (fd == -1) { - // file may being deleted temporarily by trapHandler - sleep (1); - fd = open(fileName.c_str(),O_RDONLY); - if (fd == -1) { - // no active alarms, return - return; - } - } - - ifstream activeAlarm (fileName.c_str(), ios::in); - - // acquire read lock - if (flock(fd,LOCK_SH) == -1) - { - throw runtime_error ("Lock active alarm log file error"); - } - - Alarm alarm; - - while (!activeAlarm.eof()) - { - activeAlarm >> alarm; - if (alarm.getAlarmID() != INVALID_ALARM_ID) - //don't sort -// alarmList.insert (AlarmList::value_type(alarm.getAlarmID(), alarm)); - alarmList.insert (AlarmList::value_type(INVALID_ALARM_ID, alarm)); - } - activeAlarm.close(); - - // release lock - if (flock(fd,LOCK_UN) == -1) - { - throw runtime_error ("Release lock active alarm log file error"); - } - - close(fd); - - if (ALARM_DEBUG) - { - AlarmList :: iterator i; - for (i = alarmList.begin(); i != alarmList.end(); ++i) + // create new file + if (fd = open(fileName.c_str(),O_RDONLY) >= 0) + { // lock file + if (fcntl(fd, F_SETLKW, &fl) != 0) { - cout << i->second << endl; + ostringstream oss; + oss << "getActiveAlarm: error locking file " << + fileName << + ": " << + strerror(errno) << + ", proceding anyway."; + cerr << oss.str() << endl; + } + + ifstream activeAlarm (fileName.c_str(), ios::in); + + Alarm alarm; + + while (!activeAlarm.eof()) + { + activeAlarm >> alarm; + if (alarm.getAlarmID() != INVALID_ALARM_ID) + //don't sort + // alarmList.insert (AlarmList::value_type(alarm.getAlarmID(), alarm)); + alarmList.insert (AlarmList::value_type(INVALID_ALARM_ID, alarm)); + } + activeAlarm.close(); + + fl.l_type = F_UNLCK; //unlock + fcntl(fd, F_SETLK, &fl); + + close(fd); + + if (ALARM_DEBUG) + { + AlarmList :: iterator i; + for (i = alarmList.begin(); i != alarmList.end(); ++i) + { + cout << i->second << endl; + } } } + return; } @@ -540,6 +595,9 @@ void ALARMManager::getActiveAlarm(AlarmList& alarmList) const *****************************************************************************************/ void ALARMManager::getAlarm(std::string date, AlarmList& alarmList) const { + struct flock fl; + int fd; + string alarmFile = "/tmp/alarms"; //make 1 alarm log file made up of archive and current alarm.log @@ -568,68 +626,79 @@ void ALARMManager::getAlarm(std::string date, AlarmList& alarmList) const cmd = "cat " + ALARM_FILE + " >> /tmp/alarms"; (void)system(cmd.c_str()); - int fd = open(alarmFile.c_str(),O_RDONLY); - - if (fd == -1) - // doesn't exist yet, return - return; + memset(&fl, 0, sizeof(fl)); + fl.l_type = F_RDLCK; // read lock + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; //lock whole file - ifstream hisAlarm (alarmFile.c_str(), ios::in); - - // acquire read lock - if (flock(fd,LOCK_SH) == -1) - { - throw runtime_error ("Lock alarm log file error"); - } - - //get mm / dd / yy from incoming date - string mm = date.substr(0,2); - string dd = date.substr(3,2); - string yy = date.substr(6,2); - - Alarm alarm; - - while (!hisAlarm.eof()) - { - hisAlarm >> alarm; - if (alarm.getAlarmID() != INVALID_ALARM_ID) { - time_t cal = alarm.getTimestampSeconds(); - struct tm tm; - localtime_r(&cal, &tm); - char tmp[3]; - strftime (tmp, 3, "%m", &tm); - string alarm_mm = tmp; - alarm_mm = alarm_mm.substr(0,2); - strftime (tmp, 3, "%d", &tm); - string alarm_dd = tmp; - alarm_dd = alarm_dd.substr(0,2); - strftime (tmp, 3, "%y", &tm); - string alarm_yy = tmp; - alarm_yy = alarm_yy.substr(0,2); - - if ( mm == alarm_mm && dd == alarm_dd && yy == alarm_yy ) - //don't sort - // alarmList.insert (AlarmList::value_type(alarm.getAlarmID(), alarm)); - alarmList.insert (AlarmList::value_type(INVALID_ALARM_ID, alarm)); - } - } - hisAlarm.close(); - unlink (alarmFile.c_str()); - - // release lock - if (flock(fd,LOCK_UN) == -1) - { - throw runtime_error ("Release lock active alarm log file error"); - } - - if (ALARM_DEBUG) - { - AlarmList :: iterator i; - for (i = alarmList.begin(); i != alarmList.end(); ++i) + // create new file + if (fd = open(alarmFile.c_str(),O_RDONLY) >= 0) + { // lock file + if (fcntl(fd, F_SETLKW, &fl) != 0) { - cout << i->second << endl; + ostringstream oss; + oss << "getAlarm: error locking file " << + alarmFile << + ": " << + strerror(errno) << + ", proceding anyway."; + cerr << oss.str() << endl; } + + ifstream hisAlarm (alarmFile.c_str(), ios::in); + + //get mm / dd / yy from incoming date + string mm = date.substr(0,2); + string dd = date.substr(3,2); + string yy = date.substr(6,2); + + Alarm alarm; + + while (!hisAlarm.eof()) + { + hisAlarm >> alarm; + if (alarm.getAlarmID() != INVALID_ALARM_ID) { + time_t cal = alarm.getTimestampSeconds(); + struct tm tm; + localtime_r(&cal, &tm); + char tmp[3]; + strftime (tmp, 3, "%m", &tm); + string alarm_mm = tmp; + alarm_mm = alarm_mm.substr(0,2); + strftime (tmp, 3, "%d", &tm); + string alarm_dd = tmp; + alarm_dd = alarm_dd.substr(0,2); + strftime (tmp, 3, "%y", &tm); + string alarm_yy = tmp; + alarm_yy = alarm_yy.substr(0,2); + + if ( mm == alarm_mm && dd == alarm_dd && yy == alarm_yy ) + //don't sort + // alarmList.insert (AlarmList::value_type(alarm.getAlarmID(), alarm)); + alarmList.insert (AlarmList::value_type(INVALID_ALARM_ID, alarm)); + } + } + hisAlarm.close(); + unlink (alarmFile.c_str()); + + fl.l_type = F_UNLCK; //unlock + fcntl(fd, F_SETLK, &fl); + + + if (ALARM_DEBUG) + { + AlarmList :: iterator i; + for (i = alarmList.begin(); i != alarmList.end(); ++i) + { + cout << i->second << endl; + } + } + + close(fd); } + + return; } } //namespace alarmmanager diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 724650f08..200249675 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -2840,6 +2840,8 @@ int ProcessMonitor::updateLog(std::string action, std::string level) { MonitorLog log; Oam oam; + struct flock fl; + int fd; string fileName; oam.getSystemConfig("SystemLogConfigFile", fileName); @@ -3080,29 +3082,46 @@ int ProcessMonitor::updateLog(std::string action, std::string level) unlink (fileName.c_str()); ofstream newFile (fileName.c_str()); - // create new file - int fd = open(fileName.c_str(),O_RDWR|O_CREAT, 0644); - - // Aquire an exclusive lock - if (flock(fd,LOCK_EX) == -1) { - log.writeLog(__LINE__, "ERROR: file lock failure on " + fileName, LOG_TYPE_ERROR ); - close(fd); - return -1; - } - - copy(lines.begin(), lines.end(), ostream_iterator(newFile, "\n")); - newFile.close(); - - // Release lock - if (flock(fd,LOCK_UN) == -1) - { - log.writeLog(__LINE__, "ERROR: file unlock failure on " + fileName, LOG_TYPE_ERROR ); - close(fd); - return -1; - } - close(fd); + memset(&fl, 0, sizeof(fl)); + fl.l_type = F_RDLCK; // read lock + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; //lock whole file - oam.syslogAction("restart"); + // create new file + if ((fd = open(fileName.c_str(), O_RDWR|O_CREAT, 0644)) >= 0) + { // lock file + + if (fcntl(fd, F_SETLKW, &fl) != 0) + { + ostringstream oss; + oss << "ProcessMonitor::updateLog: error locking file " << + fileName << + ": " << + strerror(errno) << + ", proceding anyway."; + cerr << oss.str() << endl; + } + + copy(lines.begin(), lines.end(), ostream_iterator(newFile, "\n")); + newFile.close(); + + fl.l_type = F_UNLCK; //unlock + fcntl(fd, F_SETLK, &fl); + + close(fd); + + oam.syslogAction("restart"); + } + else + { + ostringstream oss; + oss << "ProcessMonitor::updateLog: error opening file " << + fileName << + ": " << + strerror(errno); + throw runtime_error(oss.str()); + } } //update file priviledges From d30bf8dba1965c4f682bfe007204a4142fb51d21 Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 25 Aug 2017 10:54:50 -0500 Subject: [PATCH 06/36] MCOL-890 add logic to perform dictionary lookup for GROUP_CONCAT in the presence of COUNT() --- dbcon/joblist/joblistfactory.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/dbcon/joblist/joblistfactory.cpp b/dbcon/joblist/joblistfactory.cpp index 99079710a..b6b7a1ed6 100644 --- a/dbcon/joblist/joblistfactory.cpp +++ b/dbcon/joblist/joblistfactory.cpp @@ -871,6 +871,25 @@ const JobStepVector doAggProject(const CalpontSelectExecutionPlan* csep, JobInfo TupleInfo ti(setExpTupleInfo(ct, eid, gcc->alias(), jobInfo)); tupleKey = ti.key; jobInfo.returnedColVec.push_back(make_pair(tupleKey, gcc->aggOp())); + // not a tokenOnly column. Mark all the columns involved + srcp = gcc->functionParms(); + const RowColumn* rowCol = dynamic_cast(srcp.get()); + if (rowCol) + { + const std::vector& cols = rowCol->columnVec(); + for (vector::const_iterator j = cols.begin(); j != cols.end(); j++) + { + sc = dynamic_cast(j->get()); + if (sc) + { + CalpontSystemCatalog::OID tblOid = tableOid(sc, jobInfo.csc); + alias = extractTableAlias(sc); + ct = sc->colType(); + TupleInfo ti(setTupleInfo(ct, sc->oid(), jobInfo, tblOid, sc, alias)); + jobInfo.tokenOnly[ti.key] = false; + } + } + } continue; } From 9827bd804035431cbcef089ce36ea326d2ba6ea4 Mon Sep 17 00:00:00 2001 From: David Hill Date: Tue, 29 Aug 2017 16:12:45 -0500 Subject: [PATCH 07/36] MCOL-892: backed out changes for this fix, leave the flock for logs --- oamapps/alarmmanager/alarmmanager.cpp | 343 ++++++++++---------------- 1 file changed, 137 insertions(+), 206 deletions(-) diff --git a/oamapps/alarmmanager/alarmmanager.cpp b/oamapps/alarmmanager/alarmmanager.cpp index 49f6c8638..5d3990b2d 100644 --- a/oamapps/alarmmanager/alarmmanager.cpp +++ b/oamapps/alarmmanager/alarmmanager.cpp @@ -97,9 +97,6 @@ ALARMManager::~ALARMManager() *****************************************************************************************/ void rewriteActiveLog (const AlarmList& alarmList) { - struct flock fl; - int fd; - if (ALARM_DEBUG) { LoggingID lid(11); MessageLog ml(lid); @@ -113,52 +110,30 @@ void rewriteActiveLog (const AlarmList& alarmList) // delete the old file unlink (ACTIVE_ALARM_FILE.c_str()); - memset(&fl, 0, sizeof(fl)); - fl.l_type = F_RDLCK; // read lock - fl.l_whence = SEEK_SET; - fl.l_start = 0; - fl.l_len = 0; //lock whole file - // create new file - if ((fd = open(ACTIVE_ALARM_FILE.c_str(), O_RDWR|O_CREAT, 0664)) >= 0) - { // lock file - if (fcntl(fd, F_SETLKW, &fl) != 0) - { - ostringstream oss; - oss << "rewriteActiveLog: error locking file " << - ACTIVE_ALARM_FILE << - ": " << - strerror(errno) << - ", proceding anyway."; - cerr << oss.str() << endl; - } - - ofstream activeAlarmFile (ACTIVE_ALARM_FILE.c_str()); - - AlarmList::const_iterator i; - for (i = alarmList.begin(); i != alarmList.end(); ++i) - { - activeAlarmFile << i->second; - } - - activeAlarmFile.close(); - - fl.l_type = F_UNLCK; //unlock - fcntl(fd, F_SETLK, &fl); - - close(fd); - } - else - { - ostringstream oss; - oss << "rewriteActiveLog: error opening file " << - ACTIVE_ALARM_FILE << - ": " << - strerror(errno); - throw runtime_error(oss.str()); - } + int fd = open(ACTIVE_ALARM_FILE.c_str(), O_RDWR|O_CREAT, 0644); - return; + // Aquire an exclusive lock + if (flock(fd,LOCK_EX) == -1) { + throw runtime_error ("Lock active alarm log file error"); + } + + ofstream activeAlarmFile (ACTIVE_ALARM_FILE.c_str()); + + AlarmList::const_iterator i; + for (i = alarmList.begin(); i != alarmList.end(); ++i) + { + activeAlarmFile << i->second; + } + + activeAlarmFile.close(); + + // Release lock + if (flock(fd,LOCK_UN)==-1) + { + throw runtime_error ("Release lock active alarm log file error"); + } + close(fd); } /***************************************************************************************** @@ -169,9 +144,6 @@ void rewriteActiveLog (const AlarmList& alarmList) *****************************************************************************************/ void logAlarm (const Alarm& calAlarm, const string& fileName) { - struct flock fl; - int fd; - if (ALARM_DEBUG) { LoggingID lid(11); MessageLog ml(lid); @@ -182,48 +154,24 @@ void logAlarm (const Alarm& calAlarm, const string& fileName) ml.logDebugMessage(msg); } - memset(&fl, 0, sizeof(fl)); - fl.l_type = F_RDLCK; // read lock - fl.l_whence = SEEK_SET; - fl.l_start = 0; - fl.l_len = 0; //lock whole file + int fd = open(fileName.c_str(), O_RDWR|O_CREAT, 0644); + ofstream AlarmFile (fileName.c_str(), ios::app); - // create new file - if ((fd = open(fileName.c_str(), O_RDWR|O_CREAT, 0664)) >= 0) - { // lock file - if (fcntl(fd, F_SETLKW, &fl) != 0) - { - ostringstream oss; - oss << "logAlarm: error locking file " << - fileName << - ": " << - strerror(errno) << - ", proceding anyway."; - cerr << oss.str() << endl; - } - - ofstream AlarmFile (fileName.c_str(), ios::app); + // Aquire an exclusive lock + if (flock(fd,LOCK_EX) == -1) { + throw runtime_error ("Lock file error: " + fileName); + } - AlarmFile << calAlarm; - AlarmFile.close(); - - fl.l_type = F_UNLCK; //unlock - fcntl(fd, F_SETLK, &fl); - - close(fd); - } - else - { - ostringstream oss; - oss << "logAlarm: error opening file " << - fileName << - ": " << - strerror(errno); - throw runtime_error(oss.str()); - } + AlarmFile << calAlarm; + AlarmFile.close(); - return; + // Release lock + if (flock(fd,LOCK_UN)==-1) + { + throw runtime_error ("Release lock file error: " + fileName); + } + close(fd); } /***************************************************************************************** @@ -375,6 +323,7 @@ void configAlarm (Alarm& calAlarm) oam.setAlarmConfig (alarmID, "LastIssueTime", now); oam.setAlarmConfig (alarmID, "Occurrences", 1); } + else { // increment counter and check the ctnThreshold @@ -428,7 +377,7 @@ void configAlarm (Alarm& calAlarm) * *****************************************************************************************/ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int state, - std::string repModuleName, std::string repProcessName) + std::string repModuleName, std::string repProcessName) { #ifdef SKIP_ALARM @@ -526,62 +475,58 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st *****************************************************************************************/ void ALARMManager::getActiveAlarm(AlarmList& alarmList) const { - struct flock fl; - int fd; - //add-on to fileName with mount name if on non Parent Module Oam oam; string fileName = ACTIVE_ALARM_FILE; - memset(&fl, 0, sizeof(fl)); - fl.l_type = F_RDLCK; // read lock - fl.l_whence = SEEK_SET; - fl.l_start = 0; - fl.l_len = 0; //lock whole file + int fd = open(fileName.c_str(),O_RDONLY); - // create new file - if (fd = open(fileName.c_str(),O_RDONLY) >= 0) - { // lock file - if (fcntl(fd, F_SETLKW, &fl) != 0) - { - ostringstream oss; - oss << "getActiveAlarm: error locking file " << - fileName << - ": " << - strerror(errno) << - ", proceding anyway."; - cerr << oss.str() << endl; - } - - ifstream activeAlarm (fileName.c_str(), ios::in); - - Alarm alarm; - - while (!activeAlarm.eof()) - { - activeAlarm >> alarm; - if (alarm.getAlarmID() != INVALID_ALARM_ID) - //don't sort - // alarmList.insert (AlarmList::value_type(alarm.getAlarmID(), alarm)); - alarmList.insert (AlarmList::value_type(INVALID_ALARM_ID, alarm)); - } - activeAlarm.close(); - - fl.l_type = F_UNLCK; //unlock - fcntl(fd, F_SETLK, &fl); + if (fd == -1) { + // file may being deleted temporarily by trapHandler + sleep (1); + fd = open(fileName.c_str(),O_RDONLY); + if (fd == -1) { + // no active alarms, return + return; + } + } - close(fd); + ifstream activeAlarm (fileName.c_str(), ios::in); - if (ALARM_DEBUG) - { - AlarmList :: iterator i; - for (i = alarmList.begin(); i != alarmList.end(); ++i) - { - cout << i->second << endl; - } - } + // acquire read lock + if (flock(fd,LOCK_SH) == -1) + { + throw runtime_error ("Lock active alarm log file error"); } + Alarm alarm; + + while (!activeAlarm.eof()) + { + activeAlarm >> alarm; + if (alarm.getAlarmID() != INVALID_ALARM_ID) + //don't sort +// alarmList.insert (AlarmList::value_type(alarm.getAlarmID(), alarm)); + alarmList.insert (AlarmList::value_type(INVALID_ALARM_ID, alarm)); + } + activeAlarm.close(); + + // release lock + if (flock(fd,LOCK_UN) == -1) + { + throw runtime_error ("Release lock active alarm log file error"); + } + + close(fd); + + if (ALARM_DEBUG) + { + AlarmList :: iterator i; + for (i = alarmList.begin(); i != alarmList.end(); ++i) + { + cout << i->second << endl; + } + } return; } @@ -595,9 +540,6 @@ void ALARMManager::getActiveAlarm(AlarmList& alarmList) const *****************************************************************************************/ void ALARMManager::getAlarm(std::string date, AlarmList& alarmList) const { - struct flock fl; - int fd; - string alarmFile = "/tmp/alarms"; //make 1 alarm log file made up of archive and current alarm.log @@ -626,79 +568,68 @@ void ALARMManager::getAlarm(std::string date, AlarmList& alarmList) const cmd = "cat " + ALARM_FILE + " >> /tmp/alarms"; (void)system(cmd.c_str()); - memset(&fl, 0, sizeof(fl)); - fl.l_type = F_RDLCK; // read lock - fl.l_whence = SEEK_SET; - fl.l_start = 0; - fl.l_len = 0; //lock whole file - - // create new file - if (fd = open(alarmFile.c_str(),O_RDONLY) >= 0) - { // lock file - if (fcntl(fd, F_SETLKW, &fl) != 0) - { - ostringstream oss; - oss << "getAlarm: error locking file " << - alarmFile << - ": " << - strerror(errno) << - ", proceding anyway."; - cerr << oss.str() << endl; - } - - ifstream hisAlarm (alarmFile.c_str(), ios::in); - - //get mm / dd / yy from incoming date - string mm = date.substr(0,2); - string dd = date.substr(3,2); - string yy = date.substr(6,2); - - Alarm alarm; - - while (!hisAlarm.eof()) - { - hisAlarm >> alarm; - if (alarm.getAlarmID() != INVALID_ALARM_ID) { - time_t cal = alarm.getTimestampSeconds(); - struct tm tm; - localtime_r(&cal, &tm); - char tmp[3]; - strftime (tmp, 3, "%m", &tm); - string alarm_mm = tmp; - alarm_mm = alarm_mm.substr(0,2); - strftime (tmp, 3, "%d", &tm); - string alarm_dd = tmp; - alarm_dd = alarm_dd.substr(0,2); - strftime (tmp, 3, "%y", &tm); - string alarm_yy = tmp; - alarm_yy = alarm_yy.substr(0,2); - - if ( mm == alarm_mm && dd == alarm_dd && yy == alarm_yy ) - //don't sort - // alarmList.insert (AlarmList::value_type(alarm.getAlarmID(), alarm)); - alarmList.insert (AlarmList::value_type(INVALID_ALARM_ID, alarm)); - } - } - hisAlarm.close(); - unlink (alarmFile.c_str()); + int fd = open(alarmFile.c_str(),O_RDONLY); - fl.l_type = F_UNLCK; //unlock - fcntl(fd, F_SETLK, &fl); + if (fd == -1) + // doesn't exist yet, return + return; + ifstream hisAlarm (alarmFile.c_str(), ios::in); + + // acquire read lock + if (flock(fd,LOCK_SH) == -1) + { + throw runtime_error ("Lock alarm log file error"); + } + + //get mm / dd / yy from incoming date + string mm = date.substr(0,2); + string dd = date.substr(3,2); + string yy = date.substr(6,2); + + Alarm alarm; - if (ALARM_DEBUG) - { - AlarmList :: iterator i; - for (i = alarmList.begin(); i != alarmList.end(); ++i) - { - cout << i->second << endl; - } + while (!hisAlarm.eof()) + { + hisAlarm >> alarm; + if (alarm.getAlarmID() != INVALID_ALARM_ID) { + time_t cal = alarm.getTimestampSeconds(); + struct tm tm; + localtime_r(&cal, &tm); + char tmp[3]; + strftime (tmp, 3, "%m", &tm); + string alarm_mm = tmp; + alarm_mm = alarm_mm.substr(0,2); + strftime (tmp, 3, "%d", &tm); + string alarm_dd = tmp; + alarm_dd = alarm_dd.substr(0,2); + strftime (tmp, 3, "%y", &tm); + string alarm_yy = tmp; + alarm_yy = alarm_yy.substr(0,2); + + if ( mm == alarm_mm && dd == alarm_dd && yy == alarm_yy ) + //don't sort + // alarmList.insert (AlarmList::value_type(alarm.getAlarmID(), alarm)); + alarmList.insert (AlarmList::value_type(INVALID_ALARM_ID, alarm)); } - - close(fd); + } + hisAlarm.close(); + unlink (alarmFile.c_str()); + + // release lock + if (flock(fd,LOCK_UN) == -1) + { + throw runtime_error ("Release lock active alarm log file error"); } - return; + if (ALARM_DEBUG) + { + AlarmList :: iterator i; + for (i = alarmList.begin(); i != alarmList.end(); ++i) + { + cout << i->second << endl; + } + } } } //namespace alarmmanager From 2ce8c9262b8e80399a40864597cc7a091fde1b49 Mon Sep 17 00:00:00 2001 From: david hill Date: Tue, 29 Aug 2017 17:01:02 -0500 Subject: [PATCH 08/36] MCOL-892 - moved the setting to mysql rep after rollover is done --- procmgr/processmanager.cpp | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index 1079699fb..06ad5ff42 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -6804,13 +6804,6 @@ void startSystemThread(oam::DeviceNetworkList Devicenetworklist) status = oam::API_FAILURE; } - log.writeLog(__LINE__, "Setup MySQL Replication for startsystemthread", LOG_TYPE_DEBUG); - oam::DeviceNetworkList devicenetworklistRep; - processManager.setMySQLReplication(devicenetworklistRep); - - //set query system state not ready - processManager.setQuerySystemState(true); - // Bug 4554: Wait until DMLProc is finished with rollback if (status == oam::API_SUCCESS) { @@ -6875,19 +6868,25 @@ void startSystemThread(oam::DeviceNetworkList Devicenetworklist) // wait some more sleep(2); } - processManager.setSystemState(rtn); + + log.writeLog(__LINE__, "Setup MySQL Replication for startsystemthread", LOG_TYPE_DEBUG); + oam::DeviceNetworkList devicenetworklistRep; + processManager.setMySQLReplication(devicenetworklistRep); + + //set query system state ready + processManager.setQuerySystemState(true); + + //run command to build system table if they don't already exist + sleep(5); + int ret = processManager.buildSystemTables("pm1"); + if (ret == oam::API_SUCCESS ) + log.writeLog(__LINE__, "System Catalog Successfully Built by ProcMgr", LOG_TYPE_DEBUG); + else + log.writeLog(__LINE__, "System Catalog Successfully not built by ProcMgr, ret code = " + oam.itoa(ret), LOG_TYPE_DEBUG); + + processManager.setSystemState(rtn); } - //run command to build system table if they don't already exist - sleep(5); - int ret = processManager.buildSystemTables("pm1"); - if (ret == oam::API_SUCCESS ) - log.writeLog(__LINE__, "System Catalog Successfully Built by ProcMgr", LOG_TYPE_DEBUG); - else - log.writeLog(__LINE__, "System Catalog Successfully not built by ProcMgr, ret code = " + oam.itoa(ret), LOG_TYPE_DEBUG); - - log.writeLog(__LINE__, "startSystemThread Exit", LOG_TYPE_DEBUG); - // exit thread log.writeLog(__LINE__, "startSystemThread Exit", LOG_TYPE_DEBUG); startsystemthreadStatus = status; From f6975c8988ca3032256278373d5bf8f616635370 Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 31 Aug 2017 11:34:52 -0500 Subject: [PATCH 09/36] MCOL-875 - add code to cleanup lib directory --- oam/install_scripts/pre-uninstall | 3 +++ 1 file changed, 3 insertions(+) diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall index 8709d369c..bbcc95109 100755 --- a/oam/install_scripts/pre-uninstall +++ b/oam/install_scripts/pre-uninstall @@ -181,6 +181,9 @@ else fi fi +#remove file in library directory # mcol-875 +rm -f $installdir/lib/* + #tell user to run post configure script echo " " echo "Mariab Columnstore uninstall completed" From a6ae09dedb24872e3a9a19f76b87ebfef7c856cc Mon Sep 17 00:00:00 2001 From: david hill Date: Fri, 1 Sep 2017 14:45:26 -0500 Subject: [PATCH 10/36] removed new code that delete the ..lib files --- oam/install_scripts/pre-uninstall | 3 --- 1 file changed, 3 deletions(-) diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall index bbcc95109..8709d369c 100755 --- a/oam/install_scripts/pre-uninstall +++ b/oam/install_scripts/pre-uninstall @@ -181,9 +181,6 @@ else fi fi -#remove file in library directory # mcol-875 -rm -f $installdir/lib/* - #tell user to run post configure script echo " " echo "Mariab Columnstore uninstall completed" From d5a3cf1cf621fdcfe4e3d55df5511ad18e82b319 Mon Sep 17 00:00:00 2001 From: david hill Date: Wed, 13 Sep 2017 10:50:33 -0500 Subject: [PATCH 11/36] MCOL-837 - commented out check for FilesPerColumnPartition --- oamapps/postConfigure/postConfigure.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index 35196edf8..b4fc8b3ba 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -2592,7 +2592,7 @@ int main(int argc, char *argv[]) //check 'files per parition' with number of dbroots - checkFilesPerPartion(DBRootCount, sysConfig); + //checkFilesPerPartion(DBRootCount, sysConfig); //setup DBRM Controller sysConfig->setConfig("DBRM_Controller", "NumWorkers", oam.itoa(DBRMworkernodeID)); From efac1730495cfc30c8a7d1e9f0fb6c0df62661eb Mon Sep 17 00:00:00 2001 From: David Hall Date: Thu, 14 Sep 2017 23:29:54 -0500 Subject: [PATCH 12/36] MCOL-911 getAggCols needs to look into the objects --- dbcon/execplan/aggregatecolumn.cpp | 14 +++++++++ dbcon/execplan/constantfilter.cpp | 16 +++++++--- dbcon/execplan/simplefilter.cpp | 50 ++++++++++++++++++------------ 3 files changed, 57 insertions(+), 23 deletions(-) diff --git a/dbcon/execplan/aggregatecolumn.cpp b/dbcon/execplan/aggregatecolumn.cpp index 1cb30c016..924c64bc9 100644 --- a/dbcon/execplan/aggregatecolumn.cpp +++ b/dbcon/execplan/aggregatecolumn.cpp @@ -55,15 +55,29 @@ void getAggCols(execplan::ParseTree* n, void* obj) SimpleFilter *sf = dynamic_cast(tn); ConstantFilter *cf = dynamic_cast(tn); if (sc) + { list->push_back(sc); + } else if (fc) + { + fc->hasAggregate(); list->insert(list->end(), fc->aggColumnList().begin(), fc->aggColumnList().end()); + } else if (ac) + { + ac->hasAggregate(); list->insert(list->end(), ac->aggColumnList().begin(), ac->aggColumnList().end()); + } else if (sf) + { + sf->hasAggregate(); list->insert(list->end(), sf->aggColumnList().begin(), sf->aggColumnList().end()); + } else if (cf) + { + cf->hasAggregate(); list->insert(list->end(), cf->aggColumnList().begin(), cf->aggColumnList().end()); + } } /** diff --git a/dbcon/execplan/constantfilter.cpp b/dbcon/execplan/constantfilter.cpp index 79e79773a..044856ad3 100644 --- a/dbcon/execplan/constantfilter.cpp +++ b/dbcon/execplan/constantfilter.cpp @@ -121,14 +121,22 @@ ostream& operator<<(ostream& output, const ConstantFilter& rhs) bool ConstantFilter::hasAggregate() { - fAggColumnList.clear(); - for (uint32_t i = 0; i < fFilterList.size(); i++) + if (fAggColumnList.empty()) { - if (fFilterList[i]->hasAggregate()) + for (uint32_t i = 0; i < fFilterList.size(); i++) { - return true; + if (fFilterList[i]->hasAggregate()) + { + fAggColumnList.insert(fAggColumnList.end(), + fFilterList[i]->aggColumnList().begin(), + fFilterList[i]->aggColumnList().end()); + } } } + if (!fAggColumnList.empty()) + { + return true; + } return false; } diff --git a/dbcon/execplan/simplefilter.cpp b/dbcon/execplan/simplefilter.cpp index bead8ece5..c7bdc74f0 100644 --- a/dbcon/execplan/simplefilter.cpp +++ b/dbcon/execplan/simplefilter.cpp @@ -644,29 +644,41 @@ void SimpleFilter::setSimpleColumnList() bool SimpleFilter::hasAggregate() { - AggregateColumn *lac = dynamic_cast(fLhs); - AggregateColumn *rac = dynamic_cast(fRhs); - fAggColumnList.clear(); + if (fAggColumnList.empty()) + { + AggregateColumn *lac = dynamic_cast(fLhs); + AggregateColumn *rac = dynamic_cast(fRhs); + fAggColumnList.clear(); - if (lac) + if (lac) + { + fAggColumnList.insert(fAggColumnList.end(), lac); + } + if (rac) + { + fAggColumnList.insert(fAggColumnList.end(), rac); + } + + if (fLhs) + { + if (fLhs->hasAggregate()) + fAggColumnList.insert(fAggColumnList.end(), + fLhs->aggColumnList().begin(), + fLhs->aggColumnList().end()); + } + + if (fRhs) + { + if (fRhs->hasAggregate()) + fAggColumnList.insert(fAggColumnList.end(), + fRhs->aggColumnList().begin(), + fRhs->aggColumnList().end()); + } + } + if (!fAggColumnList.empty()) { return true; } - else if (fLhs) - { - if (fLhs->hasAggregate()) - return true; - } - - if (rac) - { - return true; - } - else if (fRhs) - { - if (fRhs->hasAggregate()) - return true; - } return false; } From f15b2390a320cc31e7cada07104984a8bbd1fe2a Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Fri, 15 Sep 2017 10:05:19 +0100 Subject: [PATCH 13/36] MCOL-859 DDL System catalog mutex deadlock A race between a DDL change and a system catalog version cache update can cause a deadlock in DDLProc. This makes the mutex recursive. --- dbcon/execplan/calpontsystemcatalog.cpp | 4 ++-- dbcon/execplan/calpontsystemcatalog.h | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index 635927c7b..acc49ef7f 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -1187,7 +1187,7 @@ const CalpontSystemCatalog::ColType CalpontSystemCatalog::colTypeDct(const OID& } // check map first cached column type - boost::mutex::scoped_lock lk3(fDctTokenMapLock); + boost::recursive_mutex::scoped_lock lk3(fDctTokenMapLock); DctTokenMap::const_iterator iter = fDctTokenMap.find(dictOid); if (iter != fDctTokenMap.end()) return colType(iter->second); @@ -5317,7 +5317,7 @@ void CalpontSystemCatalog::flushCache() buildSysTablemap(); lk3.unlock(); - boost::mutex::scoped_lock lk4(fDctTokenMapLock); + boost::recursive_mutex::scoped_lock lk4(fDctTokenMapLock); fDctTokenMap.clear(); buildSysDctmap(); lk4.unlock(); diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index 2936b2daa..641a40b49 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -869,7 +869,8 @@ private: typedef std::map DctTokenMap; DctTokenMap fDctTokenMap; - boost::mutex fDctTokenMapLock; + // MCOL-859: this can lock when already locked in the same thread + boost::recursive_mutex fDctTokenMapLock; typedef std::map TableNameMap; TableNameMap fTableNameMap; From fa1b487bcd891531d9b70e92d0ca4a52ff96047f Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Fri, 15 Sep 2017 11:17:58 +0100 Subject: [PATCH 14/36] MCOL-877 Fix I_S / LDI cpimport escaping Backslashes need escaping otherwise they are not sent to cpimport correctly. --- dbcon/mysql/ha_calpont_dml.cpp | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/dbcon/mysql/ha_calpont_dml.cpp b/dbcon/mysql/ha_calpont_dml.cpp index 6f115689e..ef953ce2b 100755 --- a/dbcon/mysql/ha_calpont_dml.cpp +++ b/dbcon/mysql/ha_calpont_dml.cpp @@ -711,6 +711,7 @@ int ha_calpont_impl_write_batch_row_(uchar *buf, TABLE* table, cal_impl_if::cal_ uint16_t colpos = 0; buf = buf + ci.headerLength; // Number of bytes used for null bits. //@Bug 6122 if all columns have not null constraint, there is no information in the header + std::string escape; char nullBits = *bufHdr++; if (!ci.useXbit) { @@ -831,8 +832,11 @@ int ha_calpont_impl_write_batch_row_(uchar *buf, TABLE* table, cal_impl_if::cal_ if (current_thd->variables.sql_mode & MODE_PAD_CHAR_TO_FULL_LENGTH) { // Pad to the full length of the field - fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, ci.columnTypes[colpos].colWidth, - buf, ci.enclosed_by, ci.delimiter); + escape.assign((char*)buf, ci.columnTypes[colpos].colWidth); + boost::replace_all(escape, "\\", "\\\\"); + + fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, (int)escape.length(), + escape.c_str(), ci.enclosed_by, ci.delimiter); } else { @@ -841,8 +845,10 @@ int ha_calpont_impl_write_batch_row_(uchar *buf, TABLE* table, cal_impl_if::cal_ bitmap_set_bit(table->read_set, field->field_index); String attribute; field->val_str(&attribute); - fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, attribute.length(), - buf, ci.enclosed_by, ci.delimiter); + escape.assign((char*)buf, attribute.length()); + boost::replace_all(escape, "\\", "\\\\"); + fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, (int)escape.length(), + escape.c_str(), ci.enclosed_by, ci.delimiter); } } @@ -895,7 +901,9 @@ int ha_calpont_impl_write_batch_row_(uchar *buf, TABLE* table, cal_impl_if::cal_ dataLength = *(int16_t*) buf; buf = buf + 2 ; } - fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, dataLength, buf, ci.enclosed_by, ci.delimiter); + escape.assign((char*)buf, dataLength); + boost::replace_all(escape, "\\", "\\\\"); + fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, (int)escape.length(), escape.c_str(), ci.enclosed_by, ci.delimiter); } else //utf8 { @@ -911,8 +919,11 @@ int ha_calpont_impl_write_batch_row_(uchar *buf, TABLE* table, cal_impl_if::cal_ } if ( dataLength > ci.columnTypes[colpos].colWidth) dataLength = ci.columnTypes[colpos].colWidth; - - fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, dataLength, buf, ci.enclosed_by, ci.delimiter); + + escape.assign((char*)buf, dataLength); + boost::replace_all(escape, "\\", "\\\\"); + + fprintf(ci.filePtr, "%c%.*s%c%c", ci.enclosed_by, (int)escape.length(), escape.c_str(), ci.enclosed_by, ci.delimiter); } } //buf += ci.columnTypes[colpos].colWidth; From 4954a4282531bf786464e7e407d73c85997e7a28 Mon Sep 17 00:00:00 2001 From: david hill Date: Mon, 18 Sep 2017 09:55:13 -0500 Subject: [PATCH 15/36] fix missing@ in the remote scp scripts --- oam/install_scripts/remote_scp_get.sh | 2 +- oam/install_scripts/remote_scp_put.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/oam/install_scripts/remote_scp_get.sh b/oam/install_scripts/remote_scp_get.sh index aee668b13..703cc4b90 100755 --- a/oam/install_scripts/remote_scp_get.sh +++ b/oam/install_scripts/remote_scp_get.sh @@ -23,7 +23,7 @@ if { $PASSWORD == "ssh" } { # send command # #expect -re {[$#] } -send "scp -v $USERNAME$SERVER:$FILE .\n" +send "scp -v $USERNAME@$SERVER:$FILE .\n" expect { "Exit status 0" { exit 0 } "Exit status 1" { exit 1 } diff --git a/oam/install_scripts/remote_scp_put.sh b/oam/install_scripts/remote_scp_put.sh index 7eb6c1ecb..a97d907ad 100755 --- a/oam/install_scripts/remote_scp_put.sh +++ b/oam/install_scripts/remote_scp_put.sh @@ -22,7 +22,7 @@ if { $PASSWORD == "ssh" } { # # send command # -send "scp -v $FILE $USERNAME$SERVER:$FILE\n" +send "scp -v $FILE $USERNAME@$SERVER:$FILE\n" expect { "Exit status 0" { exit 0 } "Exit status 1" { exit 1 } From de919c7f051405676f09f0d2632afa4ea39a43a3 Mon Sep 17 00:00:00 2001 From: David Hall Date: Mon, 18 Sep 2017 12:45:41 -0500 Subject: [PATCH 16/36] MCOL-786 Redistribute: Round robin target dbroots --- .../we_redistributecontrolthread.cpp | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/writeengine/redistribute/we_redistributecontrolthread.cpp b/writeengine/redistribute/we_redistributecontrolthread.cpp index 652b48565..afefac369 100644 --- a/writeengine/redistribute/we_redistributecontrolthread.cpp +++ b/writeengine/redistribute/we_redistributecontrolthread.cpp @@ -393,6 +393,14 @@ int RedistributeControlThread::makeRedistributePlan() // It's possible that a source that is "removed" on the command line is not empty. // This can happen if a partition exists on all dbroots. + // WCOL-786: use nextDbroot to start the loop looking for a suitible target + // where we left off with the previous partition. This gives each target + // an opportunity to get some of the data. In the case of dbroot removal, + // there is often the same number of partitions on each remaining dbroot. + // This logic tends to roundrobin which dbroot gets the next batch. + set::iterator nextDbroot = targetDbroots.begin(); + set::iterator targetDbroot; + int targetCnt = (int)targetDbroots.size(); // Loop through the sources, looking for dbroots that are not targets that also still contain partitions for (set::iterator sourceDbroot = sourceDbroots.begin(); sourceDbroot != sourceDbroots.end(); @@ -416,15 +424,22 @@ int RedistributeControlThread::makeRedistributePlan() // the partition. uint64_t partCount = std::numeric_limits::max(); int tdbroot = 0; - for (set::iterator targetDbroot = targetDbroots.begin(); - targetDbroot != targetDbroots.end(); - ++targetDbroot) + targetDbroot = nextDbroot; + // MCOL-786. Start at targetDbroot and loop around back to the same spot. + for (int tbd=0; tbd < targetCnt; ++tbd) { + if (targetDbroot == targetDbroots.end()) + { + targetDbroot == targetDbroots.begin(); + } if (dbPartVec[*targetDbroot].size() < partCount) { tdbroot = *targetDbroot; partCount = dbPartVec[*targetDbroot].size(); + nextDbroot = targetDbroot; + ++nextDbroot; } + ++targetDbroot; } if (tdbroot == 0) { From 752a05cbcad5f91aac43ae187622dfcff24d6807 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 20 Sep 2017 14:00:26 +0100 Subject: [PATCH 17/36] MCOL-895 Sort system catalog ridList It is possible to have the columns for a table in a different order within the system catalog. This causes problems for things like WriteEngine so this patch sorts the columns based on position for the system catalog cache. --- dbcon/execplan/calpontsystemcatalog.cpp | 30 +++++++++++++++++++++---- dbcon/execplan/calpontsystemcatalog.h | 2 ++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp index 635927c7b..ead981791 100644 --- a/dbcon/execplan/calpontsystemcatalog.cpp +++ b/dbcon/execplan/calpontsystemcatalog.cpp @@ -3090,18 +3090,35 @@ const CalpontSystemCatalog::RIDList CalpontSystemCatalog::columnRIDs(const Table ctList[i].nextvalue = ((*it)->GetData(i)); } } - + // MCOL-895 sort ctList, we can't specify an ORDER BY to do this yet + std::sort(ctList, ctList + ti.numOfCols, ctListSort); + // populate colinfo cache lk3.lock(); for (int i = 0; i < ti.numOfCols; i++) fColinfomap[ctList[i].columnOID] = ctList[i]; lk3.unlock(); - + + // Re-sort the output based on the sorted ctList + // Don't need to do this for the cached list as this will be already sorted + RIDList rlOut; + for (int i = 0; i < ti.numOfCols; i++) + { + OID objid = ctList[i].columnOID; + for (size_t j = 0; j < rl.size(); j++) + { + if (rl[j].objnum == objid) + { + rlOut.push_back(rl[j]); + } + } + } + delete [] ctList; // delete col[9]; - if (rl.size() != 0) + if (rlOut.size() != 0) { - return rl; + return rlOut; } Message::Args args; @@ -5785,5 +5802,10 @@ vector getAllSysCatOIDs() return ret; } +bool ctListSort(const CalpontSystemCatalog::ColType& a, const CalpontSystemCatalog::ColType& b) +{ + return a.colPosition < b.colPosition; +} + } // namespace execplan // vim:sw=4 ts=4: diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h index 2936b2daa..bd5a87463 100644 --- a/dbcon/execplan/calpontsystemcatalog.h +++ b/dbcon/execplan/calpontsystemcatalog.h @@ -1215,6 +1215,8 @@ std::ostream& operator<<(std::ostream& os, const CalpontSystemCatalog::ColType& const std::string colDataTypeToString(CalpontSystemCatalog::ColDataType cdt); +bool ctListSort(const CalpontSystemCatalog::ColType& a, const CalpontSystemCatalog::ColType& b); + } //namespace execplan #endif //EXECPLAN_CALPONTSYSTEMCATALOG_H From 29edb0a1f79e8e32f77d24372f1697f5b5370377 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 2 Oct 2017 11:47:56 +0100 Subject: [PATCH 18/36] MCOL-944 Fix aggregate type switch Aggregates could switch from unsigned to signed prior to execution. This causes issues with NULL values since signed and unsigned NULLs are different. --- dbcon/joblist/tupleaggregatestep.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index 595cb174d..f577fb1da 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -3194,13 +3194,21 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( if (typeProj[colProj] != CalpontSystemCatalog::DOUBLE && typeProj[colProj] != CalpontSystemCatalog::FLOAT) { - typeAggPm.push_back(CalpontSystemCatalog::BIGINT); + if (isUnsigned(typeProj[colProj])) + { + typeAggPm.push_back(CalpontSystemCatalog::UBIGINT); + precisionAggPm.push_back(20); + } + else + { + typeAggPm.push_back(CalpontSystemCatalog::BIGINT); + precisionAggPm.push_back(19); + } uint32_t scale = scaleProj[colProj]; // for int average, FE expects a decimal if (aggOp == ROWAGG_AVG) scale = jobInfo.scaleOfAvg[aggKey]; // scale += 4; scaleAggPm.push_back(scale); - precisionAggPm.push_back(19); widthAggPm.push_back(bigIntWidth); } else From 3487789d73aa84b43842cee010e1a16f16de0cf3 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Mon, 2 Oct 2017 18:48:28 +0100 Subject: [PATCH 19/36] MCOL-944 Fix more signed/unsigned cases in agg Applies the first fix to more places --- dbcon/joblist/tupleaggregatestep.cpp | 143 +++++++++++++++++++++++---- 1 file changed, 123 insertions(+), 20 deletions(-) diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index f577fb1da..f652d0187 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -1173,7 +1173,14 @@ void TupleAggregateStep::prep1PhaseAggregate( scaleAgg.push_back(0); // work around count() in select subquery precisionAgg.push_back(9999); - typeAgg.push_back(CalpontSystemCatalog::BIGINT); + if (isUnsigned(typeProj[colProj])) + { + typeAgg.push_back(CalpontSystemCatalog::UBIGINT); + } + else + { + typeAgg.push_back(CalpontSystemCatalog::BIGINT); + } widthAgg.push_back(bigIntWidth); } break; @@ -1211,7 +1218,14 @@ void TupleAggregateStep::prep1PhaseAggregate( keysAgg.push_back(key); scaleAgg.push_back(0); precisionAgg.push_back(-16); // for connector to skip null check - typeAgg.push_back(CalpontSystemCatalog::BIGINT); + if (isUnsigned(typeProj[colProj])) + { + typeAgg.push_back(CalpontSystemCatalog::UBIGINT); + } + else + { + typeAgg.push_back(CalpontSystemCatalog::BIGINT); + } widthAgg.push_back(bigIntWidth); } break; @@ -1561,13 +1575,21 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( if (typeProj[colProj] != CalpontSystemCatalog::DOUBLE && typeProj[colProj] != CalpontSystemCatalog::FLOAT) { - typeAgg.push_back(CalpontSystemCatalog::BIGINT); - uint32_t scale = scaleProj[colProj]; + if (isUnsigned(typeProj[colProj])) + { + typeAgg.push_back(CalpontSystemCatalog::UBIGINT); + precisionAgg.push_back(20); + } + else + { + typeAgg.push_back(CalpontSystemCatalog::BIGINT); + precisionAgg.push_back(19); + } + uint32_t scale = scaleProj[colProj]; // for int average, FE expects a decimal if (aggOp == ROWAGG_AVG) scale = jobInfo.scaleOfAvg[aggKey]; // scale += 4; scaleAgg.push_back(scale); - precisionAgg.push_back(19); widthAgg.push_back(bigIntWidth); } else @@ -1594,7 +1616,14 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( scaleAgg.push_back(0); // work around count() in select subquery precisionAgg.push_back(9999); - typeAgg.push_back(CalpontSystemCatalog::BIGINT); + if (isUnsigned(typeProj[colProj])) + { + typeAgg.push_back(CalpontSystemCatalog::UBIGINT); + } + else + { + typeAgg.push_back(CalpontSystemCatalog::BIGINT); + } widthAgg.push_back(bigIntWidth); colAgg++; } @@ -1654,7 +1683,15 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( scaleAgg.push_back(0); precisionAgg.push_back(-16); // for connector to skip null check typeAgg.push_back(CalpontSystemCatalog::BIGINT); - widthAgg.push_back(bigIntWidth); + if (isUnsigned(typeProj[colProj])) + { + typeAgg.push_back(CalpontSystemCatalog::UBIGINT); + } + else + { + typeAgg.push_back(CalpontSystemCatalog::BIGINT); + } + widthAgg.push_back(bigIntWidth); colAgg++; } break; @@ -1758,13 +1795,21 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( if (typeAgg[colAgg] != CalpontSystemCatalog::DOUBLE && typeAgg[colAgg] != CalpontSystemCatalog::FLOAT) { - typeAggDist.push_back(CalpontSystemCatalog::BIGINT); + if (isUnsigned(typeAgg[colAgg])) + { + typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); + precisionAggDist.push_back(20); + } + else + { + typeAggDist.push_back(CalpontSystemCatalog::BIGINT); + precisionAggDist.push_back(19); + } uint32_t scale = scaleProj[colAgg]; // for int average, FE expects a decimal if (aggOp == ROWAGG_DISTINCT_AVG) scale = jobInfo.scaleOfAvg[retKey]; // scale += 4; scaleAggDist.push_back(scale); - precisionAggDist.push_back(19); widthAggDist.push_back(bigIntWidth); } else @@ -1784,7 +1829,14 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( scaleAggDist.push_back(0); // work around count() in select subquery precisionAggDist.push_back(9999); - typeAggDist.push_back(CalpontSystemCatalog::BIGINT); + if (isUnsigned(typeAgg[colAgg])) + { + typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); + } + else + { + typeAggDist.push_back(CalpontSystemCatalog::BIGINT); + } widthAggDist.push_back(bigIntWidth); } break; @@ -1853,8 +1905,16 @@ void TupleAggregateStep::prep1PhaseDistinctAggregate( oidsAggDist.push_back(oidsAgg[colAgg]); keysAggDist.push_back(retKey); scaleAggDist.push_back(0); - precisionAggDist.push_back(19); - typeAggDist.push_back(CalpontSystemCatalog::BIGINT); + if (isUnsigned(typeAgg[colAgg])) + { + typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); + precisionAggDist.push_back(20); + } + else + { + typeAggDist.push_back(CalpontSystemCatalog::BIGINT); + precisionAggDist.push_back(19); + } widthAggDist.push_back(bigIntWidth); } } @@ -2585,7 +2645,14 @@ void TupleAggregateStep::prep2PhasesAggregate( scaleAggPm.push_back(0); // work around count() in select subquery precisionAggPm.push_back(9999); - typeAggPm.push_back(CalpontSystemCatalog::BIGINT); + if (isUnsigned(typeProj[colProj])) + { + typeAggPm.push_back(CalpontSystemCatalog::UBIGINT); + } + else + { + typeAggPm.push_back(CalpontSystemCatalog::BIGINT); + } widthAggPm.push_back(bigIntWidth); colAggPm++; } @@ -2644,8 +2711,15 @@ void TupleAggregateStep::prep2PhasesAggregate( keysAggPm.push_back(aggKey); scaleAggPm.push_back(0); precisionAggPm.push_back(-16); // for connector to skip null check - typeAggPm.push_back(CalpontSystemCatalog::BIGINT); - widthAggPm.push_back(bigIntWidth); + if (isUnsigned(typeProj[colProj])) + { + typeAggPm.push_back(CalpontSystemCatalog::UBIGINT); + } + else + { + typeAggPm.push_back(CalpontSystemCatalog::BIGINT); + } + widthAggPm.push_back(bigIntWidth); colAggPm++; } break; @@ -3235,7 +3309,14 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( scaleAggPm.push_back(0); // work around count() in select subquery precisionAggPm.push_back(9999); - typeAggPm.push_back(CalpontSystemCatalog::BIGINT); + if (isUnsigned(typeProj[colProj])) + { + typeAggPm.push_back(CalpontSystemCatalog::UBIGINT); + } + else + { + typeAggPm.push_back(CalpontSystemCatalog::BIGINT); + } widthAggPm.push_back(bigIntWidth); colAggPm++; } @@ -3294,7 +3375,14 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( keysAggPm.push_back(aggKey); scaleAggPm.push_back(0); precisionAggPm.push_back(-16); // for connector to skip null check - typeAggPm.push_back(CalpontSystemCatalog::BIGINT); + if (isUnsigned(typeProj[colProj])) + { + typeAggPm.push_back(CalpontSystemCatalog::UBIGINT); + } + else + { + typeAggPm.push_back(CalpontSystemCatalog::BIGINT); + } widthAggPm.push_back(bigIntWidth); colAggPm++; } @@ -3416,13 +3504,21 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( if (typeAggUm[colUm] != CalpontSystemCatalog::DOUBLE && typeAggUm[colUm] != CalpontSystemCatalog::FLOAT) { - typeAggDist.push_back(CalpontSystemCatalog::BIGINT); + if (isUnsigned(typeAggUm[colUm])) + { + typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); + precisionAggDist.push_back(20); + } + else + { + typeAggDist.push_back(CalpontSystemCatalog::BIGINT); + precisionAggDist.push_back(19); + } uint32_t scale = scaleAggUm[colUm]; // for int average, FE expects a decimal if (aggOp == ROWAGG_DISTINCT_AVG) scale = jobInfo.scaleOfAvg[retKey]; // scale += 4; scaleAggDist.push_back(scale); - precisionAggDist.push_back(19); widthAggDist.push_back(bigIntWidth); } else @@ -3445,7 +3541,14 @@ void TupleAggregateStep::prep2PhasesDistinctAggregate( scaleAggDist.push_back(0); // work around count() in select subquery precisionAggDist.push_back(9999); - typeAggDist.push_back(CalpontSystemCatalog::BIGINT); + if (isUnsigned(typeAggUm[colUm])) + { + typeAggDist.push_back(CalpontSystemCatalog::UBIGINT); + } + else + { + typeAggDist.push_back(CalpontSystemCatalog::BIGINT); + } widthAggDist.push_back(bigIntWidth); } break; From 07d68aa8b10c46e975371cbd1367f0dfcad2f77f Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 6 Oct 2017 13:34:22 -0500 Subject: [PATCH 20/36] MCOL-898 In func_if, handle isNull better --- utils/funcexp/func_if.cpp | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/utils/funcexp/func_if.cpp b/utils/funcexp/func_if.cpp index bd3cf9dd0..f62d0d5ce 100644 --- a/utils/funcexp/func_if.cpp +++ b/utils/funcexp/func_if.cpp @@ -35,9 +35,10 @@ using namespace rowgroup; namespace { -bool boolVal(SPTP& parm, Row& row, bool& isNull) +bool boolVal(SPTP& parm, Row& row) { bool ret = true; + bool isNull = false; // Keep it local. We don't want to mess with the global one here. try { ret = parm->getBoolVal(row, isNull); @@ -119,13 +120,12 @@ int64_t Func_if::getIntVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, isNull) && !isNull) + if (boolVal(parm[0], row)) { return parm[1]->data()->getIntVal(row, isNull); } else { - isNull = false; return parm[2]->data()->getIntVal(row, isNull); } } @@ -137,14 +137,12 @@ string Func_if::getStrVal(Row& row, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, isNull)) + if (boolVal(parm[0], row)) { - isNull = false; return parm[1]->data()->getStrVal(row, isNull); } else { - isNull = false; return parm[2]->data()->getStrVal(row, isNull); } } @@ -155,14 +153,12 @@ IDB_Decimal Func_if::getDecimalVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, isNull)) + if (boolVal(parm[0], row)) { - isNull = false; return parm[1]->data()->getDecimalVal(row, isNull); } else { - isNull = false; return parm[2]->data()->getDecimalVal(row, isNull); } } @@ -173,14 +169,12 @@ double Func_if::getDoubleVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, isNull)) + if (boolVal(parm[0], row)) { - isNull = false; return parm[1]->data()->getDoubleVal(row, isNull); } else { - isNull = false; return parm[2]->data()->getDoubleVal(row, isNull); } } @@ -191,14 +185,12 @@ int32_t Func_if::getDateIntVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, isNull)) + if (boolVal(parm[0], row)) { - isNull = false; return parm[1]->data()->getDateIntVal(row, isNull); } else { - isNull = false; return parm[2]->data()->getDateIntVal(row, isNull); } } @@ -209,14 +201,12 @@ int64_t Func_if::getDatetimeIntVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType&) { - if (boolVal(parm[0], row, isNull)) + if (boolVal(parm[0], row)) { - isNull = false; return parm[1]->data()->getDatetimeIntVal(row, isNull); } else { - isNull = false; return parm[2]->data()->getDatetimeIntVal(row, isNull); } } From 5c34be958b9d8b6b86fae2563485d5e01fe3c4f3 Mon Sep 17 00:00:00 2001 From: david hill Date: Thu, 19 Oct 2017 15:48:39 -0500 Subject: [PATCH 21/36] mcol-943 --- oam/etc/Columnstore.xml | 1 - oam/etc/Columnstore.xml.singleserver | 1 - .../disable-rep-columnstore.sh | 4 +- oam/install_scripts/master-rep-columnstore.sh | 12 +- oam/install_scripts/module_installer.sh | 31 +++-- oam/install_scripts/slave-rep-columnstore.sh | 14 ++- oam/oamcpp/liboamcpp.cpp | 44 +++---- oam/oamcpp/liboamcpp.h | 4 +- oamapps/mcsadmin/mcsadmin.cpp | 61 +--------- oamapps/postConfigure/getMySQLpw.cpp | 67 ++++++++--- oamapps/postConfigure/helpers.cpp | 23 ---- oamapps/postConfigure/helpers.h | 1 - oamapps/postConfigure/postConfigure.cpp | 113 +----------------- procmgr/main.cpp | 10 +- procmgr/processmanager.cpp | 63 ++-------- procmon/main.cpp | 13 +- procmon/processmonitor.cpp | 44 +++---- 17 files changed, 155 insertions(+), 351 deletions(-) diff --git a/oam/etc/Columnstore.xml b/oam/etc/Columnstore.xml index f9ecbc116..a70de11a0 100644 --- a/oam/etc/Columnstore.xml +++ b/oam/etc/Columnstore.xml @@ -463,7 +463,6 @@ unassigned n 3306 - unassigned /dev/xvd diff --git a/oam/etc/Columnstore.xml.singleserver b/oam/etc/Columnstore.xml.singleserver index 5324cb892..8945e6367 100644 --- a/oam/etc/Columnstore.xml.singleserver +++ b/oam/etc/Columnstore.xml.singleserver @@ -458,7 +458,6 @@ unassigned unassigned 3306 - unassigned