From 098e49bd01bfc0ae07da480bb092c716f337bd1d Mon Sep 17 00:00:00 2001 From: David Hall Date: Wed, 23 Jan 2019 10:29:12 -0600 Subject: [PATCH 01/11] MCOL-2001 Make the parameters for redistribute case insensitive --- oamapps/mcsadmin/mcsadmin.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index f2ea97596..8b31353ac 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -32,6 +32,7 @@ extern int h_errno; #include "boost/filesystem/path.hpp" #include "boost/scoped_ptr.hpp" #include "boost/tokenizer.hpp" +#include "boost/algorithm/string/predicate.hpp" #include "sessionmanager.h" #include "dbrm.h" #include "messagequeue.h" @@ -677,7 +678,7 @@ int processCommand(string* arguments) vector srcDbroots; // all of the currently configured dbroots vector destDbroots; // srcDbroots - removeDbroots set::iterator dbiter; - if (arguments[1] == "start") + if (boost::iequals(arguments[1], "start")) { // Get a list of all the configured dbroots in the xml file. DBRootConfigList dbRootConfigList; @@ -688,7 +689,7 @@ int processCommand(string* arguments) // The user may choose to redistribute in such a way as to // leave certain dbroots empty, presumably for later removal. - if (arguments[2] == "remove") + if (boost::iequals(arguments[2], "remove")) { int dbroot; bool error = false; @@ -792,7 +793,7 @@ int processCommand(string* arguments) SendToWES(oam, bs); } - else if (arguments[1] == "stop") + else if (boost::iequals(arguments[1], "stop")) { ByteStream bs; // message WES ID, sequence #, action id @@ -802,7 +803,7 @@ int processCommand(string* arguments) bs.append((const ByteStream::byte*) &header, sizeof(header)); SendToWES(oam, bs); } - else if (arguments[1] == "status") + else if (boost::iequals(arguments[1], "status")) { ByteStream bs; // message WES ID, sequence #, action id From 6d65b138523e04c67ac16b1619086a87852f177b Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Sat, 26 Jan 2019 16:30:02 +0300 Subject: [PATCH 02/11] MCOL-901 Significanlty reduced memory consumption for group_concat(). RowGroup default constructor allocates memory too generously. Removed commented code from groupconcat.cpp. --- dbcon/joblist/groupconcat.cpp | 56 ++++++++--------------------------- utils/rowgroup/rowgroup.cpp | 13 ++++---- 2 files changed, 20 insertions(+), 49 deletions(-) diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index afc91a2ec..11429d053 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -344,8 +344,6 @@ void GroupConcatAgUM::merge(const rowgroup::Row& inRow, int64_t i) joblist::GroupConcatAgUM* gccAg = *((joblist::GroupConcatAgUM**)(data + inRow.getOffset(i))); fConcator->merge(gccAg->concator().get()); -// don't reset -// gccAg->orderBy().reset(); } @@ -393,12 +391,11 @@ GroupConcator::~GroupConcator() void GroupConcator::initialize(const rowgroup::SP_GroupConcat& gcc) { + // MCOL-901 This value comes from the Server and it is + // too high(3MB) to allocate it for every instance. fGroupConcatLen = gcc->fSize; fCurrentLength -= strlen(gcc->fSeparator.c_str()); - fOutputString.reset(new uint8_t[fGroupConcatLen + 2]); - memset(fOutputString.get(), 0, fGroupConcatLen + 2); - fConstCols = gcc->fConstCols; fConstantLen = strlen(gcc->fSeparator.c_str()); @@ -477,7 +474,6 @@ void GroupConcator::outputRow(std::ostringstream& oss, const rowgroup::Row& row) case CalpontSystemCatalog::TEXT: { oss << row.getStringField(*i).c_str(); - //oss << row.getStringField(*i); break; } @@ -620,11 +616,6 @@ int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row) { int64_t colWidth = row.getStringLength(*i); fieldLen += colWidth; // getStringLength() does the same thing as below - //assert(!row.usesStringTable()); - //int64_t colWidth = row.getColumnWidth(*i); - //uint8_t* pStr = row.getData() + row.getOffset(*i); - //while ((*pStr++ > 0) && (fieldLen < colWidth)) - // fieldLen++; break; } @@ -752,7 +743,6 @@ void GroupConcatOrderBy::initialize(const rowgroup::SP_GroupConcat& gcc) uint64_t GroupConcatOrderBy::getKeyLength() const { // only distinct the concatenated columns - //return (fRow0.getOffset(fConcatColumns.size()) - 2); return fConcatColumns.size() - 1; // cols 0 to fConcatColumns will be conpared } @@ -771,8 +761,6 @@ void GroupConcatOrderBy::processRow(const rowgroup::Row& row) if (fCurrentLength < fGroupConcatLen) { copyRow(row, &fRow0); - //cout << "length < GB limit: " << fRow0.toString() << endl; - //memcpy(fRow0.getData(), row.getData(), row.getSize()); // the RID is no meaning here, use it to store the estimated length. int16_t estLen = lengthEstimate(fRow0); fRow0.setRid(estLen); @@ -862,7 +850,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc) if (fDistinct) fDistinctMap->insert(row.fData); - //fDistinctMap->insert(make_pair((row.fData+2), row.fData)); } else if (fOrderByCond.size() > 0 && fRule.less(row.fData, fOrderByQueue.top().fData)) @@ -876,8 +863,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc) { fDistinctMap->erase(swapRow.fData); fDistinctMap->insert(row.fData); - //fDistinctMap->erase(fDistinctMap->find(swapRow.fData + 2)); - //fDistinctMap->insert(make_pair((row.fData+2), row.fData)); } row1.setData(row.fData); @@ -893,7 +878,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc) void GroupConcatOrderBy::getResult(uint8_t* buff, const string& sep) { -#if 1 ostringstream oss; bool addSep = false; @@ -919,28 +903,12 @@ void GroupConcatOrderBy::getResult(uint8_t* buff, const string& sep) rowStack.pop(); } - strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen); -#else - ostringstream oss; - bool addSep = false; - priority_queue::reverse_iterator rit; + size_t resultSize = oss.str().size(); + fOutputString.reset(new uint8_t[resultSize + 2]); + memset(fOutputString.get(), 0, resultSize + 2); - for (rit = fOrderByQueue.rbegin(); rit != fOrderByQueue.rend(); ++rit) - { - if (addSep) - oss << sep; - else - addSep = true; - - const OrderByRow& topRow = *rit; - fRow0.setData(topRow.fData); - outputRow(oss, fRow0); - } - - fOrderByQueue.clear(); - - strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen); -#endif + strncpy((char*)fOutputString.get(), + oss.str().c_str(), resultSize); } uint8_t* GroupConcator::getResult(const string& sep) @@ -1010,7 +978,6 @@ void GroupConcatNoOrder::initialize(const rowgroup::SP_GroupConcat& gcc) throw IDBExcept(fErrorCode); } - //fData.reset(new uint8_t[fRowGroup.getDataSize(fRowsPerRG)]); fData.reinit(fRowGroup, fRowsPerRG); fRowGroup.setData(&fData); fRowGroup.resetRowGroup(0); @@ -1025,7 +992,6 @@ void GroupConcatNoOrder::processRow(const rowgroup::Row& row) if (fCurrentLength < fGroupConcatLen && concatColIsNull(row) == false) { copyRow(row, &fRow); - //memcpy(fRow.getData(), row.getData(), row.getSize()); // the RID is no meaning here, use it to store the estimated length. int16_t estLen = lengthEstimate(fRow); @@ -1049,7 +1015,6 @@ void GroupConcatNoOrder::processRow(const rowgroup::Row& row) fDataQueue.push(fData); fData.reinit(fRowGroup, fRowsPerRG); - //fData.reset(new uint8_t[fRowGroup.getDataSize(fRowsPerRG)]); fRowGroup.setData(&fData); fRowGroup.resetRowGroup(0); fRowGroup.getRow(0, &fRow); @@ -1100,7 +1065,12 @@ void GroupConcatNoOrder::getResult(uint8_t* buff, const string& sep) fDataQueue.pop(); } - strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen); + size_t resultSize = oss.str().size(); + fOutputString.reset(new uint8_t[resultSize + 2]); + memset(fOutputString.get(), 0, resultSize + 2); + + strncpy((char*)fOutputString.get(), + oss.str().c_str(), resultSize); } diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 3411f6fa6..3ecb4998f 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -1211,12 +1211,13 @@ int64_t Row::getSignedNullValue(uint32_t colIndex) const RowGroup::RowGroup() : columnCount(0), data(NULL), rgData(NULL), strings(NULL), useStringTable(true), hasLongStringField(false), sTableThreshold(20) { - oldOffsets.reserve(1024); - oids.reserve(1024); - keys.reserve(1024); - types.reserve(1024); - scale.reserve(1024); - precision.reserve(1024); + // 1024 is too generous to waste. + oldOffsets.reserve(10); + oids.reserve(10); + keys.reserve(10); + types.reserve(10); + scale.reserve(10); + precision.reserve(10); } RowGroup::RowGroup(uint32_t colCount, From 773a665dabd8d5de7a00c4563f17713e06a79363 Mon Sep 17 00:00:00 2001 From: Roman Nozdrin Date: Wed, 2 Jan 2019 21:09:55 +0300 Subject: [PATCH 03/11] MCOL-2050 ORDER BY with OFFSET in subquery now produces correct result. --- dbcon/joblist/limitedorderby.cpp | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/dbcon/joblist/limitedorderby.cpp b/dbcon/joblist/limitedorderby.cpp index 64caf078c..de7f9bcb6 100644 --- a/dbcon/joblist/limitedorderby.cpp +++ b/dbcon/joblist/limitedorderby.cpp @@ -173,10 +173,10 @@ void LimitedOrderBy::finalize() if (fOrderByQueue.size() > 0) { - uint64_t newSize = fRowsPerRG * fRowGroup.getRowSize(); - fMemSize += newSize; + uint64_t memSizeInc = fRowsPerRG * fRowGroup.getRowSize(); + fMemSize += memSizeInc; - if (!fRm->getMemory(newSize, fSessionMemLimit)) + if (!fRm->getMemory(memSizeInc, fSessionMemLimit)) { cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) << " @" << __FILE__ << ":" << __LINE__; @@ -185,10 +185,12 @@ void LimitedOrderBy::finalize() uint64_t offset = 0; uint64_t i = 0; + // Reduce queue size by an offset value if it applicable. + uint64_t queueSizeWoOffset = fOrderByQueue.size() > fStart ? + fOrderByQueue.size() - fStart : 0; list tempRGDataList; - - // Skip first LIMIT rows in the the RowGroup - if ( fCount <= fOrderByQueue.size() ) + + if ( fCount <= queueSizeWoOffset ) { offset = fCount % fRowsPerRG; if(!offset && fCount > 0) @@ -196,11 +198,11 @@ void LimitedOrderBy::finalize() } else { - offset = fOrderByQueue.size() % fRowsPerRG; - if(!offset && fOrderByQueue.size() > 0) + offset = queueSizeWoOffset % fRowsPerRG; + if(!offset && queueSizeWoOffset > 0) offset = fRowsPerRG; } - + list::iterator tempListIter = tempRGDataList.begin(); i = 0; @@ -217,7 +219,6 @@ void LimitedOrderBy::finalize() const OrderByRow& topRow = fOrderByQueue.top(); row1.setData(topRow.fData); copyRow(row1, &fRow0); - //cerr << "LimitedOrderBy::finalize fRow0 " << fRow0.toString() << endl; fRowGroup.incRowCount(); offset--; fRow0.prevRow(rSize); @@ -226,9 +227,9 @@ void LimitedOrderBy::finalize() if(offset == (uint64_t)-1) { tempRGDataList.push_front(fData); - fMemSize += newSize; + fMemSize += memSizeInc; - if (!fRm->getMemory(newSize, fSessionMemLimit)) + if (!fRm->getMemory(memSizeInc, fSessionMemLimit)) { cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) << " @" << __FILE__ << ":" << __LINE__; From dd05b3af13f834edeac57b22dfdd4d93cfac81b5 Mon Sep 17 00:00:00 2001 From: sneJ- Date: Fri, 1 Feb 2019 15:00:26 +0100 Subject: [PATCH 04/11] MCOL-2129 - added a new postConfigure flag -xr to reverse dns lookup given hostnames and use that value instead of the original hostname value --- oamapps/postConfigure/postConfigure.cpp | 209 ++++++++++++++---------- 1 file changed, 121 insertions(+), 88 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index c92697127..c4e9b2126 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -55,7 +55,6 @@ #include #include #include -#include #include /* for strncpy */ #include @@ -111,8 +110,6 @@ typedef std::vector ModuleIpList; void offLineAppCheck(); bool setOSFiles(string parentOAMModuleName, int serverTypeInstall); bool checkSaveConfigFile(); -string getModuleName(); -bool setModuleName(string moduleName); bool updateBash(); bool makeModuleFile(string moduleName, string parentOAMModuleName); bool updateProcessConfig(); @@ -126,6 +123,7 @@ bool singleServerDBrootSetup(); bool copyFstab(string moduleName); bool attachVolume(string instanceName, string volumeName, string deviceName, string dbrootPath); void singleServerConfigSetup(Config* sysConfig); +std::string resolveHostNameToReverseDNSName(std::string hostname); void remoteInstallThread(void*); @@ -193,6 +191,7 @@ bool single_server_quick_install = false; bool multi_server_quick_install = false; bool amazon_quick_install = false; bool doNotResolveHostNames = false; +bool resolveHostNamesToReverseDNSNames = false; string DataFileEnvFile; @@ -316,7 +315,7 @@ int main(int argc, char* argv[]) cout << " Enter one of the options within [], if available, or" << endl; cout << " Enter a new value" << endl << endl; cout << endl; - cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-numBlocksPct][-totalUmMemory]" << endl; + cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-x][-xr][-numBlocksPct][-totalUmMemory]" << endl; cout << " -h Help" << endl; cout << " -c Config File to use to extract configuration data, default is Columnstore.xml.rpmsave" << endl; cout << " -u Upgrade, Install using the Config File from -c, default to Columnstore.xml.rpmsave" << endl; @@ -332,6 +331,7 @@ int main(int argc, char* argv[]) cout << " -pm-ip-addrs Performance Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; cout << " -um-ip-addrs User Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; cout << " -x Do not resolve IP Addresses from host names" << endl; + cout << " -xr Resolve host names into their reverse DNS host names. Only applied in combination with -x" << endl; cout << " -numBlocksPct amount of physical memory to utilize for disk block caching" << endl; cout << " (percentages of the total memory need to be stated without suffix, explcit values with suffixes M or G)" << endl; cout << " -totalUmMemory amount of physical memory to utilize for joins, intermediate results and set operations on the UM" << endl; @@ -341,6 +341,10 @@ int main(int argc, char* argv[]) else if (string("-x") == argv[i]) { doNotResolveHostNames = true; + } + else if (string("-xr") == argv[i]) + { + resolveHostNamesToReverseDNSNames = true; } else if( string("-qs") == argv[i] ) { @@ -517,7 +521,7 @@ int main(int argc, char* argv[]) else { cout << " ERROR: Invalid Argument = " << argv[i] << endl; - cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-numBlocksPct][-totalUmMemory]" << endl; + cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-x][-xr][-numBlocksPct][-totalUmMemory]" << endl; exit (1); } } @@ -2566,7 +2570,12 @@ int main(int argc, char* argv[]) //get IP Address string IPAddress; if (doNotResolveHostNames) - IPAddress = newModuleHostName; + if (resolveHostNamesToReverseDNSNames) { + IPAddress = resolveHostNameToReverseDNSName(newModuleHostName); + } + else { + IPAddress = newModuleHostName; + } else IPAddress = oam.getIPAddress( newModuleHostName); @@ -6550,7 +6559,12 @@ bool glusterSetup(string password, bool doNotResolveHostNames) //get IP Address string IPAddress; if (doNotResolveHostNames) - IPAddress = moduleHostName; + if (resolveHostNamesToReverseDNSNames) { + IPAddress = resolveHostNameToReverseDNSName(moduleHostName); + } + else { + IPAddress = moduleHostName; + } else IPAddress = oam.getIPAddress( moduleHostName); @@ -7011,29 +7025,29 @@ bool glusterSetup(string password, bool doNotResolveHostNames) command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-uid " + oam.itoa(user) + " >> /tmp/glusterCommands.txt 2>&1";; status = system(command.c_str()); - if (WEXITSTATUS(status) != 0 ) - { - cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; - exit(1); - } +if (WEXITSTATUS(status) != 0) +{ + cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; + exit(1); +} - command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-gid " + oam.itoa(group) + " >> /tmp/glusterCommands.txt 2>&1";; - status = system(command.c_str()); +command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-gid " + oam.itoa(group) + " >> /tmp/glusterCommands.txt 2>&1";; +status = system(command.c_str()); - if (WEXITSTATUS(status) != 0 ) - { - cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; - exit(1); - } +if (WEXITSTATUS(status) != 0) +{ + cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; + exit(1); +} - command = "sudo gluster volume start dbroot" + oam.itoa(dbrootID) + " >> /tmp/glusterCommands.txt 2>&1"; - status = system(command.c_str()); +command = "sudo gluster volume start dbroot" + oam.itoa(dbrootID) + " >> /tmp/glusterCommands.txt 2>&1"; +status = system(command.c_str()); - if (WEXITSTATUS(status) != 0 ) - { - cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; - exit(1); - } +if (WEXITSTATUS(status) != 0) +{ + cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; + exit(1); +} } cout << "DONE" << endl; @@ -7047,70 +7061,89 @@ bool glusterSetup(string password, bool doNotResolveHostNames) void singleServerConfigSetup(Config* sysConfig) { - try - { - sysConfig->setConfig("ExeMgr1", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("ExeMgr1", "Module", "pm1"); - sysConfig->setConfig("ProcMgr", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("ProcMgr_Alarm", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("ProcStatusControl", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("pm1_ProcessMonitor", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("pm1_ServerMonitor", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("pm1_WriteEngineServer", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("DDLProc", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("DMLProc", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS1", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS2", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS3", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS4", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS5", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS6", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS7", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS8", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS9", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS10", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS11", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS12", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS13", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS14", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS15", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS16", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS17", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS18", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS19", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS20", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS21", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS22", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS23", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS24", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS25", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS26", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS27", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS28", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS29", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS30", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS31", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS32", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("SystemModuleConfig", "ModuleCount2", "0"); - sysConfig->setConfig("SystemModuleConfig", "ModuleIPAddr1-1-3", "127.0.0.1"); - sysConfig->setConfig("SystemModuleConfig", "ModuleHostName1-1-3", "localhost"); - sysConfig->setConfig("DBRM_Controller", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("DBRM_Worker1", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("DBRM_Worker1", "Module", "pm1"); - sysConfig->setConfig("DBBC", "NumBlocksPct", "50"); - sysConfig->setConfig("Installation", "InitialInstallFlag", "y"); - sysConfig->setConfig("Installation", "SingleServerInstall", "y"); - sysConfig->setConfig("HashJoin", "TotalUmMemory", "25%"); - } - catch (...) - { - cout << "ERROR: Problem setting for Single Server in the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } + try + { + sysConfig->setConfig("ExeMgr1", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("ExeMgr1", "Module", "pm1"); + sysConfig->setConfig("ProcMgr", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("ProcMgr_Alarm", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("ProcStatusControl", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("pm1_ProcessMonitor", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("pm1_ServerMonitor", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("pm1_WriteEngineServer", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("DDLProc", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("DMLProc", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS1", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS2", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS3", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS4", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS5", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS6", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS7", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS8", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS9", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS10", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS11", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS12", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS13", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS14", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS15", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS16", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS17", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS18", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS19", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS20", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS21", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS22", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS23", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS24", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS25", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS26", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS27", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS28", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS29", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS30", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS31", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS32", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("SystemModuleConfig", "ModuleCount2", "0"); + sysConfig->setConfig("SystemModuleConfig", "ModuleIPAddr1-1-3", "127.0.0.1"); + sysConfig->setConfig("SystemModuleConfig", "ModuleHostName1-1-3", "localhost"); + sysConfig->setConfig("DBRM_Controller", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("DBRM_Worker1", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("DBRM_Worker1", "Module", "pm1"); + sysConfig->setConfig("DBBC", "NumBlocksPct", "50"); + sysConfig->setConfig("Installation", "InitialInstallFlag", "y"); + sysConfig->setConfig("Installation", "SingleServerInstall", "y"); + sysConfig->setConfig("HashJoin", "TotalUmMemory", "25%"); + } + catch (...) + { + cout << "ERROR: Problem setting for Single Server in the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } - return; + return; } - +/** + Resolves the given hostname into its reverse DNS name. + + @param hostname the hostname to resolve. + @return the reverse dns name of given hostname or an empty string in case the hostname could not be resolved. +*/ +std::string resolveHostNameToReverseDNSName(std::string hostname) { + struct hostent *hp = gethostbyname(hostname.c_str()); + if (hp == NULL) { + std::cout << "Error: Couldn't resolve hostname " << hostname << " to ip address" << std::endl; + return ""; + } + struct hostent *rl = gethostbyaddr(hp->h_addr_list[0], sizeof hp->h_addr_list[0], AF_INET); + if (rl == NULL) { + std::cout << "Error: Couldn't resolve ip address of hostname " << hostname << " back to a hostname" << std::endl; + return ""; + } + hostname = rl->h_name; + return hostname; +} // vim:ts=4 sw=4: From bf4a215cc8d721be09cd2c311ab95c951ae8cef9 Mon Sep 17 00:00:00 2001 From: David Hall Date: Mon, 4 Feb 2019 12:32:04 -0600 Subject: [PATCH 05/11] MCOL-2001 make redistribute parameters case insensitive. --- oamapps/mcsadmin/mcsadmin.cpp | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 8b31353ac..dd7262d1e 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -32,7 +32,6 @@ extern int h_errno; #include "boost/filesystem/path.hpp" #include "boost/scoped_ptr.hpp" #include "boost/tokenizer.hpp" -#include "boost/algorithm/string/predicate.hpp" #include "sessionmanager.h" #include "dbrm.h" #include "messagequeue.h" @@ -678,7 +677,11 @@ int processCommand(string* arguments) vector srcDbroots; // all of the currently configured dbroots vector destDbroots; // srcDbroots - removeDbroots set::iterator dbiter; - if (boost::iequals(arguments[1], "start")) +#if _MSC_VER + if (_strnicmp(arguments[1].c_str(), "start", 5) == 0)) +#else + if (strncasecmp(arguments[1].c_str(), "start", 5) == 0) +#endif { // Get a list of all the configured dbroots in the xml file. DBRootConfigList dbRootConfigList; @@ -689,7 +692,11 @@ int processCommand(string* arguments) // The user may choose to redistribute in such a way as to // leave certain dbroots empty, presumably for later removal. - if (boost::iequals(arguments[2], "remove")) +#if _MSC_VER + if (_strnicmp(arguments[1].c_str(), "remove", 6) == 0)) +#else + if (strncasecmp(arguments[1].c_str(), "remove", 6) == 0) +#endif { int dbroot; bool error = false; @@ -793,7 +800,11 @@ int processCommand(string* arguments) SendToWES(oam, bs); } - else if (boost::iequals(arguments[1], "stop")) +#if _MSC_VER + if (_strnicmp(arguments[1].c_str(), "stop", 4) == 0)) +#else + if (strncasecmp(arguments[1].c_str(), "stop", 4) == 0) +#endif { ByteStream bs; // message WES ID, sequence #, action id @@ -803,7 +814,11 @@ int processCommand(string* arguments) bs.append((const ByteStream::byte*) &header, sizeof(header)); SendToWES(oam, bs); } - else if (boost::iequals(arguments[1], "status")) +#if _MSC_VER + if (_strnicmp(arguments[1].c_str(), "status", 6) == 0)) +#else + if (strncasecmp(arguments[1].c_str(), "status", 6) == 0) +#endif { ByteStream bs; // message WES ID, sequence #, action id From b81455744d938c442d248b09b10ebbee65e4e2ee Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Tue, 5 Feb 2019 13:43:21 -0600 Subject: [PATCH 06/11] Revert "Kill Jemalloc" This reverts commit e03f7dc88503c7044ebc5befdd432238e31d3f16. --- CMakeLists.txt | 8 ++++++- FindJeMalloc.cmake | 44 ++++++++++++++++++++++++++++++++++++++ build/infinidb.spec.in | 1 + build/infinidb.spec.in.4.5 | 1 + 4 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 FindJeMalloc.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 0932307ed..2645adc5c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -92,6 +92,12 @@ if (NOT LIBXML2_FOUND) MESSAGE(FATAL_ERROR "Could not find a usable libxml2 development environment!") endif() +INCLUDE (FindJeMalloc.cmake) +if (NOT JEMALLOC_FOUND) + message(NOTICE "jemalloc not found! Compiling with standard malloc") + SET(JEMALLOC_LIBRARIES "") +endif() + FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable") if(NOT AWK_EXECUTABLE) message(FATAL_ERROR "awk not found!") @@ -123,7 +129,7 @@ SET (ENGINE_LOCALDIR "${INSTALL_ENGINE}/local") SET (ENGINE_MYSQLDIR "${INSTALL_ENGINE}/mysql") SET (ENGINE_TOOLSDIR "${INSTALL_ENGINE}/tools") -SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt) +SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt ${JEMALLOC_LIBRARIES}) SET (ENGINE_OAM_LIBS oamcpp alarmmanager) SET (ENGINE_BRM_LIBS brm idbdatafile cacheutils rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS}) SET (ENGINE_EXEC_LIBS joblist execplan windowfunction joiner rowgroup funcexp udfsdk dataconvert common compress mysqlcl_idb querystats querytele thrift threadpool ${ENGINE_BRM_LIBS}) diff --git a/FindJeMalloc.cmake b/FindJeMalloc.cmake new file mode 100644 index 000000000..c97bb1b9f --- /dev/null +++ b/FindJeMalloc.cmake @@ -0,0 +1,44 @@ +# - Try to find jemalloc headers and libraries. +# +# Usage of this module as follows: +# +# find_package(JeMalloc) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# JEMALLOC_ROOT_DIR Set this variable to the root installation of +# jemalloc if the module has problems finding +# the proper installation path. +# +# Variables defined by this module: +# +# JEMALLOC_FOUND System has jemalloc libs/headers +# JEMALLOC_LIBRARIES The jemalloc library/libraries +# JEMALLOC_INCLUDE_DIR The location of jemalloc headers + +find_path(JEMALLOC_ROOT_DIR + NAMES include/jemalloc/jemalloc.h +) + +find_library(JEMALLOC_LIBRARIES + NAMES jemalloc + HINTS ${JEMALLOC_ROOT_DIR}/lib +) + +find_path(JEMALLOC_INCLUDE_DIR + NAMES jemalloc/jemalloc.h + HINTS ${JEMALLOC_ROOT_DIR}/include +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(JeMalloc DEFAULT_MSG + JEMALLOC_LIBRARIES + JEMALLOC_INCLUDE_DIR +) + +mark_as_advanced( + JEMALLOC_ROOT_DIR + JEMALLOC_LIBRARIES + JEMALLOC_INCLUDE_DIR +) diff --git a/build/infinidb.spec.in b/build/infinidb.spec.in index 20a44e834..2531044a0 100644 --- a/build/infinidb.spec.in +++ b/build/infinidb.spec.in @@ -336,6 +336,7 @@ rm -rf $RPM_BUILD_ROOT $RPM_BUILD_DIR/%{name}-%{version}-%{release} /usr/local/mariadb/columnstore/lib/libmysqlcl_idb.so.1.0.0 /usr/local/mariadb/columnstore/lib/libquerystats.so.1.0.0 /usr/local/mariadb/columnstore/lib/libwriteengineredistribute.so.1.0.0 +/usr/local/mariadb/columnstore/lib/libjemalloc.so.3.3.0 /usr/local/mariadb/columnstore/lib/libidbdatafile.so.1.0.0 /usr/local/mariadb/columnstore/lib/hdfs-20.so /usr/local/mariadb/columnstore/lib/hdfs-12.so diff --git a/build/infinidb.spec.in.4.5 b/build/infinidb.spec.in.4.5 index 966f2a73a..b8145d580 100755 --- a/build/infinidb.spec.in.4.5 +++ b/build/infinidb.spec.in.4.5 @@ -338,6 +338,7 @@ rm -rf $RPM_BUILD_ROOT $RPM_BUILD_DIR/%{name}-%{version}-%{release} /usr/local/Calpont/lib/libmysqlcl_idb.so.1.0.0 /usr/local/Calpont/lib/libquerystats.so.1.0.0 /usr/local/Calpont/lib/libwriteengineredistribute.so.1.0.0 +/usr/local/Calpont/lib/libjemalloc.so.3.3.0 /usr/local/Calpont/lib/libidbdatafile.so.1.0.0 /usr/local/Calpont/lib/hdfs-20.so /usr/local/Calpont/lib/hdfs-12.so From 52a741e6efdd7d22b29701ab1cce0ae86278886e Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Tue, 5 Feb 2019 17:18:02 -0600 Subject: [PATCH 07/11] MCOL-2136: Force use of jemalloc for build and install. --- CMakeLists.txt | 2 +- build/infinidb.spec.in | 1 - build/infinidb.spec.in.4.5 | 1 - cpackEngineDEB.cmake | 2 +- cpackEngineRPM.cmake | 4 ++-- 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2645adc5c..c72a74d61 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -94,7 +94,7 @@ endif() INCLUDE (FindJeMalloc.cmake) if (NOT JEMALLOC_FOUND) - message(NOTICE "jemalloc not found! Compiling with standard malloc") + message(FATAL_ERROR "jemalloc not found!") SET(JEMALLOC_LIBRARIES "") endif() diff --git a/build/infinidb.spec.in b/build/infinidb.spec.in index 2531044a0..20a44e834 100644 --- a/build/infinidb.spec.in +++ b/build/infinidb.spec.in @@ -336,7 +336,6 @@ rm -rf $RPM_BUILD_ROOT $RPM_BUILD_DIR/%{name}-%{version}-%{release} /usr/local/mariadb/columnstore/lib/libmysqlcl_idb.so.1.0.0 /usr/local/mariadb/columnstore/lib/libquerystats.so.1.0.0 /usr/local/mariadb/columnstore/lib/libwriteengineredistribute.so.1.0.0 -/usr/local/mariadb/columnstore/lib/libjemalloc.so.3.3.0 /usr/local/mariadb/columnstore/lib/libidbdatafile.so.1.0.0 /usr/local/mariadb/columnstore/lib/hdfs-20.so /usr/local/mariadb/columnstore/lib/hdfs-12.so diff --git a/build/infinidb.spec.in.4.5 b/build/infinidb.spec.in.4.5 index b8145d580..966f2a73a 100755 --- a/build/infinidb.spec.in.4.5 +++ b/build/infinidb.spec.in.4.5 @@ -338,7 +338,6 @@ rm -rf $RPM_BUILD_ROOT $RPM_BUILD_DIR/%{name}-%{version}-%{release} /usr/local/Calpont/lib/libmysqlcl_idb.so.1.0.0 /usr/local/Calpont/lib/libquerystats.so.1.0.0 /usr/local/Calpont/lib/libwriteengineredistribute.so.1.0.0 -/usr/local/Calpont/lib/libjemalloc.so.3.3.0 /usr/local/Calpont/lib/libidbdatafile.so.1.0.0 /usr/local/Calpont/lib/hdfs-20.so /usr/local/Calpont/lib/hdfs-12.so diff --git a/cpackEngineDEB.cmake b/cpackEngineDEB.cmake index 30d3d0ab9..13865c05b 100644 --- a/cpackEngineDEB.cmake +++ b/cpackEngineDEB.cmake @@ -60,7 +60,7 @@ SET(CPACK_DEBIAN_LIBS_PACKAGE_PROVIDES "mariadb-columnstore-libs") SET(CPACK_DEBIAN_PLATFORM_PACKAGE_PROVIDES "mariadb-columnstore-platform") SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_PROVIDES "mariadb-columnstore-storage-engine") -SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, libboost-all-dev, mariadb-columnstore-libs") +SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, libboost-all-dev, mariadb-columnstore-libs, libjemalloc1") SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_DEPENDS "mariadb-columnstore-libs") diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake index 98974325d..12da14498 100644 --- a/cpackEngineRPM.cmake +++ b/cpackEngineRPM.cmake @@ -90,9 +90,9 @@ if (${REDHAT_VERSION_NUMBER} EQUAL 6) # Disable auto require as this will also try to pull Boost via RPM SET(CPACK_RPM_PACKAGE_AUTOREQPROV " no") elseif (${SUSE_VERSION_NUMBER} EQUAL 12) - SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost-devel >= 1.54.0" "mariadb-columnstore-libs") + SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost-devel >= 1.54.0" "mariadb-columnstore-libs" "jemalloc") else () - SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "mariadb-columnstore-libs") + SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "mariadb-columnstore-libs" "jemalloc") endif() SETA(CPACK_RPM_storage-engine_PACKAGE_REQUIRES "mariadb-columnstore-libs") From f805478eb91173676012c481e8dc81511707963e Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Fri, 8 Feb 2019 17:55:50 +0000 Subject: [PATCH 08/11] MCOL-2149 Fix cpimport decimal saturation If we saturate int64_t during string -> decimal conversion then end processing there instead of continuing. This preserves a good saturation value. --- writeengine/shared/we_convertor.cpp | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index 8ad344ce3..7dcad3986 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -182,19 +182,26 @@ long long Convertor::convertDecimalString( { long double dval = strtold(field, NULL); long long ret = 0; - + // move scale digits to the left of the decimal point for (int i = 0; i < scale; i++) dval *= 10; - + // range check against int64 - if (dval > LLONG_MAX || dval < LLONG_MIN) + if (dval > LLONG_MAX) + { errno = ERANGE; - else - errno = 0; - + return LLONG_MAX; + } + if (dval < LLONG_MIN) + { + errno = ERANGE; + return LLONG_MIN; + } + errno = 0; + ret = dval; - + // get the fractional part of what's left & round ret up or down. dval -= ret; if (dval >= 0.5 && ret < LLONG_MAX) From 9c71f3d48086b5963a020cc9bd9d2316ca22a0cd Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 13 Feb 2019 21:23:08 +0000 Subject: [PATCH 09/11] Bump version to 1.0.17 --- README.md | 8 ++++---- VERSION | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3ad645174..bac34bdd2 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -#MariaDB ColumnStore Storage/Execution engine 1.0.16 -MariaDB ColumnStore 1.0.16 is the development version of MariaDB ColumnStore. -It is built by porting InfiniDB 4.6.7 on MariaDB 10.1.35 and adding entirely +#MariaDB ColumnStore Storage/Execution engine 1.0.17 +MariaDB ColumnStore 1.0.17 is the development version of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.7 on MariaDB 10.1 and adding entirely new features not found anywhere else. -#MariaDB ColumnStore 1.0.16 is an GA release. +#MariaDB ColumnStore 1.0.17 is an GA release. #Building This repository is not meant to be built independently outside of the server. This repository is integrated into http://mariadb-corporation/mariadb-columnstore-server (ie, the *server*) as a git submodule. As such, you can find complete build instructions on *the server* page. diff --git a/VERSION b/VERSION index b13a94d3f..7e6391a76 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 COLUMNSTORE_VERSION_MINOR=0 -COLUMNSTORE_VERSION_PATCH=16 +COLUMNSTORE_VERSION_PATCH=17 COLUMNSTORE_VERSION_RELEASE=1 From 620519384593c0b2ada9dcb24ce52b9a6e1463c8 Mon Sep 17 00:00:00 2001 From: Andrew Hutchings Date: Wed, 13 Feb 2019 21:50:44 +0000 Subject: [PATCH 10/11] Disable jemalloc on SLES --- CMakeLists.txt | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 387d34f3a..2053bbad7 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,11 +112,16 @@ if (NOT SNAPPY_FOUND) MESSAGE(FATAL_ERROR "Snappy not found please install snappy-devel for CentOS/RedHat or libsnappy-dev for Ubuntu/Debian") endif() -INCLUDE (FindJeMalloc.cmake) -if (NOT JEMALLOC_FOUND) - message(FATAL_ERROR "jemalloc not found!") +# Jemalloc has issues with SLES 12, so disable for now +IF (EXISTS "/etc/SuSE-release") SET(JEMALLOC_LIBRARIES "") -endif() +ELSE () + INCLUDE (FindJeMalloc.cmake) + if (NOT JEMALLOC_FOUND) + message(FATAL_ERROR "jemalloc not found!") + SET(JEMALLOC_LIBRARIES "") + endif() +ENDIF () FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable") if(NOT AWK_EXECUTABLE) From 7dfcc3c69ca8dd641c35864275f1cfb0cd0dbf23 Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Thu, 14 Feb 2019 10:01:27 -0600 Subject: [PATCH 11/11] MCOL-1607. Let postconfigure store hostnames in the configuration. Made postconfig resolve hostname -> IP for its 'am I PM1' check. Made the text of one of the prompts more consistent. Made one of the paths it goes down to get ip addresses do hostname -> IP. Unrelated to ticket, just something Daniel noticed. --- oamapps/postConfigure/postConfigure.cpp | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index c4e9b2126..fdfc1482c 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -662,9 +662,14 @@ int main(int argc, char* argv[]) if (moduleconfig.hostConfigList.size() > 0 ) { HostConfigList::iterator pt1 = moduleconfig.hostConfigList.begin(); - string PM1ipAdd = (*pt1).IPAddr; + + // MCOL-1607. The 'am I pm1?' check below requires an ipaddr. + string PM1ipAdd = oam.getIPAddress((*pt1).IPAddr.c_str()); + if (PM1ipAdd.empty()) + PM1ipAdd = (*pt1).IPAddr; // this is what it was doing before + //cout << PM1ipAdd << endl; - + if ( PM1ipAdd != "127.0.0.1" ) { if ( PM1ipAdd != "0.0.0.0") @@ -2602,6 +2607,13 @@ int main(int argc, char* argv[]) callFree(pcommand); } + + if (!doNotResolveHostNames) + { + string ugh = oam.getIPAddress(newModuleIPAddr); + if (ugh.length() > 0) + newModuleIPAddr = ugh; + } if (newModuleIPAddr == "127.0.0.1" || newModuleIPAddr == "0.0.0.0" || newModuleIPAddr == "128.0.0.1") { @@ -6580,7 +6592,7 @@ bool glusterSetup(string password, bool doNotResolveHostNames) //prompt for IP address while (true) { - prompt = "Enter PM #" + oam.itoa(DataRedundancyConfigs[pm].pmID) + " IP Address of " + moduleHostName + " (" + moduleIPAddr + ") > "; + prompt = "Enter PM #" + oam.itoa(DataRedundancyConfigs[pm].pmID) + " IP Address or hostname of " + moduleHostName + " (" + moduleIPAddr + ") > "; pcommand = callReadline(prompt.c_str()); if (pcommand) @@ -6589,6 +6601,13 @@ bool glusterSetup(string password, bool doNotResolveHostNames) callFree(pcommand); } + + if (!doNotResolveHostNames) + { + string ugh = oam.getIPAddress(moduleIPAddr); + if (ugh.length() > 0) + moduleIPAddr = ugh; + } if (moduleIPAddr == "127.0.0.1" || moduleIPAddr == "0.0.0.0" || moduleIPAddr == "128.0.0.1") {