diff --git a/CMakeLists.txt b/CMakeLists.txt index 7379cd23c..94fdde894 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,6 +112,17 @@ if (NOT SNAPPY_FOUND) MESSAGE(FATAL_ERROR "Snappy not found please install snappy-devel for CentOS/RedHat or libsnappy-dev for Ubuntu/Debian") endif() +# Jemalloc has issues with SLES 12, so disable for now +IF (EXISTS "/etc/SuSE-release") + SET(JEMALLOC_LIBRARIES "") +ELSE () + INCLUDE (FindJeMalloc.cmake) + if (NOT JEMALLOC_FOUND) + message(FATAL_ERROR "jemalloc not found!") + SET(JEMALLOC_LIBRARIES "") + endif() +ENDIF () + FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable") if(NOT AWK_EXECUTABLE) message(FATAL_ERROR "awk not found!") @@ -163,7 +174,7 @@ SET (ENGINE_LOCALDIR "${INSTALL_ENGINE}/local") SET (ENGINE_MYSQLDIR "${INSTALL_ENGINE}/mysql") SET (ENGINE_TOOLSDIR "${INSTALL_ENGINE}/tools") -SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client) +SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client ${JEMALLOC_LIBRARIES}) SET (ENGINE_OAM_LIBS oamcpp alarmmanager) SET (ENGINE_BRM_LIBS brm idbdatafile cacheutils rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS}) SET (ENGINE_EXEC_LIBS joblist execplan windowfunction joiner rowgroup funcexp udfsdk regr dataconvert common compress querystats querytele thrift threadpool ${ENGINE_BRM_LIBS}) diff --git a/FindJeMalloc.cmake b/FindJeMalloc.cmake new file mode 100644 index 000000000..c97bb1b9f --- /dev/null +++ b/FindJeMalloc.cmake @@ -0,0 +1,44 @@ +# - Try to find jemalloc headers and libraries. +# +# Usage of this module as follows: +# +# find_package(JeMalloc) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# JEMALLOC_ROOT_DIR Set this variable to the root installation of +# jemalloc if the module has problems finding +# the proper installation path. +# +# Variables defined by this module: +# +# JEMALLOC_FOUND System has jemalloc libs/headers +# JEMALLOC_LIBRARIES The jemalloc library/libraries +# JEMALLOC_INCLUDE_DIR The location of jemalloc headers + +find_path(JEMALLOC_ROOT_DIR + NAMES include/jemalloc/jemalloc.h +) + +find_library(JEMALLOC_LIBRARIES + NAMES jemalloc + HINTS ${JEMALLOC_ROOT_DIR}/lib +) + +find_path(JEMALLOC_INCLUDE_DIR + NAMES jemalloc/jemalloc.h + HINTS ${JEMALLOC_ROOT_DIR}/include +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(JeMalloc DEFAULT_MSG + JEMALLOC_LIBRARIES + JEMALLOC_INCLUDE_DIR +) + +mark_as_advanced( + JEMALLOC_ROOT_DIR + JEMALLOC_LIBRARIES + JEMALLOC_INCLUDE_DIR +) diff --git a/cpackEngineDEB.cmake b/cpackEngineDEB.cmake index 549e0d53a..4cae76f6a 100644 --- a/cpackEngineDEB.cmake +++ b/cpackEngineDEB.cmake @@ -65,11 +65,11 @@ if (EXISTS "/etc/debian_version") set(DEBIAN_VERSION_NUMBER "${CMAKE_MATCH_1}") endif () if ("${DEBIAN_VERSION_NUMBER}" EQUAL "8") - SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1") + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1, libjemalloc1") elseif ("${DEBIAN_VERSION_NUMBER}" EQUAL "9") - SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1v5, libreadline5") + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1v5, libreadline5, libjemalloc1") else() - SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libboost-all-dev, libreadline-dev, rsync, libsnappy1v5, net-tools") + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libboost-all-dev, libreadline-dev, rsync, libsnappy1v5, net-tools, libjemalloc1") endif () SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_DEPENDS "mariadb-columnstore-libs") diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake index c4fc765cc..83a707dbd 100644 --- a/cpackEngineRPM.cmake +++ b/cpackEngineRPM.cmake @@ -91,9 +91,9 @@ if (${REDHAT_VERSION_NUMBER} EQUAL 6) # Disable auto require as this will also try to pull Boost via RPM SET(CPACK_RPM_PACKAGE_AUTOREQPROV " no") elseif (${SUSE_VERSION_NUMBER} EQUAL 12) - SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost-devel >= 1.54.0" "mariadb-columnstore-libs" "libsnappy1") + SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost-devel >= 1.54.0" "mariadb-columnstore-libs" "libsnappy1" "jemalloc") else () - SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "mariadb-columnstore-libs" "snappy") + SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "mariadb-columnstore-libs" "snappy" "jemalloc") endif() SETA(CPACK_RPM_storage-engine_PACKAGE_REQUIRES "mariadb-columnstore-libs") diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index f74782d49..e359b9c35 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -344,8 +344,6 @@ void GroupConcatAgUM::merge(const rowgroup::Row& inRow, int64_t i) joblist::GroupConcatAgUM* gccAg = *((joblist::GroupConcatAgUM**)(data + inRow.getOffset(i))); fConcator->merge(gccAg->concator().get()); -// don't reset -// gccAg->orderBy().reset(); } @@ -393,12 +391,11 @@ GroupConcator::~GroupConcator() void GroupConcator::initialize(const rowgroup::SP_GroupConcat& gcc) { + // MCOL-901 This value comes from the Server and it is + // too high(3MB) to allocate it for every instance. fGroupConcatLen = gcc->fSize; fCurrentLength -= strlen(gcc->fSeparator.c_str()); - fOutputString.reset(new uint8_t[fGroupConcatLen + 2]); - memset(fOutputString.get(), 0, fGroupConcatLen + 2); - fConstCols = gcc->fConstCols; fConstantLen = strlen(gcc->fSeparator.c_str()); @@ -477,7 +474,6 @@ void GroupConcator::outputRow(std::ostringstream& oss, const rowgroup::Row& row) case CalpontSystemCatalog::TEXT: { oss << row.getStringField(*i).c_str(); - //oss << row.getStringField(*i); break; } @@ -626,11 +622,6 @@ int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row) { int64_t colWidth = row.getStringLength(*i); fieldLen += colWidth; // getStringLength() does the same thing as below - //assert(!row.usesStringTable()); - //int64_t colWidth = row.getColumnWidth(*i); - //uint8_t* pStr = row.getData() + row.getOffset(*i); - //while ((*pStr++ > 0) && (fieldLen < colWidth)) - // fieldLen++; break; } @@ -759,7 +750,6 @@ void GroupConcatOrderBy::initialize(const rowgroup::SP_GroupConcat& gcc) uint64_t GroupConcatOrderBy::getKeyLength() const { // only distinct the concatenated columns - //return (fRow0.getOffset(fConcatColumns.size()) - 2); return fConcatColumns.size() - 1; // cols 0 to fConcatColumns will be conpared } @@ -778,8 +768,6 @@ void GroupConcatOrderBy::processRow(const rowgroup::Row& row) if (fCurrentLength < fGroupConcatLen) { copyRow(row, &fRow0); - //cout << "length < GB limit: " << fRow0.toString() << endl; - //memcpy(fRow0.getData(), row.getData(), row.getSize()); // the RID is no meaning here, use it to store the estimated length. int16_t estLen = lengthEstimate(fRow0); fRow0.setRid(estLen); @@ -869,7 +857,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc) if (fDistinct) fDistinctMap->insert(row.fData); - //fDistinctMap->insert(make_pair((row.fData+2), row.fData)); } else if (fOrderByCond.size() > 0 && fRule.less(row.fData, fOrderByQueue.top().fData)) @@ -883,8 +870,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc) { fDistinctMap->erase(swapRow.fData); fDistinctMap->insert(row.fData); - //fDistinctMap->erase(fDistinctMap->find(swapRow.fData + 2)); - //fDistinctMap->insert(make_pair((row.fData+2), row.fData)); } row1.setData(row.fData); @@ -900,7 +885,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc) void GroupConcatOrderBy::getResult(uint8_t* buff, const string& sep) { -#if 1 ostringstream oss; bool addSep = false; @@ -926,28 +910,12 @@ void GroupConcatOrderBy::getResult(uint8_t* buff, const string& sep) rowStack.pop(); } - strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen); -#else - ostringstream oss; - bool addSep = false; - priority_queue::reverse_iterator rit; + size_t resultSize = oss.str().size(); + fOutputString.reset(new uint8_t[resultSize + 2]); + memset(fOutputString.get(), 0, resultSize + 2); - for (rit = fOrderByQueue.rbegin(); rit != fOrderByQueue.rend(); ++rit) - { - if (addSep) - oss << sep; - else - addSep = true; - - const OrderByRow& topRow = *rit; - fRow0.setData(topRow.fData); - outputRow(oss, fRow0); - } - - fOrderByQueue.clear(); - - strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen); -#endif + strncpy((char*)fOutputString.get(), + oss.str().c_str(), resultSize); } uint8_t* GroupConcator::getResult(const string& sep) @@ -1017,7 +985,6 @@ void GroupConcatNoOrder::initialize(const rowgroup::SP_GroupConcat& gcc) throw IDBExcept(fErrorCode); } - //fData.reset(new uint8_t[fRowGroup.getDataSize(fRowsPerRG)]); fData.reinit(fRowGroup, fRowsPerRG); fRowGroup.setData(&fData); fRowGroup.resetRowGroup(0); @@ -1032,7 +999,6 @@ void GroupConcatNoOrder::processRow(const rowgroup::Row& row) if (fCurrentLength < fGroupConcatLen && concatColIsNull(row) == false) { copyRow(row, &fRow); - //memcpy(fRow.getData(), row.getData(), row.getSize()); // the RID is no meaning here, use it to store the estimated length. int16_t estLen = lengthEstimate(fRow); @@ -1056,7 +1022,6 @@ void GroupConcatNoOrder::processRow(const rowgroup::Row& row) fDataQueue.push(fData); fData.reinit(fRowGroup, fRowsPerRG); - //fData.reset(new uint8_t[fRowGroup.getDataSize(fRowsPerRG)]); fRowGroup.setData(&fData); fRowGroup.resetRowGroup(0); fRowGroup.getRow(0, &fRow); @@ -1107,7 +1072,12 @@ void GroupConcatNoOrder::getResult(uint8_t* buff, const string& sep) fDataQueue.pop(); } - strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen); + size_t resultSize = oss.str().size(); + fOutputString.reset(new uint8_t[resultSize + 2]); + memset(fOutputString.get(), 0, resultSize + 2); + + strncpy((char*)fOutputString.get(), + oss.str().c_str(), resultSize); } diff --git a/dbcon/joblist/limitedorderby.cpp b/dbcon/joblist/limitedorderby.cpp index 64caf078c..de7f9bcb6 100644 --- a/dbcon/joblist/limitedorderby.cpp +++ b/dbcon/joblist/limitedorderby.cpp @@ -173,10 +173,10 @@ void LimitedOrderBy::finalize() if (fOrderByQueue.size() > 0) { - uint64_t newSize = fRowsPerRG * fRowGroup.getRowSize(); - fMemSize += newSize; + uint64_t memSizeInc = fRowsPerRG * fRowGroup.getRowSize(); + fMemSize += memSizeInc; - if (!fRm->getMemory(newSize, fSessionMemLimit)) + if (!fRm->getMemory(memSizeInc, fSessionMemLimit)) { cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) << " @" << __FILE__ << ":" << __LINE__; @@ -185,10 +185,12 @@ void LimitedOrderBy::finalize() uint64_t offset = 0; uint64_t i = 0; + // Reduce queue size by an offset value if it applicable. + uint64_t queueSizeWoOffset = fOrderByQueue.size() > fStart ? + fOrderByQueue.size() - fStart : 0; list tempRGDataList; - - // Skip first LIMIT rows in the the RowGroup - if ( fCount <= fOrderByQueue.size() ) + + if ( fCount <= queueSizeWoOffset ) { offset = fCount % fRowsPerRG; if(!offset && fCount > 0) @@ -196,11 +198,11 @@ void LimitedOrderBy::finalize() } else { - offset = fOrderByQueue.size() % fRowsPerRG; - if(!offset && fOrderByQueue.size() > 0) + offset = queueSizeWoOffset % fRowsPerRG; + if(!offset && queueSizeWoOffset > 0) offset = fRowsPerRG; } - + list::iterator tempListIter = tempRGDataList.begin(); i = 0; @@ -217,7 +219,6 @@ void LimitedOrderBy::finalize() const OrderByRow& topRow = fOrderByQueue.top(); row1.setData(topRow.fData); copyRow(row1, &fRow0); - //cerr << "LimitedOrderBy::finalize fRow0 " << fRow0.toString() << endl; fRowGroup.incRowCount(); offset--; fRow0.prevRow(rSize); @@ -226,9 +227,9 @@ void LimitedOrderBy::finalize() if(offset == (uint64_t)-1) { tempRGDataList.push_front(fData); - fMemSize += newSize; + fMemSize += memSizeInc; - if (!fRm->getMemory(newSize, fSessionMemLimit)) + if (!fRm->getMemory(memSizeInc, fSessionMemLimit)) { cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) << " @" << __FILE__ << ":" << __LINE__; diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 68355980a..362fb4b00 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -759,8 +759,11 @@ int processCommand(string* arguments) vector srcDbroots; // all of the currently configured dbroots vector destDbroots; // srcDbroots - removeDbroots set::iterator dbiter; - - if (arguments[1] == "start") +#if _MSC_VER + if (_strnicmp(arguments[1].c_str(), "start", 5) == 0)) +#else + if (strncasecmp(arguments[1].c_str(), "start", 5) == 0) +#endif { // Get a list of all the configured dbroots in the xml file. DBRootConfigList dbRootConfigList; @@ -772,7 +775,11 @@ int processCommand(string* arguments) // The user may choose to redistribute in such a way as to // leave certain dbroots empty, presumably for later removal. - if (arguments[2] == "remove") +#if _MSC_VER + if (_strnicmp(arguments[1].c_str(), "remove", 6) == 0)) +#else + if (strncasecmp(arguments[1].c_str(), "remove", 6) == 0) +#endif { int dbroot; bool error = false; @@ -891,7 +898,11 @@ int processCommand(string* arguments) SendToWES(oam, bs); } - else if (arguments[1] == "stop") +#if _MSC_VER + if (_strnicmp(arguments[1].c_str(), "stop", 4) == 0)) +#else + if (strncasecmp(arguments[1].c_str(), "stop", 4) == 0) +#endif { ByteStream bs; // message WES ID, sequence #, action id @@ -901,7 +912,11 @@ int processCommand(string* arguments) bs.append((const ByteStream::byte*) &header, sizeof(header)); SendToWES(oam, bs); } - else if (arguments[1] == "status") +#if _MSC_VER + if (_strnicmp(arguments[1].c_str(), "status", 6) == 0)) +#else + if (strncasecmp(arguments[1].c_str(), "status", 6) == 0) +#endif { ByteStream bs; // message WES ID, sequence #, action id diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index c92697127..fdfc1482c 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -55,7 +55,6 @@ #include #include #include -#include #include /* for strncpy */ #include @@ -111,8 +110,6 @@ typedef std::vector ModuleIpList; void offLineAppCheck(); bool setOSFiles(string parentOAMModuleName, int serverTypeInstall); bool checkSaveConfigFile(); -string getModuleName(); -bool setModuleName(string moduleName); bool updateBash(); bool makeModuleFile(string moduleName, string parentOAMModuleName); bool updateProcessConfig(); @@ -126,6 +123,7 @@ bool singleServerDBrootSetup(); bool copyFstab(string moduleName); bool attachVolume(string instanceName, string volumeName, string deviceName, string dbrootPath); void singleServerConfigSetup(Config* sysConfig); +std::string resolveHostNameToReverseDNSName(std::string hostname); void remoteInstallThread(void*); @@ -193,6 +191,7 @@ bool single_server_quick_install = false; bool multi_server_quick_install = false; bool amazon_quick_install = false; bool doNotResolveHostNames = false; +bool resolveHostNamesToReverseDNSNames = false; string DataFileEnvFile; @@ -316,7 +315,7 @@ int main(int argc, char* argv[]) cout << " Enter one of the options within [], if available, or" << endl; cout << " Enter a new value" << endl << endl; cout << endl; - cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-numBlocksPct][-totalUmMemory]" << endl; + cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-x][-xr][-numBlocksPct][-totalUmMemory]" << endl; cout << " -h Help" << endl; cout << " -c Config File to use to extract configuration data, default is Columnstore.xml.rpmsave" << endl; cout << " -u Upgrade, Install using the Config File from -c, default to Columnstore.xml.rpmsave" << endl; @@ -332,6 +331,7 @@ int main(int argc, char* argv[]) cout << " -pm-ip-addrs Performance Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; cout << " -um-ip-addrs User Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; cout << " -x Do not resolve IP Addresses from host names" << endl; + cout << " -xr Resolve host names into their reverse DNS host names. Only applied in combination with -x" << endl; cout << " -numBlocksPct amount of physical memory to utilize for disk block caching" << endl; cout << " (percentages of the total memory need to be stated without suffix, explcit values with suffixes M or G)" << endl; cout << " -totalUmMemory amount of physical memory to utilize for joins, intermediate results and set operations on the UM" << endl; @@ -341,6 +341,10 @@ int main(int argc, char* argv[]) else if (string("-x") == argv[i]) { doNotResolveHostNames = true; + } + else if (string("-xr") == argv[i]) + { + resolveHostNamesToReverseDNSNames = true; } else if( string("-qs") == argv[i] ) { @@ -517,7 +521,7 @@ int main(int argc, char* argv[]) else { cout << " ERROR: Invalid Argument = " << argv[i] << endl; - cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-numBlocksPct][-totalUmMemory]" << endl; + cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-x][-xr][-numBlocksPct][-totalUmMemory]" << endl; exit (1); } } @@ -658,9 +662,14 @@ int main(int argc, char* argv[]) if (moduleconfig.hostConfigList.size() > 0 ) { HostConfigList::iterator pt1 = moduleconfig.hostConfigList.begin(); - string PM1ipAdd = (*pt1).IPAddr; + + // MCOL-1607. The 'am I pm1?' check below requires an ipaddr. + string PM1ipAdd = oam.getIPAddress((*pt1).IPAddr.c_str()); + if (PM1ipAdd.empty()) + PM1ipAdd = (*pt1).IPAddr; // this is what it was doing before + //cout << PM1ipAdd << endl; - + if ( PM1ipAdd != "127.0.0.1" ) { if ( PM1ipAdd != "0.0.0.0") @@ -2566,7 +2575,12 @@ int main(int argc, char* argv[]) //get IP Address string IPAddress; if (doNotResolveHostNames) - IPAddress = newModuleHostName; + if (resolveHostNamesToReverseDNSNames) { + IPAddress = resolveHostNameToReverseDNSName(newModuleHostName); + } + else { + IPAddress = newModuleHostName; + } else IPAddress = oam.getIPAddress( newModuleHostName); @@ -2593,6 +2607,13 @@ int main(int argc, char* argv[]) callFree(pcommand); } + + if (!doNotResolveHostNames) + { + string ugh = oam.getIPAddress(newModuleIPAddr); + if (ugh.length() > 0) + newModuleIPAddr = ugh; + } if (newModuleIPAddr == "127.0.0.1" || newModuleIPAddr == "0.0.0.0" || newModuleIPAddr == "128.0.0.1") { @@ -6550,7 +6571,12 @@ bool glusterSetup(string password, bool doNotResolveHostNames) //get IP Address string IPAddress; if (doNotResolveHostNames) - IPAddress = moduleHostName; + if (resolveHostNamesToReverseDNSNames) { + IPAddress = resolveHostNameToReverseDNSName(moduleHostName); + } + else { + IPAddress = moduleHostName; + } else IPAddress = oam.getIPAddress( moduleHostName); @@ -6566,7 +6592,7 @@ bool glusterSetup(string password, bool doNotResolveHostNames) //prompt for IP address while (true) { - prompt = "Enter PM #" + oam.itoa(DataRedundancyConfigs[pm].pmID) + " IP Address of " + moduleHostName + " (" + moduleIPAddr + ") > "; + prompt = "Enter PM #" + oam.itoa(DataRedundancyConfigs[pm].pmID) + " IP Address or hostname of " + moduleHostName + " (" + moduleIPAddr + ") > "; pcommand = callReadline(prompt.c_str()); if (pcommand) @@ -6575,6 +6601,13 @@ bool glusterSetup(string password, bool doNotResolveHostNames) callFree(pcommand); } + + if (!doNotResolveHostNames) + { + string ugh = oam.getIPAddress(moduleIPAddr); + if (ugh.length() > 0) + moduleIPAddr = ugh; + } if (moduleIPAddr == "127.0.0.1" || moduleIPAddr == "0.0.0.0" || moduleIPAddr == "128.0.0.1") { @@ -7011,29 +7044,29 @@ bool glusterSetup(string password, bool doNotResolveHostNames) command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-uid " + oam.itoa(user) + " >> /tmp/glusterCommands.txt 2>&1";; status = system(command.c_str()); - if (WEXITSTATUS(status) != 0 ) - { - cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; - exit(1); - } +if (WEXITSTATUS(status) != 0) +{ + cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; + exit(1); +} - command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-gid " + oam.itoa(group) + " >> /tmp/glusterCommands.txt 2>&1";; - status = system(command.c_str()); +command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-gid " + oam.itoa(group) + " >> /tmp/glusterCommands.txt 2>&1";; +status = system(command.c_str()); - if (WEXITSTATUS(status) != 0 ) - { - cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; - exit(1); - } +if (WEXITSTATUS(status) != 0) +{ + cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; + exit(1); +} - command = "sudo gluster volume start dbroot" + oam.itoa(dbrootID) + " >> /tmp/glusterCommands.txt 2>&1"; - status = system(command.c_str()); +command = "sudo gluster volume start dbroot" + oam.itoa(dbrootID) + " >> /tmp/glusterCommands.txt 2>&1"; +status = system(command.c_str()); - if (WEXITSTATUS(status) != 0 ) - { - cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; - exit(1); - } +if (WEXITSTATUS(status) != 0) +{ + cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; + exit(1); +} } cout << "DONE" << endl; @@ -7047,70 +7080,89 @@ bool glusterSetup(string password, bool doNotResolveHostNames) void singleServerConfigSetup(Config* sysConfig) { - try - { - sysConfig->setConfig("ExeMgr1", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("ExeMgr1", "Module", "pm1"); - sysConfig->setConfig("ProcMgr", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("ProcMgr_Alarm", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("ProcStatusControl", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("pm1_ProcessMonitor", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("pm1_ServerMonitor", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("pm1_WriteEngineServer", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("DDLProc", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("DMLProc", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS1", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS2", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS3", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS4", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS5", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS6", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS7", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS8", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS9", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS10", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS11", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS12", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS13", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS14", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS15", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS16", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS17", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS18", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS19", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS20", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS21", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS22", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS23", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS24", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS25", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS26", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS27", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS28", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS29", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS30", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS31", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS32", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("SystemModuleConfig", "ModuleCount2", "0"); - sysConfig->setConfig("SystemModuleConfig", "ModuleIPAddr1-1-3", "127.0.0.1"); - sysConfig->setConfig("SystemModuleConfig", "ModuleHostName1-1-3", "localhost"); - sysConfig->setConfig("DBRM_Controller", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("DBRM_Worker1", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("DBRM_Worker1", "Module", "pm1"); - sysConfig->setConfig("DBBC", "NumBlocksPct", "50"); - sysConfig->setConfig("Installation", "InitialInstallFlag", "y"); - sysConfig->setConfig("Installation", "SingleServerInstall", "y"); - sysConfig->setConfig("HashJoin", "TotalUmMemory", "25%"); - } - catch (...) - { - cout << "ERROR: Problem setting for Single Server in the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } + try + { + sysConfig->setConfig("ExeMgr1", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("ExeMgr1", "Module", "pm1"); + sysConfig->setConfig("ProcMgr", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("ProcMgr_Alarm", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("ProcStatusControl", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("pm1_ProcessMonitor", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("pm1_ServerMonitor", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("pm1_WriteEngineServer", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("DDLProc", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("DMLProc", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS1", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS2", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS3", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS4", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS5", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS6", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS7", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS8", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS9", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS10", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS11", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS12", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS13", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS14", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS15", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS16", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS17", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS18", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS19", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS20", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS21", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS22", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS23", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS24", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS25", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS26", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS27", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS28", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS29", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS30", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS31", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS32", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("SystemModuleConfig", "ModuleCount2", "0"); + sysConfig->setConfig("SystemModuleConfig", "ModuleIPAddr1-1-3", "127.0.0.1"); + sysConfig->setConfig("SystemModuleConfig", "ModuleHostName1-1-3", "localhost"); + sysConfig->setConfig("DBRM_Controller", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("DBRM_Worker1", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("DBRM_Worker1", "Module", "pm1"); + sysConfig->setConfig("DBBC", "NumBlocksPct", "50"); + sysConfig->setConfig("Installation", "InitialInstallFlag", "y"); + sysConfig->setConfig("Installation", "SingleServerInstall", "y"); + sysConfig->setConfig("HashJoin", "TotalUmMemory", "25%"); + } + catch (...) + { + cout << "ERROR: Problem setting for Single Server in the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } - return; + return; } - +/** + Resolves the given hostname into its reverse DNS name. + + @param hostname the hostname to resolve. + @return the reverse dns name of given hostname or an empty string in case the hostname could not be resolved. +*/ +std::string resolveHostNameToReverseDNSName(std::string hostname) { + struct hostent *hp = gethostbyname(hostname.c_str()); + if (hp == NULL) { + std::cout << "Error: Couldn't resolve hostname " << hostname << " to ip address" << std::endl; + return ""; + } + struct hostent *rl = gethostbyaddr(hp->h_addr_list[0], sizeof hp->h_addr_list[0], AF_INET); + if (rl == NULL) { + std::cout << "Error: Couldn't resolve ip address of hostname " << hostname << " back to a hostname" << std::endl; + return ""; + } + hostname = rl->h_name; + return hostname; +} // vim:ts=4 sw=4: diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 5d5308ba7..4303dbc27 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -1207,12 +1207,13 @@ int64_t Row::getSignedNullValue(uint32_t colIndex) const RowGroup::RowGroup() : columnCount(0), data(NULL), rgData(NULL), strings(NULL), useStringTable(true), hasLongStringField(false), sTableThreshold(20) { - oldOffsets.reserve(1024); - oids.reserve(1024); - keys.reserve(1024); - types.reserve(1024); - scale.reserve(1024); - precision.reserve(1024); + // 1024 is too generous to waste. + oldOffsets.reserve(10); + oids.reserve(10); + keys.reserve(10); + types.reserve(10); + scale.reserve(10); + precision.reserve(10); } RowGroup::RowGroup(uint32_t colCount, diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index 53af4eb59..943a8688f 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -194,15 +194,20 @@ long long Convertor::convertDecimalString( // range check against int64 - if (dval > LLONG_MAX || dval < LLONG_MIN) + if (dval > LLONG_MAX) + { errno = ERANGE; + return LLONG_MAX; + } + if (dval < LLONG_MIN) + { + errno = ERANGE; + return LLONG_MIN; + } + errno = 0; - else - - errno = 0; - ret = dval; - + // get the fractional part of what's left & round ret up or down. dval -= ret; if (dval >= 0.5 && ret < LLONG_MAX)