diff --git a/CMakeLists.txt b/CMakeLists.txt old mode 100755 new mode 100644 index c0d708d6e..94fdde894 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,6 +112,17 @@ if (NOT SNAPPY_FOUND) MESSAGE(FATAL_ERROR "Snappy not found please install snappy-devel for CentOS/RedHat or libsnappy-dev for Ubuntu/Debian") endif() +# Jemalloc has issues with SLES 12, so disable for now +IF (EXISTS "/etc/SuSE-release") + SET(JEMALLOC_LIBRARIES "") +ELSE () + INCLUDE (FindJeMalloc.cmake) + if (NOT JEMALLOC_FOUND) + message(FATAL_ERROR "jemalloc not found!") + SET(JEMALLOC_LIBRARIES "") + endif() +ENDIF () + FIND_PROGRAM(AWK_EXECUTABLE awk DOC "path to the awk executable") if(NOT AWK_EXECUTABLE) message(FATAL_ERROR "awk not found!") @@ -163,7 +174,7 @@ SET (ENGINE_LOCALDIR "${INSTALL_ENGINE}/local") SET (ENGINE_MYSQLDIR "${INSTALL_ENGINE}/mysql") SET (ENGINE_TOOLSDIR "${INSTALL_ENGINE}/tools") -SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client) +SET (ENGINE_COMMON_LIBS messageqcpp loggingcpp configcpp idbboot ${Boost_LIBRARIES} xml2 pthread rt libmysql_client ${JEMALLOC_LIBRARIES}) SET (ENGINE_OAM_LIBS oamcpp alarmmanager) SET (ENGINE_BRM_LIBS brm idbdatafile cacheutils rwlock ${ENGINE_OAM_LIBS} ${ENGINE_COMMON_LIBS}) SET (ENGINE_EXEC_LIBS joblist execplan windowfunction joiner rowgroup funcexp udfsdk regr dataconvert common compress querystats querytele thrift threadpool ${ENGINE_BRM_LIBS}) @@ -172,13 +183,17 @@ SET (ENGINE_WRITE_LIBS ddlpackageproc ddlpackage dmlpackageproc dmlpackage SET (ENGINE_COMMON_LDFLAGS "") IF (SERVER_BUILD_INCLUDE_DIR) + IF (NOT IS_ABSOLUTE ${SERVER_BUILD_INCLUDE_DIR}) SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_BINARY_DIR}/${SERVER_BUILD_INCLUDE_DIR}) + ENDIF() ELSE() SET (SERVER_BUILD_INCLUDE_DIR ${CMAKE_BINARY_DIR}/../include) ENDIF() IF (SERVER_SOURCE_ROOT_DIR) + IF (NOT IS_ABSOLUTE ${SERVER_SOURCE_ROOT_DIR}) SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_BINARY_DIR}/${SERVER_SOURCE_ROOT_DIR}) + ENDIF() ELSE() SET (SERVER_SOURCE_ROOT_DIR ${CMAKE_BINARY_DIR}/..) ENDIF() diff --git a/FindJeMalloc.cmake b/FindJeMalloc.cmake new file mode 100644 index 000000000..c97bb1b9f --- /dev/null +++ b/FindJeMalloc.cmake @@ -0,0 +1,44 @@ +# - Try to find jemalloc headers and libraries. +# +# Usage of this module as follows: +# +# find_package(JeMalloc) +# +# Variables used by this module, they can change the default behaviour and need +# to be set before calling find_package: +# +# JEMALLOC_ROOT_DIR Set this variable to the root installation of +# jemalloc if the module has problems finding +# the proper installation path. +# +# Variables defined by this module: +# +# JEMALLOC_FOUND System has jemalloc libs/headers +# JEMALLOC_LIBRARIES The jemalloc library/libraries +# JEMALLOC_INCLUDE_DIR The location of jemalloc headers + +find_path(JEMALLOC_ROOT_DIR + NAMES include/jemalloc/jemalloc.h +) + +find_library(JEMALLOC_LIBRARIES + NAMES jemalloc + HINTS ${JEMALLOC_ROOT_DIR}/lib +) + +find_path(JEMALLOC_INCLUDE_DIR + NAMES jemalloc/jemalloc.h + HINTS ${JEMALLOC_ROOT_DIR}/include +) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(JeMalloc DEFAULT_MSG + JEMALLOC_LIBRARIES + JEMALLOC_INCLUDE_DIR +) + +mark_as_advanced( + JEMALLOC_ROOT_DIR + JEMALLOC_LIBRARIES + JEMALLOC_INCLUDE_DIR +) diff --git a/VERSION b/VERSION index f5d54e457..ebe283cd3 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 -COLUMNSTORE_VERSION_MINOR=3 -COLUMNSTORE_VERSION_PATCH=0 +COLUMNSTORE_VERSION_MINOR=2 +COLUMNSTORE_VERSION_PATCH=3 COLUMNSTORE_VERSION_RELEASE=1 diff --git a/cpackEngineDEB.cmake b/cpackEngineDEB.cmake index 549e0d53a..4cae76f6a 100644 --- a/cpackEngineDEB.cmake +++ b/cpackEngineDEB.cmake @@ -65,11 +65,11 @@ if (EXISTS "/etc/debian_version") set(DEBIAN_VERSION_NUMBER "${CMAKE_MATCH_1}") endif () if ("${DEBIAN_VERSION_NUMBER}" EQUAL "8") - SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1") + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1, libjemalloc1") elseif ("${DEBIAN_VERSION_NUMBER}" EQUAL "9") - SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1v5, libreadline5") + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libreadline-dev, rsync, net-tools, libboost-all-dev, mariadb-columnstore-libs, mariadb-columnstore-server, libsnappy1v5, libreadline5, libjemalloc1") else() - SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libboost-all-dev, libreadline-dev, rsync, libsnappy1v5, net-tools") + SET(CPACK_DEBIAN_PLATFORM_PACKAGE_DEPENDS "expect, perl, openssl, file, libdbi-perl, libboost-all-dev, libreadline-dev, rsync, libsnappy1v5, net-tools, libjemalloc1") endif () SET(CPACK_DEBIAN_STORAGE-ENGINE_PACKAGE_DEPENDS "mariadb-columnstore-libs") diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake index c4fc765cc..83a707dbd 100644 --- a/cpackEngineRPM.cmake +++ b/cpackEngineRPM.cmake @@ -91,9 +91,9 @@ if (${REDHAT_VERSION_NUMBER} EQUAL 6) # Disable auto require as this will also try to pull Boost via RPM SET(CPACK_RPM_PACKAGE_AUTOREQPROV " no") elseif (${SUSE_VERSION_NUMBER} EQUAL 12) - SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost-devel >= 1.54.0" "mariadb-columnstore-libs" "libsnappy1") + SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost-devel >= 1.54.0" "mariadb-columnstore-libs" "libsnappy1" "jemalloc") else () - SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "mariadb-columnstore-libs" "snappy") + SETA(CPACK_RPM_platform_PACKAGE_REQUIRES "expect" "boost >= 1.53.0" "mariadb-columnstore-libs" "snappy" "jemalloc") endif() SETA(CPACK_RPM_storage-engine_PACKAGE_REQUIRES "mariadb-columnstore-libs") diff --git a/dbcon/joblist/groupconcat.cpp b/dbcon/joblist/groupconcat.cpp index afc91a2ec..11429d053 100644 --- a/dbcon/joblist/groupconcat.cpp +++ b/dbcon/joblist/groupconcat.cpp @@ -344,8 +344,6 @@ void GroupConcatAgUM::merge(const rowgroup::Row& inRow, int64_t i) joblist::GroupConcatAgUM* gccAg = *((joblist::GroupConcatAgUM**)(data + inRow.getOffset(i))); fConcator->merge(gccAg->concator().get()); -// don't reset -// gccAg->orderBy().reset(); } @@ -393,12 +391,11 @@ GroupConcator::~GroupConcator() void GroupConcator::initialize(const rowgroup::SP_GroupConcat& gcc) { + // MCOL-901 This value comes from the Server and it is + // too high(3MB) to allocate it for every instance. fGroupConcatLen = gcc->fSize; fCurrentLength -= strlen(gcc->fSeparator.c_str()); - fOutputString.reset(new uint8_t[fGroupConcatLen + 2]); - memset(fOutputString.get(), 0, fGroupConcatLen + 2); - fConstCols = gcc->fConstCols; fConstantLen = strlen(gcc->fSeparator.c_str()); @@ -477,7 +474,6 @@ void GroupConcator::outputRow(std::ostringstream& oss, const rowgroup::Row& row) case CalpontSystemCatalog::TEXT: { oss << row.getStringField(*i).c_str(); - //oss << row.getStringField(*i); break; } @@ -620,11 +616,6 @@ int64_t GroupConcator::lengthEstimate(const rowgroup::Row& row) { int64_t colWidth = row.getStringLength(*i); fieldLen += colWidth; // getStringLength() does the same thing as below - //assert(!row.usesStringTable()); - //int64_t colWidth = row.getColumnWidth(*i); - //uint8_t* pStr = row.getData() + row.getOffset(*i); - //while ((*pStr++ > 0) && (fieldLen < colWidth)) - // fieldLen++; break; } @@ -752,7 +743,6 @@ void GroupConcatOrderBy::initialize(const rowgroup::SP_GroupConcat& gcc) uint64_t GroupConcatOrderBy::getKeyLength() const { // only distinct the concatenated columns - //return (fRow0.getOffset(fConcatColumns.size()) - 2); return fConcatColumns.size() - 1; // cols 0 to fConcatColumns will be conpared } @@ -771,8 +761,6 @@ void GroupConcatOrderBy::processRow(const rowgroup::Row& row) if (fCurrentLength < fGroupConcatLen) { copyRow(row, &fRow0); - //cout << "length < GB limit: " << fRow0.toString() << endl; - //memcpy(fRow0.getData(), row.getData(), row.getSize()); // the RID is no meaning here, use it to store the estimated length. int16_t estLen = lengthEstimate(fRow0); fRow0.setRid(estLen); @@ -862,7 +850,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc) if (fDistinct) fDistinctMap->insert(row.fData); - //fDistinctMap->insert(make_pair((row.fData+2), row.fData)); } else if (fOrderByCond.size() > 0 && fRule.less(row.fData, fOrderByQueue.top().fData)) @@ -876,8 +863,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc) { fDistinctMap->erase(swapRow.fData); fDistinctMap->insert(row.fData); - //fDistinctMap->erase(fDistinctMap->find(swapRow.fData + 2)); - //fDistinctMap->insert(make_pair((row.fData+2), row.fData)); } row1.setData(row.fData); @@ -893,7 +878,6 @@ void GroupConcatOrderBy::merge(GroupConcator* gc) void GroupConcatOrderBy::getResult(uint8_t* buff, const string& sep) { -#if 1 ostringstream oss; bool addSep = false; @@ -919,28 +903,12 @@ void GroupConcatOrderBy::getResult(uint8_t* buff, const string& sep) rowStack.pop(); } - strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen); -#else - ostringstream oss; - bool addSep = false; - priority_queue::reverse_iterator rit; + size_t resultSize = oss.str().size(); + fOutputString.reset(new uint8_t[resultSize + 2]); + memset(fOutputString.get(), 0, resultSize + 2); - for (rit = fOrderByQueue.rbegin(); rit != fOrderByQueue.rend(); ++rit) - { - if (addSep) - oss << sep; - else - addSep = true; - - const OrderByRow& topRow = *rit; - fRow0.setData(topRow.fData); - outputRow(oss, fRow0); - } - - fOrderByQueue.clear(); - - strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen); -#endif + strncpy((char*)fOutputString.get(), + oss.str().c_str(), resultSize); } uint8_t* GroupConcator::getResult(const string& sep) @@ -1010,7 +978,6 @@ void GroupConcatNoOrder::initialize(const rowgroup::SP_GroupConcat& gcc) throw IDBExcept(fErrorCode); } - //fData.reset(new uint8_t[fRowGroup.getDataSize(fRowsPerRG)]); fData.reinit(fRowGroup, fRowsPerRG); fRowGroup.setData(&fData); fRowGroup.resetRowGroup(0); @@ -1025,7 +992,6 @@ void GroupConcatNoOrder::processRow(const rowgroup::Row& row) if (fCurrentLength < fGroupConcatLen && concatColIsNull(row) == false) { copyRow(row, &fRow); - //memcpy(fRow.getData(), row.getData(), row.getSize()); // the RID is no meaning here, use it to store the estimated length. int16_t estLen = lengthEstimate(fRow); @@ -1049,7 +1015,6 @@ void GroupConcatNoOrder::processRow(const rowgroup::Row& row) fDataQueue.push(fData); fData.reinit(fRowGroup, fRowsPerRG); - //fData.reset(new uint8_t[fRowGroup.getDataSize(fRowsPerRG)]); fRowGroup.setData(&fData); fRowGroup.resetRowGroup(0); fRowGroup.getRow(0, &fRow); @@ -1100,7 +1065,12 @@ void GroupConcatNoOrder::getResult(uint8_t* buff, const string& sep) fDataQueue.pop(); } - strncpy((char*) buff, oss.str().c_str(), fGroupConcatLen); + size_t resultSize = oss.str().size(); + fOutputString.reset(new uint8_t[resultSize + 2]); + memset(fOutputString.get(), 0, resultSize + 2); + + strncpy((char*)fOutputString.get(), + oss.str().c_str(), resultSize); } diff --git a/dbcon/joblist/joblist.cpp b/dbcon/joblist/joblist.cpp index 023b21bb6..bb71996c1 100644 --- a/dbcon/joblist/joblist.cpp +++ b/dbcon/joblist/joblist.cpp @@ -1156,11 +1156,7 @@ void JobList::abort() for (i = 0; i < fProject.size(); i++) fProject[i]->abort(); - for (i = 0; i < fQuery.size(); i++) - fQuery[i]->join(); - for (i = 0; i < fProject.size(); i++) - fProject[i]->join(); } } diff --git a/dbcon/joblist/limitedorderby.cpp b/dbcon/joblist/limitedorderby.cpp index 7da1accfe..de7f9bcb6 100644 --- a/dbcon/joblist/limitedorderby.cpp +++ b/dbcon/joblist/limitedorderby.cpp @@ -113,7 +113,6 @@ void LimitedOrderBy::processRow(const rowgroup::Row& row) if (fOrderByQueue.size() < fStart + fCount) { copyRow(row, &fRow0); - //memcpy(fRow0.getData(), row.getData(), row.getSize()); OrderByRow newRow(fRow0, fRule); fOrderByQueue.push(newRow); @@ -121,8 +120,6 @@ void LimitedOrderBy::processRow(const rowgroup::Row& row) if (fDistinct) fDistinctMap->insert(fRow0.getPointer()); - //fDistinctMap->insert(make_pair((fRow0.getData()+2), fRow0.getData())); - fRowGroup.incRowCount(); fRow0.nextRow(); @@ -150,23 +147,15 @@ void LimitedOrderBy::processRow(const rowgroup::Row& row) { OrderByRow swapRow = fOrderByQueue.top(); row1.setData(swapRow.fData); - fOrderByQueue.pop(); + copyRow(row, &row1); - if (!fDistinct) + if (fDistinct) { - copyRow(row, &row1); - //memcpy(swapRow.fData, row.getData(), row.getSize()); - } - else - { - fDistinctMap->erase(row.getPointer()); - copyRow(row, &row1); + fDistinctMap->erase(fOrderByQueue.top().fData); fDistinctMap->insert(row1.getPointer()); - //fDistinctMap->erase(fDistinctMap->find(row.getData() + 2)); - //memcpy(swapRow.fData, row.getData(), row.getSize()); - //fDistinctMap->insert(make_pair((swapRow.fData+2), swapRow.fData)); } + fOrderByQueue.pop(); fOrderByQueue.push(swapRow); } } @@ -184,10 +173,10 @@ void LimitedOrderBy::finalize() if (fOrderByQueue.size() > 0) { - uint64_t newSize = fRowsPerRG * fRowGroup.getRowSize(); - fMemSize += newSize; + uint64_t memSizeInc = fRowsPerRG * fRowGroup.getRowSize(); + fMemSize += memSizeInc; - if (!fRm->getMemory(newSize, fSessionMemLimit)) + if (!fRm->getMemory(memSizeInc, fSessionMemLimit)) { cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) << " @" << __FILE__ << ":" << __LINE__; @@ -196,10 +185,12 @@ void LimitedOrderBy::finalize() uint64_t offset = 0; uint64_t i = 0; + // Reduce queue size by an offset value if it applicable. + uint64_t queueSizeWoOffset = fOrderByQueue.size() > fStart ? + fOrderByQueue.size() - fStart : 0; list tempRGDataList; - - // Skip first LIMIT rows in the the RowGroup - if ( fCount <= fOrderByQueue.size() ) + + if ( fCount <= queueSizeWoOffset ) { offset = fCount % fRowsPerRG; if(!offset && fCount > 0) @@ -207,11 +198,11 @@ void LimitedOrderBy::finalize() } else { - offset = fOrderByQueue.size() % fRowsPerRG; - if(!offset && fOrderByQueue.size() > 0) + offset = queueSizeWoOffset % fRowsPerRG; + if(!offset && queueSizeWoOffset > 0) offset = fRowsPerRG; } - + list::iterator tempListIter = tempRGDataList.begin(); i = 0; @@ -236,9 +227,9 @@ void LimitedOrderBy::finalize() if(offset == (uint64_t)-1) { tempRGDataList.push_front(fData); - fMemSize += newSize; + fMemSize += memSizeInc; - if (!fRm->getMemory(newSize, fSessionMemLimit)) + if (!fRm->getMemory(memSizeInc, fSessionMemLimit)) { cerr << IDBErrorInfo::instance()->errorMsg(fErrorCode) << " @" << __FILE__ << ":" << __LINE__; diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index 9ddf42cc9..4c1893115 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -5554,6 +5554,8 @@ void TupleAggregateStep::threadedAggregateRowGroups(uint32_t threadID) rowIn.copyField(distRow[j], k, multiDist->subAggregators()[j]->getGroupByCols()[k].get()->fInputColumnIndex); } + // TBD This approach could potentiall + // put all values in on bucket. bucketID = distRow[j].hash(hashLens[j] - 1) % fNumOfBuckets; rowBucketVecs[bucketID][j].push_back(rowIn.getPointer()); rowIn.nextRow(); @@ -5572,6 +5574,8 @@ void TupleAggregateStep::threadedAggregateRowGroups(uint32_t threadID) for (uint64_t i = 0; i < fRowGroupIns[threadID].getRowCount(); ++i) { // The key is the groupby columns, which are the leading columns. + // TBD This approach could potentiall + // put all values in on bucket. int bucketID = rowIn.hash(hashLens[0] - 1) % fNumOfBuckets; rowBucketVecs[bucketID][0].push_back(rowIn.getPointer()); rowIn.nextRow(); diff --git a/dbcon/mysql/is_columnstore_files.cpp b/dbcon/mysql/is_columnstore_files.cpp index be0411058..2ce7b1996 100644 --- a/dbcon/mysql/is_columnstore_files.cpp +++ b/dbcon/mysql/is_columnstore_files.cpp @@ -114,7 +114,7 @@ static int generate_result(BRM::OID_t oid, BRM::DBRM* emp, TABLE* table, THD* th if (iter->blockOffset > 0) { iter++; - return 0; + continue; } try diff --git a/oam/etc/ConsoleCmds.xml b/oam/etc/ConsoleCmds.xml index 56a6e678f..17cf0581d 100644 --- a/oam/etc/ConsoleCmds.xml +++ b/oam/etc/ConsoleCmds.xml @@ -309,8 +309,10 @@ MariaDB Columnstore Packages and setup the module to make it ready to be restored Required: Module-type or Module-name being added Required: Number-of-Modules being added when Module-type is specified - Optional: Server-Hostnames/Amazon-Instance-Names seperated by commas - Optional: Server-root-password + Required: Store hostnames in the configuration instead of IP addresses (y/n) + If not Amazon, then Required: hostnames separated by commas + If Amazon, then Optional: Amazon-Instance-Names separated by commas + Optional: Server-user-password removeModule diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 539b48bbd..805c7d5d0 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -1272,10 +1272,12 @@ void Oam::setSystemConfig(const std::string module, ModuleConfig moduleconfig) * ********************************************************************/ -void Oam::addModule(DeviceNetworkList devicenetworklist, const std::string password, const std::string mysqlpw) +void Oam::addModule(DeviceNetworkList devicenetworklist, const std::string password, const std::string mysqlpw, + bool storeHostnames) { // build and send msg - int returnStatus = sendMsgToProcMgr2(ADDMODULE, devicenetworklist, FORCEFUL, ACK_YES, password, mysqlpw); + int returnStatus = sendAddModuleToProcMgr(ADDMODULE, devicenetworklist, FORCEFUL, ACK_YES, storeHostnames, + password, mysqlpw); if (returnStatus != API_SUCCESS) exceptionControl("addModule", returnStatus); @@ -10007,6 +10009,124 @@ int Oam::sendMsgToProcMgr2(messageqcpp::ByteStream::byte requestType, DeviceNetw return returnStatus; } +/* A slightly different version of sendMsgToProcMgr2. Add-module needs to send one add'l + parameter, and this was the best of a couple bad options. */ +int Oam::sendAddModuleToProcMgr(messageqcpp::ByteStream::byte requestType, DeviceNetworkList devicenetworklist, + GRACEFUL_FLAG gracefulflag, ACK_FLAG ackflag, bool storeHostnames, const std::string password, + const std::string mysqlpw) +{ + if (!checkSystemRunning()) + return API_CONN_REFUSED; + + int returnStatus = API_TIMEOUT; //default + ByteStream msg; + ByteStream receivedMSG; + ByteStream::byte msgType; + ByteStream::byte actionType; + ByteStream::byte status; + + // get current requesting process, an error will occur if process is a UI tool (not kept in Status Table) + // this will be used to determine if this is a manually or auto request down within Process-Monitor + bool requestManual; + myProcessStatus_t t; + + try + { + t = getMyProcessStatus(); + requestManual = false; // set to auto + } + catch (...) + { + requestManual = true; // set to manual + } + + // setup message + msg << (ByteStream::byte) REQUEST; + msg << requestType; + msg << (std::string) " "; + msg << (ByteStream::byte) gracefulflag; + msg << (ByteStream::byte) ackflag; + msg << (ByteStream::byte) requestManual; + msg << (uint8_t) storeHostnames; + msg << (uint16_t) devicenetworklist.size(); + + DeviceNetworkList::iterator pt = devicenetworklist.begin(); + + for ( ; pt != devicenetworklist.end() ; pt++) + { + msg << (*pt).DeviceName; + + if ( (*pt).UserTempDeviceName.empty() ) + msg << " "; + else + msg << (*pt).UserTempDeviceName; + + if ( (*pt).DisableState.empty() ) + msg << " "; + else + msg << (*pt).DisableState; + + msg << (uint16_t) (*pt).hostConfigList.size(); + + HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); + + for ( ; pt1 != (*pt).hostConfigList.end() ; pt1++) + { + msg << (*pt1).IPAddr; + msg << (*pt1).HostName; + msg << (*pt1).NicID; + } + } + + msg << password; + msg << mysqlpw; + + try + { + //send the msg to Process Manager + MessageQueueClient procmgr("ProcMgr"); + procmgr.write(msg); + + // check for Ack msg if needed + if ( ackflag == ACK_YES ) + { + // wait 15 minutes for ACK from Process Manager + struct timespec ts = { 900, 0 }; + + receivedMSG = procmgr.read(&ts); + + if (receivedMSG.length() > 0) + { + receivedMSG >> msgType; + receivedMSG >> actionType; + receivedMSG >> status; + + if ( msgType == oam::ACK && actionType == requestType) + { + // ACK for this request + returnStatus = status; + } + } + else // timeout + returnStatus = API_TIMEOUT; + } + else + // No ACK, assume write success + returnStatus = API_SUCCESS; + + // shutdown connection + procmgr.shutdown(); + } + catch (...) + { + returnStatus = API_FAILURE; + } + + return returnStatus; +} + + + /*************************************************************************** * * Function: sendMsgToProcMgr3 diff --git a/oam/oamcpp/liboamcpp.h b/oam/oamcpp/liboamcpp.h index 39e2a51de..5d5a09c6c 100644 --- a/oam/oamcpp/liboamcpp.h +++ b/oam/oamcpp/liboamcpp.h @@ -1488,7 +1488,8 @@ public: * @param DeviceNetworkConfig the Modules added * @param password Host Root Password */ - EXPORT void addModule(DeviceNetworkList devicenetworklist, const std::string password, const std::string mysqlpw); + EXPORT void addModule(DeviceNetworkList devicenetworklist, const std::string password, const std::string mysqlpw, + bool storeHostnames); /** @brief remove Module * @@ -2503,6 +2504,12 @@ private: int sendMsgToProcMgr2(messageqcpp::ByteStream::byte requestType, DeviceNetworkList devicenetworklist, GRACEFUL_FLAG gracefulflag, ACK_FLAG ackflag, const std::string password = oam::UnassignedName, const std::string mysqlpw = oam::UnassignedName); + /** @brief a slightly different version of sendMsgToProcMgr2, which is for addmodule only. + */ + int sendAddModuleToProcMgr(messageqcpp::ByteStream::byte requestType, DeviceNetworkList devicenetworklist, + GRACEFUL_FLAG gracefulflag, ACK_FLAG ackflag, bool storeHostnames, const std::string password = oam::UnassignedName, + const std::string mysqlpw = oam::UnassignedName); + /** @brief build and send request message to Process Manager * Check for status messages */ diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index cf28b15bb..362fb4b00 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -759,8 +759,11 @@ int processCommand(string* arguments) vector srcDbroots; // all of the currently configured dbroots vector destDbroots; // srcDbroots - removeDbroots set::iterator dbiter; - - if (arguments[1] == "start") +#if _MSC_VER + if (_strnicmp(arguments[1].c_str(), "start", 5) == 0)) +#else + if (strncasecmp(arguments[1].c_str(), "start", 5) == 0) +#endif { // Get a list of all the configured dbroots in the xml file. DBRootConfigList dbRootConfigList; @@ -772,7 +775,11 @@ int processCommand(string* arguments) // The user may choose to redistribute in such a way as to // leave certain dbroots empty, presumably for later removal. - if (arguments[2] == "remove") +#if _MSC_VER + if (_strnicmp(arguments[1].c_str(), "remove", 6) == 0)) +#else + if (strncasecmp(arguments[1].c_str(), "remove", 6) == 0) +#endif { int dbroot; bool error = false; @@ -891,7 +898,11 @@ int processCommand(string* arguments) SendToWES(oam, bs); } - else if (arguments[1] == "stop") +#if _MSC_VER + if (_strnicmp(arguments[1].c_str(), "stop", 4) == 0)) +#else + if (strncasecmp(arguments[1].c_str(), "stop", 4) == 0) +#endif { ByteStream bs; // message WES ID, sequence #, action id @@ -901,7 +912,11 @@ int processCommand(string* arguments) bs.append((const ByteStream::byte*) &header, sizeof(header)); SendToWES(oam, bs); } - else if (arguments[1] == "status") +#if _MSC_VER + if (_strnicmp(arguments[1].c_str(), "status", 6) == 0)) +#else + if (strncasecmp(arguments[1].c_str(), "status", 6) == 0) +#endif { ByteStream bs; // message WES ID, sequence #, action id @@ -5660,6 +5675,7 @@ int processCommand(string* arguments) DeviceNetworkList enabledevicenetworklist; HostConfig hostconfig; + bool storeHostnames = false; string moduleType; string moduleName; int moduleCount; @@ -5670,7 +5686,7 @@ int processCommand(string* arguments) umStorageNames umstoragenames; int hostArg; int dbrootPerPM = 0; - + //check if module type or module name was entered if ( arguments[1].size() == 2 ) { @@ -5685,8 +5701,47 @@ int processCommand(string* arguments) //Module Type was entered moduleType = arguments[1]; moduleCount = atoi(arguments[2].c_str()); + hostArg = 4; + + // MCOL-1607. Check whether we should store host names or IP addresses. + if (arguments[3] != "" && (arguments[3][0] == 'y' || arguments[3][0] == 'Y')) + storeHostnames = true; + + //check for a non-distrubuted install setup, dont need password + if ( DistributedInstall != "y" ) + { + if (arguments[5] != "") + password = arguments[5]; + else + { + cout << endl; + string prompt = "Enter the 'User' Password or 'ssh' if configured with ssh-keys"; + password = dataPrompt(prompt); + } + } + + if (arguments[6] != "") + dbrootPerPM = atoi(arguments[6].c_str()); + } + else + { + //Module Name was entered + if (arguments[2] == "" && cloud == oam::UnassignedName) + { + // need at least arguments + cout << endl << "**** addModule Failed : Missing a required Parameter, enter 'help' for additional information" << endl; + break; + } + + moduleName = arguments[1]; + moduleType = arguments[1].substr(0, MAX_MODULE_TYPE_SIZE); + moduleCount = 1; hostArg = 3; + // MCOL-1607. Check whether we should store host names or IP addresses. + if (arguments[2] != "" && (arguments[2][0] == 'y' || arguments[2][0] == 'Y')) + storeHostnames = true; + //check for a non-distrubuted install setup, dont need password if ( DistributedInstall != "y" ) { @@ -5703,37 +5758,6 @@ int processCommand(string* arguments) if (arguments[5] != "") dbrootPerPM = atoi(arguments[5].c_str()); } - else - { - //Module Name was entered - if (arguments[2] == "" && cloud == oam::UnassignedName) - { - // need at least arguments - cout << endl << "**** addModule Failed : Missing a required Parameter, enter 'help' for additional information" << endl; - break; - } - - moduleName = arguments[1]; - moduleType = arguments[1].substr(0, MAX_MODULE_TYPE_SIZE); - moduleCount = 1; - hostArg = 2; - - //check for a non-distrubuted install setup, dont need password - if ( DistributedInstall != "y" ) - { - if (arguments[3] != "") - password = arguments[3]; - else - { - cout << endl; - string prompt = "Enter the 'User' Password or 'ssh' if configured with ssh-keys"; - password = dataPrompt(prompt); - } - } - - if (arguments[4] != "") - dbrootPerPM = atoi(arguments[4].c_str()); - } //do we needed this check???? if ( moduleCount < 1 || moduleCount > 10 ) @@ -5925,7 +5949,24 @@ int processCommand(string* arguments) string hostName; string IPAddress; - if ( cloud == "amazon-ec2") + // MCOL-1607. Store hostnames in the config file if they entered one */ + if (storeHostnames) + { + // special case + if (cloud == "amazon-vpc" && *listPT1 == "autoassign") + { + hostName = oam::UnassignedName; + IPAddress = *listPT1; + } + else if (oam.isValidIP(*listPT1)) // they entered an IP addr + { + hostName = oam::UnassignedName; + IPAddress = *listPT1; + } + else // they entered a hostname + IPAddress = hostName = *listPT1; + } + else if ( cloud == "amazon-ec2") { hostName = *listPT1; @@ -5978,14 +6019,13 @@ int processCommand(string* arguments) // non-amazon hostName = *listPT1; IPAddress = oam.getIPAddress(hostName); - if ( IPAddress.empty() ) { // prompt for IP Address string prompt = "IP Address of " + hostName + " not found, enter IP Address or enter 'abort'"; IPAddress = dataPrompt(prompt); - - if ( IPAddress == "abort" || !oam.isValidIP(IPAddress)) + + if ( IPAddress == "abort" || !oam.isValidIP(IPAddress) ) return 1; } } @@ -6023,18 +6063,22 @@ int processCommand(string* arguments) { string prompt = "DataRedundancy is configured for dedicated network, enter a hostname"; DataRedundancyHostname = dataPrompt(prompt); - DataRedundancyIPAddress = oam.getIPAddress(DataRedundancyHostname); - - if ( DataRedundancyIPAddress.empty() ) + if (storeHostnames) + DataRedundancyIPAddress = DataRedundancyHostname; + else { - // prompt for IP Address - string prompt = "IP Address of " + DataRedundancyHostname + " not found, enter IP Address"; - DataRedundancyIPAddress = dataPrompt(prompt); + DataRedundancyIPAddress = oam.getIPAddress(DataRedundancyHostname); - if (!oam.isValidIP(DataRedundancyIPAddress)) - return 1; + if ( DataRedundancyIPAddress.empty() ) + { + // prompt for IP Address + string prompt = "IP Address of " + DataRedundancyHostname + " not found, enter IP Address"; + DataRedundancyIPAddress = dataPrompt(prompt); + + if (!oam.isValidIP(DataRedundancyIPAddress)) + return 1; + } } - sysConfig->setConfig("DataRedundancyConfig", dataDupHostName, DataRedundancyHostname); sysConfig->setConfig("DataRedundancyConfig", dataDupIPaddr, DataRedundancyIPAddress); } @@ -6105,7 +6149,7 @@ int processCommand(string* arguments) cout << "please wait..." << endl; - oam.addModule(devicenetworklist, password, mysqlpassword); + oam.addModule(devicenetworklist, password, mysqlpassword, storeHostnames); cout << "Add Module(s) successfully completed" << endl; diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 94850deae..81b69e548 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -126,10 +126,22 @@ int main(int argc, char* argv[]) mysqlpw = argv[10]; installer_debug = argv[11]; - if (argc >= 13) + string numBlocksPctParam = ""; + string totalUmMemoryParam = ""; + if (argc >= 13) { installDir = argv[12]; - else + if (argc >= 15) { + if (string(argv[13]) != "-") { + numBlocksPctParam = argv[13]; + } + if (string(argv[14]) != "-") { + totalUmMemoryParam = argv[14]; + } + } + } + else { installDir = "/usr/local/mariadb/columnstore"; + } ofstream file("/dev/null"); @@ -151,6 +163,12 @@ int main(int argc, char* argv[]) cout << mysqlpw << endl; cout << installer_debug << endl; cout << installDir << endl; + if (!numBlocksPctParam.empty()) { + cout << numBlocksPctParam << endl; + } + if (!totalUmMemoryParam.empty()) { + cout << totalUmMemoryParam << endl; + } // restore cout stream buffer cout.rdbuf (strm_buffer); @@ -255,16 +273,26 @@ int main(int argc, char* argv[]) return false; } - string numBlocksPct = "50"; - - if ( DBRootStorageType == "hdfs") - numBlocksPct = "25"; + string numBlocksPct; + if (numBlocksPctParam.empty()) { + numBlocksPct = "50"; + if (DBRootStorageType == "hdfs") + numBlocksPct = "25"; + } + else { + numBlocksPct = numBlocksPctParam; + } try { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + } } catch (...) { @@ -282,15 +310,21 @@ int main(int argc, char* argv[]) // long long total = myinfo.totalram / 1024 / 1000; // adjust max memory, 25% of total memory - string percent = "25%"; + string percent; + if (totalUmMemoryParam.empty()) { + percent = "25%"; - if ( DBRootStorageType == "hdfs") - { - percent = "12%"; + if (DBRootStorageType == "hdfs") + { + percent = "12%"; + } + + cout << " Setting 'TotalUmMemory' to " << percent << " of total memory. " << endl; + } + else { + percent = totalUmMemoryParam; + cout << " Setting 'TotalUmMemory' to " << percent << endl; } - - cout << " Setting 'TotalUmMemory' to " << percent << " of total memory. " << endl; - try { sysConfig->setConfig("HashJoin", "TotalUmMemory", percent); @@ -305,29 +339,47 @@ int main(int argc, char* argv[]) { try { - string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); - string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + string numBlocksPct; + string totalUmMemory; - if (numBlocksPct.empty() || numBlocksPct == "" ) - { - numBlocksPct = "50"; - - try - { - sysConfig->setConfig("DBBC", "NumBlocksPct", "50"); - - cout << endl << "NOTE: Setting 'NumBlocksPct' to 50%" << endl; - } - catch (...) - { - cout << "ERROR: Problem setting NumBlocksPct in the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } + if (!numBlocksPctParam.empty()) { + numBlocksPct = numBlocksPctParam; + } + else { + numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + } + if (!totalUmMemoryParam.empty()) { + totalUmMemory = totalUmMemoryParam; + } + else { + totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); } - cout << endl << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; - cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; + if (numBlocksPct.empty() || numBlocksPct == "") + { + numBlocksPct = "50"; + } + if (totalUmMemory.empty() || totalUmMemory == "") { + totalUmMemory = "25%"; + } + try + { + sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); + sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemory); + if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + } + cout << " Setting 'TotalUmMemory' to " << totalUmMemory << endl; + } + catch (...) + { + cout << "ERROR: Problem setting NumBlocksPct/TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } } catch (...) { @@ -350,7 +402,23 @@ int main(int argc, char* argv[]) // are we using settings from previous config file? if ( reuseConfig == "n" ) { - cout << endl << "NOTE: Using the default setting for 'NumBlocksPct' at 70%" << endl; + string numBlocksPct = "70"; + string totalUmMemory = "50%"; + + if (!numBlocksPctParam.empty()) { + numBlocksPct = numBlocksPctParam; + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Using the default setting for 'NumBlocksPct' at " << numBlocksPct << "%" << endl; + } + if (!totalUmMemoryParam.empty()) { + totalUmMemory = totalUmMemoryParam; + cout << endl << "Setting 'TotalUmMemory' to " << totalUmMemory << endl; + } + else { + cout << endl << "Setting 'TotalUmMemory' to " << totalUmMemory << " of total memory." << endl; + } try { @@ -358,17 +426,14 @@ int main(int argc, char* argv[]) } catch (...) {} - // adjust max memory, 50% of total memory - - cout << endl << "Setting 'TotalUmMemory' to 50% of total memory." << endl; - try { - sysConfig->setConfig("HashJoin", "TotalUmMemory", "50%"); + sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); + sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemory); } catch (...) { - cout << "ERROR: Problem setting TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl; + cout << "ERROR: Problem setting TotalUmMemory/NumBlocksPct in the MariaDB ColumnStore System Configuration file" << endl; exit(1); } } @@ -376,15 +441,47 @@ int main(int argc, char* argv[]) { try { - string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); - string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + string numBlocksPct; + string totalUmMemory; - if (numBlocksPct.empty() || numBlocksPct == "" ) + if (!numBlocksPctParam.empty()) { + numBlocksPct = numBlocksPctParam; + } + else { + numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + } + if (!totalUmMemoryParam.empty()) { + totalUmMemory = totalUmMemoryParam; + } + else { + totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + } + + if (numBlocksPct.empty() || numBlocksPct == "") + { numBlocksPct = "70"; + } + if (totalUmMemory.empty() || totalUmMemory == "") { + totalUmMemory = "50%"; + } + try + { + sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); + sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemory); - cout << endl << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; - cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; - + if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + } + cout << " Setting 'TotalUmMemory' to " << totalUmMemory << endl; + } + catch (...) + { + cout << "ERROR: Problem setting NumBlocksPct/TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } } catch (...) { diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index bb828b5be..fdfc1482c 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -14,7 +14,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ + MA 02110-1301, USA. */ /****************************************************************************************** * $Id: postConfigure.cpp 64 2006-10-12 22:21:51Z dhill $ @@ -55,7 +55,6 @@ #include #include #include -#include #include /* for strncpy */ #include @@ -111,8 +110,6 @@ typedef std::vector ModuleIpList; void offLineAppCheck(); bool setOSFiles(string parentOAMModuleName, int serverTypeInstall); bool checkSaveConfigFile(); -string getModuleName(); -bool setModuleName(string moduleName); bool updateBash(); bool makeModuleFile(string moduleName, string parentOAMModuleName); bool updateProcessConfig(); @@ -126,10 +123,11 @@ bool singleServerDBrootSetup(); bool copyFstab(string moduleName); bool attachVolume(string instanceName, string volumeName, string deviceName, string dbrootPath); void singleServerConfigSetup(Config* sysConfig); +std::string resolveHostNameToReverseDNSName(std::string hostname); void remoteInstallThread(void*); -bool glusterSetup(string password); +bool glusterSetup(string password, bool doNotResolveHostNames); std::string launchInstance(ModuleIP moduleip); @@ -192,6 +190,8 @@ bool nonDistributeFlag = false; bool single_server_quick_install = false; bool multi_server_quick_install = false; bool amazon_quick_install = false; +bool doNotResolveHostNames = false; +bool resolveHostNamesToReverseDNSNames = false; string DataFileEnvFile; @@ -230,6 +230,8 @@ int main(int argc, char* argv[]) string cmd; string pmIpAddrs = ""; string umIpAddrs = ""; + string numBlocksPctParam = ""; + string totalUmMemoryParam = ""; // struct sysinfo myinfo; @@ -275,12 +277,12 @@ int main(int argc, char* argv[]) usergroup = getgid(); string SUDO = ""; - if (user != 0) + if (user != 0) { rootUser = false; SUDO = "sudo "; } - + char* p = getenv("USER"); if (p && *p) @@ -298,7 +300,7 @@ int main(int argc, char* argv[]) for ( int i = 1; i < argc; i++ ) { - if( string("-h") == argv[i] ) + if( string("-h") == argv[i] ) { cout << endl; cout << "This is the MariaDB ColumnStore System Configuration and Installation tool." << endl; @@ -313,7 +315,7 @@ int main(int argc, char* argv[]) cout << " Enter one of the options within [], if available, or" << endl; cout << " Enter a new value" << endl << endl; cout << endl; - cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count]" << endl; + cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-x][-xr][-numBlocksPct][-totalUmMemory]" << endl; cout << " -h Help" << endl; cout << " -c Config File to use to extract configuration data, default is Columnstore.xml.rpmsave" << endl; cout << " -u Upgrade, Install using the Config File from -c, default to Columnstore.xml.rpmsave" << endl; @@ -322,13 +324,27 @@ int main(int argc, char* argv[]) cout << " -qs Quick Install - Single Server" << endl; cout << " -qm Quick Install - Multi Server" << endl; cout << " -port MariaDB ColumnStore Port Address" << endl; - cout << " -i Non-root Install directory, Only use for non-root installs" << endl; - cout << " -n Non-distributed install, meaning postConfigure will not install packages on remote nodes" << endl; - cout << " -d Distributed install, meaning postConfigure will install packages on remote nodes" << endl; + cout << " -i Non-root Install directory, Only use for non-root installs" << endl; + cout << " -n Non-distributed install, meaning postConfigure will not install packages on remote nodes" << endl; + cout << " -d Distributed install, meaning postConfigure will install packages on remote nodes" << endl; cout << " -sn System Name" << endl; cout << " -pm-ip-addrs Performance Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; cout << " -um-ip-addrs User Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; + cout << " -x Do not resolve IP Addresses from host names" << endl; + cout << " -xr Resolve host names into their reverse DNS host names. Only applied in combination with -x" << endl; + cout << " -numBlocksPct amount of physical memory to utilize for disk block caching" << endl; + cout << " (percentages of the total memory need to be stated without suffix, explcit values with suffixes M or G)" << endl; + cout << " -totalUmMemory amount of physical memory to utilize for joins, intermediate results and set operations on the UM" << endl; + cout << " (percentages of the total memory need to be stated with suffix %, explcit values with suffixes M or G)" << endl; exit (0); + } + else if (string("-x") == argv[i]) + { + doNotResolveHostNames = true; + } + else if (string("-xr") == argv[i]) + { + resolveHostNamesToReverseDNSNames = true; } else if( string("-qs") == argv[i] ) { @@ -349,33 +365,33 @@ int main(int argc, char* argv[]) nodeps = "--nodeps"; else if ( string("-o") == argv[i] ) startOfflinePrompt = true; - else if( string("-c") == argv[i] ) + else if( string("-c") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: Config File not provided" << endl; exit (1); } oldFileName = argv[i]; - if ( oldFileName.find("Columnstore.xml") == string::npos ) + if ( oldFileName.find("Columnstore.xml") == string::npos ) { cout << " ERROR: Config File is not a Columnstore.xml file name" << endl; exit (1); } } - else if( string("-p") == argv[i] ) + else if( string("-p") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: Password not provided" << endl; exit (1); } password = argv[i]; - if ( password.find("-") != string::npos ) + if ( password.find("-") != string::npos ) { cout << " ERROR: Valid Password not provided" << endl; exit (1); @@ -394,10 +410,10 @@ int main(int argc, char* argv[]) nonDistribute = false; nonDistributeFlag = true; } - else if( string("-port") == argv[i] ) + else if( string("-port") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: MariaDB ColumnStore Port ID not supplied" << endl; exit (1); @@ -411,10 +427,10 @@ int main(int argc, char* argv[]) exit (1); } } - else if( string("-i") == argv[i] ) + else if( string("-i") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: Path not provided" << endl; exit (1); @@ -422,64 +438,94 @@ int main(int argc, char* argv[]) installDir = argv[i]; } - else if( string("-sn") == argv[i] ) + else if( string("-sn") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: System-name not provided" << endl; exit (1); } systemName = argv[i]; } - else if( string("-pm-ip-addrs") == argv[i] ) + else if( string("-pm-ip-addrs") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: PM-IP-ADRESSES not provided" << endl; exit (1); } pmIpAddrs = argv[i]; } - else if( string("-um-ip-addrs") == argv[i] ) + else if( string("-um-ip-addrs") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: UM-IP-ADRESSES not provided" << endl; exit (1); } umIpAddrs = argv[i]; } - else if( string("-pm-count") == argv[i] ) + else if( string("-pm-count") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: PM-COUNT not provided" << endl; exit (1); } pmNumber = atoi(argv[i]); } - else if( string("-um-count") == argv[i] ) + else if( string("-um-count") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: UM-COUNT not provided" << endl; exit (1); } umNumber = atoi(argv[i]); } + else if ( string("-numBlocksPct") == argv[i] ) + { + i++; + if (i >= argc) + { + cout << " ERROR: Memory settings for numBlocksPct not provided" << endl; + exit(1); + } + numBlocksPctParam = argv[i]; + // check that the parameter ends with a number M or G + if (!(isdigit(*numBlocksPctParam.rbegin()) || *numBlocksPctParam.rbegin() == 'M' || *numBlocksPctParam.rbegin() == 'G')) { + cout << " ERROR: Memory settings for numBlocksPct need to end on a digit, M or G" << endl; + exit(1); + } + } + else if (string("-totalUmMemory") == argv[i]) + { + i++; + if (i >= argc) + { + cout << " ERROR: Memory settings for totalUmMemory not provided" << endl; + exit(1); + } + totalUmMemoryParam = argv[i]; + // check that the parameter ends with a %, M, or G + if (!(*totalUmMemoryParam.rbegin() == '%' || *totalUmMemoryParam.rbegin() == 'M' || *totalUmMemoryParam.rbegin() == 'G')) { + cout << " ERROR: Memory settings for totalUmMemory need to end on %, M or G" << endl; + exit(1); + } + } else { cout << " ERROR: Invalid Argument = " << argv[i] << endl; - cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count]" << endl; + cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-x][-xr][-numBlocksPct][-totalUmMemory]" << endl; exit (1); } } - + //check if quick install multi-server has been given ip address if (multi_server_quick_install) { @@ -553,7 +599,7 @@ int main(int argc, char* argv[]) // redirectStandardOutputToFile(postConfigureLog, false ); } } - + //check if MariaDB ColumnStore is up and running if (oam.checkSystemRunning()) { @@ -567,7 +613,7 @@ int main(int argc, char* argv[]) cout << "ERROR: Configuration File not setup" << endl; exit(1); } - + //determine package type string EEPackageType; @@ -605,7 +651,7 @@ int main(int argc, char* argv[]) cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); } - + //check for local ip address as pm1 ModuleConfig moduleconfig; @@ -616,9 +662,14 @@ int main(int argc, char* argv[]) if (moduleconfig.hostConfigList.size() > 0 ) { HostConfigList::iterator pt1 = moduleconfig.hostConfigList.begin(); - string PM1ipAdd = (*pt1).IPAddr; + + // MCOL-1607. The 'am I pm1?' check below requires an ipaddr. + string PM1ipAdd = oam.getIPAddress((*pt1).IPAddr.c_str()); + if (PM1ipAdd.empty()) + PM1ipAdd = (*pt1).IPAddr; // this is what it was doing before + //cout << PM1ipAdd << endl; - + if ( PM1ipAdd != "127.0.0.1" ) { if ( PM1ipAdd != "0.0.0.0") @@ -789,7 +840,7 @@ int main(int argc, char* argv[]) } cout << endl; - + if (single_server_quick_install) { cout << "===== Quick Install Single-Server Configuration =====" << endl << endl; @@ -797,13 +848,13 @@ int main(int argc, char* argv[]) cout << "Single-Server install is used when there will only be 1 server configured" << endl; cout << "on the system. It can also be used for production systems, if the plan is" << endl; cout << "to stay single-server." << endl; - + singleServerInstall = "1"; } else if (multi_server_quick_install) { cout << "===== Quick Install Multi-Server Configuration =====" << endl << endl; - + cout << "Multi-Server install defaulting to using local storage" << endl; singleServerInstall = "2"; @@ -811,7 +862,7 @@ int main(int argc, char* argv[]) else if (amazon_quick_install) { cout << "===== Quick Install Amazon Configuration =====" << endl << endl; - + cout << "Amazon AMI EC2 install defaulting to using local storage" << endl; singleServerInstall = "2"; @@ -842,7 +893,7 @@ int main(int argc, char* argv[]) else singleServerInstall = "2"; - while(true) + while(true) { string temp = singleServerInstall; prompt = "Select the type of System Server install [1=single, 2=multi] (" + singleServerInstall + ") > "; @@ -889,7 +940,7 @@ int main(int argc, char* argv[]) //setup to Columnstore.xml file for single server singleServerConfigSetup(sysConfig); } - + //module ProcessConfig.xml to setup all apps on the pm if ( !updateProcessConfig() ) cout << "Update ProcessConfig.xml error" << endl; @@ -952,7 +1003,6 @@ int main(int argc, char* argv[]) offLineAppCheck(); checkMysqlPort(mysqlPort, sysConfig); - if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; @@ -961,13 +1011,20 @@ int main(int argc, char* argv[]) cout << endl << "===== Performing Configuration Setup and MariaDB ColumnStore Startup =====" << endl; - cmd = installDir + "/bin/installer dummy.rpm dummy.rpm dummy.rpm dummy.rpm dummy.rpm initial dummy " + reuseConfig + " --nodeps ' ' 1 " + installDir; + if (numBlocksPctParam.empty()) { + numBlocksPctParam = "-"; + } + if (totalUmMemoryParam.empty()) { + totalUmMemoryParam = "-"; + } + + cmd = installDir + "/bin/installer dummy.rpm dummy.rpm dummy.rpm dummy.rpm dummy.rpm initial dummy " + reuseConfig + " --nodeps ' ' 1 " + installDir + " " + numBlocksPctParam + " " + totalUmMemoryParam; system(cmd.c_str()); exit(0); } // perform multi-node install - + try { sysConfig->setConfig(InstallSection, "SingleServerInstall", "n"); @@ -987,7 +1044,7 @@ int main(int argc, char* argv[]) // // Multi-server install // - + ModuleIP InputModuleIP; ModuleIpList InputModuleIPList; @@ -1001,7 +1058,7 @@ int main(int argc, char* argv[]) } catch(...) {} - + if (umIpAddrs == "" ) { // set Server Type Installation to combined @@ -1026,10 +1083,10 @@ int main(int argc, char* argv[]) InputModuleIP.moduleName = module; InputModuleIPList.push_back(InputModuleIP); } - + umNumber = id-1; } - + if (pmIpAddrs != "" ) { int id = 1; @@ -1045,16 +1102,16 @@ int main(int argc, char* argv[]) InputModuleIP.moduleName = module; InputModuleIPList.push_back(InputModuleIP); } - + pmNumber = id-1; } - if ( !writeConfig(sysConfig) ) + if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); } - + MaxNicID = 1; } else @@ -1067,13 +1124,13 @@ int main(int argc, char* argv[]) } catch(...) {} - + try { sysConfig->setConfig(InstallSection, "Cloud", "amazon-vpc"); } catch(...) {} - + if (umNumber == 0 ) { // set Server Type Installation to combined @@ -1084,12 +1141,12 @@ int main(int argc, char* argv[]) {} } - if ( !writeConfig(sysConfig) ) + if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); } - + MaxNicID = 1; } } @@ -1350,7 +1407,7 @@ int main(int argc, char* argv[]) //amazon install setup check bool amazonInstall = false; string cloud = oam::UnassignedName; - + if (!multi_server_quick_install) { string amazonLog = tmpDir + "/amazon.log"; @@ -1367,7 +1424,7 @@ int main(int argc, char* argv[]) // not running on amazon with ec2-api-tools if (amazon_quick_install) { - cout << "ERROR: Amazon Quick Installer was specified, bu the Amazon CLI API packages isnt installed, exiting" << endl; + cout << "ERROR: Amazon Quick Installer was specified, but the Amazon CLI API packages is not installed, exiting" << endl; exit(1); } @@ -1380,7 +1437,7 @@ int main(int argc, char* argv[]) // not running on amazon with ec2-api-tools if (amazon_quick_install) { - cout << "ERROR: Amazon Quick Installer was specified, bu the AMazon CLI API packages isnt installed, exiting" << endl; + cout << "ERROR: Amazon Quick Installer was specified, but the Amazon CLI API packages is not installed, exiting" << endl; exit(1); } @@ -1390,7 +1447,7 @@ int main(int argc, char* argv[]) amazonInstall = true; } } - + try { cloud = sysConfig->getConfig(InstallSection, "Cloud"); @@ -1623,36 +1680,48 @@ int main(int argc, char* argv[]) case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server { // are we using settings from previous config file? - if ( reuseConfig == "n" ) + if (reuseConfig == "n") { - if ( !uncommentCalpontXml("NumBlocksPct") ) - { - cout << "Update Columnstore.xml NumBlocksPct Section" << endl; - exit(1); - } string numBlocksPct; - try - { - numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + // if numBlocksPct was set as command line parameter use the command line parameter value + if (!numBlocksPctParam.empty()) { + numBlocksPct = numBlocksPctParam; } - catch (...) - {} + else { + if (!uncommentCalpontXml("NumBlocksPct")) + { + cout << "Update Columnstore.xml NumBlocksPct Section" << endl; + exit(1); + } - if ( numBlocksPct == "70" || numBlocksPct.empty() ) - { - numBlocksPct = "50"; + try + { + numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + } + catch (...) + { + } - if (hdfs) - numBlocksPct = "25"; + if (numBlocksPct == "70" || numBlocksPct.empty()) + { + numBlocksPct = "50"; + + if (hdfs) + numBlocksPct = "25"; + } } - try { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + } } catch (...) { @@ -1660,11 +1729,20 @@ int main(int argc, char* argv[]) exit(1); } - string percent = "25%"; + + string percent; - if (hdfs) - { - percent = "12%"; + if (!totalUmMemoryParam.empty()) { // if totalUmMemory was set as command line parameter use the command line parameter value + percent = totalUmMemoryParam; + } + else { //otherwise use reasonable defaults + + percent = "25%"; + + if (hdfs) + { + percent = "12%"; + } } cout << " Setting 'TotalUmMemory' to " << percent << endl; @@ -1689,22 +1767,38 @@ int main(int argc, char* argv[]) { try { - string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); - cout << endl; - if ( numBlocksPct.empty() ) - cout << "NOTE: Using the default setting for 'NumBlocksPct' at 70%" << endl; - else - cout << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; + if (!numBlocksPctParam.empty()) { // if numBlocksPct was set as command line parameter use the command line parameter value + sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPctParam); + if (*numBlocksPctParam.rbegin() == 'M' || *numBlocksPctParam.rbegin() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << "%" << endl; + } + } + else { //otherwise use the settings from the previous config file + string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); - string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + if (numBlocksPct.empty()) + cout << "NOTE: Using the default setting for 'NumBlocksPct' at 70%" << endl; + else + cout << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; + } - cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; + if (!totalUmMemoryParam.empty()) { // if totalUmMemory was set as command line parameter use the command line parameter value + sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemoryParam); + cout << " Setting 'TotalUmMemory' to " << totalUmMemoryParam << endl; + } + else { //otherwise use the settings from the previous config file + string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; + } } catch (...) { - cout << "ERROR: Problem reading NumBlocksPct/TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl; + cout << "ERROR: Problem reading/writing NumBlocksPct/TotalUmMemory in/to the MariaDB ColumnStore System Configuration file" << endl; exit(1); } } @@ -1717,35 +1811,39 @@ int main(int argc, char* argv[]) // are we using settings from previous config file? if ( reuseConfig == "n" ) { - - - - - - - string numBlocksPct; - try - { - numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + // if numBlocksPct was set as command line parameter use the command line parameter value + if (!numBlocksPctParam.empty()) { + numBlocksPct = numBlocksPctParam; } - catch (...) - {} + else { + try + { + numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + } + catch (...) + { + } - if ( numBlocksPct.empty() ) - { - numBlocksPct = "70"; + if (numBlocksPct.empty()) + { + numBlocksPct = "70"; - if (hdfs) - numBlocksPct = "35"; + if (hdfs) + numBlocksPct = "35"; + } } try { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - - cout << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + } } catch (...) { @@ -1753,11 +1851,18 @@ int main(int argc, char* argv[]) exit(1); } - string percent = "50%"; + string percent; - if (hdfs) - { - percent = "25%"; + if (!totalUmMemoryParam.empty()) { // if totalUmMemory was set as command line parameter use the command line parameter value + percent = totalUmMemoryParam; + } + else { //otherwise use reasonable defaults + percent = "50%"; + + if (hdfs) + { + percent = "25%"; + } } cout << " Setting 'TotalUmMemory' to " << percent << endl; @@ -1782,20 +1887,36 @@ int main(int argc, char* argv[]) { try { - string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + if (!numBlocksPctParam.empty()) { // if numBlocksPct was set as command line parameter use the command line parameter value + sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPctParam); + if (*numBlocksPctParam.rbegin() == 'M' || *numBlocksPctParam.rbegin() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << "%" << endl; + } + } + else { //otherwise use the settings from the previous config file + string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); - if ( numBlocksPct.empty() ) - cout << "NOTE: Using the default setting for 'NumBlocksPct' at 70%" << endl; - else - cout << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; + if (numBlocksPct.empty()) + cout << "NOTE: Using the default setting for 'NumBlocksPct' at 70%" << endl; + else + cout << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; + } - string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); - - cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; + if (!totalUmMemoryParam.empty()) { // if totalUmMemory was set as command line parameter use the command line parameter value + sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemoryParam); + cout << " Setting 'TotalUmMemory' to " << totalUmMemoryParam << endl; + } + else { //otherwise use reasonable defaults + string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; + } } catch (...) { - cout << "ERROR: Problem reading NumBlocksPct/TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl; + cout << "ERROR: Problem reading/writing NumBlocksPct/TotalUmMemory in/to the MariaDB ColumnStore System Configuration file" << endl; exit(1); } } @@ -1877,14 +1998,14 @@ int main(int argc, char* argv[]) catch (...) {} } - + if ( moduleType == "um") if ( umNumber != 0 ) moduleCount = umNumber; - + if ( moduleType == "pm") if ( pmNumber != 0 ) - moduleCount = pmNumber; + moduleCount = pmNumber; //verify and setup of modules count switch ( IserverTypeInstall ) @@ -2044,7 +2165,7 @@ int main(int argc, char* argv[]) moduleDisableState = oam::ENABLEDSTATE; //setup HostName/IPAddress for each NIC - + string moduleHostName = oam::UnassignedName; string moduleIPAddr = oam::UnassignedIpAddr; @@ -2063,7 +2184,7 @@ int main(int argc, char* argv[]) } } } - + unsigned int nicID=1; for( ; nicID < MaxNicID +1 ; nicID++ ) { @@ -2076,9 +2197,9 @@ int main(int argc, char* argv[]) for( ; listPT != sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.end() ; listPT++) { - if (newModuleName == (*listPT).DeviceName) + if (newModuleName == (*listPT).DeviceName) { - if ( nicID == 1 ) + if ( nicID == 1 ) { moduleDisableState = (*listPT).DisableState; @@ -2091,7 +2212,7 @@ int main(int argc, char* argv[]) for( ; pt1 != (*listPT).hostConfigList.end() ; pt1++) { - if ((*pt1).NicID == nicID) + if ((*pt1).NicID == nicID) { moduleHostName = (*pt1).HostName; moduleIPAddr = (*pt1).IPAddr; @@ -2103,8 +2224,8 @@ int main(int argc, char* argv[]) } } } - - if ( nicID == 1 ) + + if ( nicID == 1 ) { if ( moduleDisableState != oam::ENABLEDSTATE ) { @@ -2452,7 +2573,16 @@ int main(int argc, char* argv[]) if ( moduleIPAddr == oam::UnassignedIpAddr ) { //get IP Address - string IPAddress = oam.getIPAddress( newModuleHostName); + string IPAddress; + if (doNotResolveHostNames) + if (resolveHostNamesToReverseDNSNames) { + IPAddress = resolveHostNameToReverseDNSName(newModuleHostName); + } + else { + IPAddress = newModuleHostName; + } + else + IPAddress = oam.getIPAddress( newModuleHostName); if ( !IPAddress.empty() ) newModuleIPAddr = IPAddress; @@ -2468,7 +2598,7 @@ int main(int argc, char* argv[]) //prompt for IP address while (true) { - prompt = "Enter Nic Interface #" + oam.itoa(nicID) + " IP Address of " + newModuleHostName + " (" + newModuleIPAddr + ") > "; + prompt = "Enter Nic Interface #" + oam.itoa(nicID) + " IP Address or hostname of " + newModuleHostName + " (" + newModuleIPAddr + ") > "; pcommand = callReadline(prompt.c_str()); if (pcommand) @@ -2477,6 +2607,13 @@ int main(int argc, char* argv[]) callFree(pcommand); } + + if (!doNotResolveHostNames) + { + string ugh = oam.getIPAddress(newModuleIPAddr); + if (ugh.length() > 0) + newModuleIPAddr = ugh; + } if (newModuleIPAddr == "127.0.0.1" || newModuleIPAddr == "0.0.0.0" || newModuleIPAddr == "128.0.0.1") { @@ -2489,7 +2626,7 @@ int main(int argc, char* argv[]) continue; } - if (oam.isValidIP(newModuleIPAddr)) + if (oam.isValidIP(newModuleIPAddr) || doNotResolveHostNames) { //check and see if hostname already used bool matchFound = false; @@ -2948,7 +3085,7 @@ int main(int argc, char* argv[]) } } } - + vector dbroots; string tempdbrootList; @@ -3497,7 +3634,7 @@ int main(int argc, char* argv[]) } if ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM || - pmNumber > 1 ) + pmNumber > 1 ) { if ( password.empty() ) { @@ -3531,10 +3668,10 @@ int main(int argc, char* argv[]) if ( p1 == p2 ) { password = p2; break; - } + } else cout << "Password mismatch, please re-enter" << endl; - } + } //add single quote for special characters if ( password != "ssh" ) @@ -3543,7 +3680,7 @@ int main(int argc, char* argv[]) } } - } + } } int thread_id = 0; @@ -3824,7 +3961,7 @@ int main(int argc, char* argv[]) cout << " DONE" << endl; } } - + //configure data redundancy if (DataRedundancy) { @@ -3897,7 +4034,7 @@ int main(int argc, char* argv[]) { cout << endl << "===== Configuring MariaDB ColumnStore Data Redundancy Functionality =====" << endl << endl; - if (!glusterSetup(password)) + if (!glusterSetup(password, doNotResolveHostNames)) { cout << "ERROR: Problem setting up ColumnStore Data Redundancy" << endl; exit(1); @@ -4039,7 +4176,7 @@ int main(int argc, char* argv[]) } } } - + string dbbuilderLog = tmpDir + "/dbbuilder.log"; if (hdfs) @@ -4060,7 +4197,7 @@ int main(int argc, char* argv[]) //send message to procmon's to run upgrade script int status = sendUpgradeRequest(IserverTypeInstall, pmwithum); - + if ( status != 0 ) { cout << endl << "MariaDB Columnstore Install Failed" << endl << endl; exit(1); @@ -4423,7 +4560,7 @@ bool updateProcessConfig() vector oldModule; string newModule = ">pm"; oldModule.push_back(">um"); - + string fileName = installDir + "/etc/ProcessConfig.xml"; //Save a copy of the original version @@ -4564,9 +4701,9 @@ bool uncommentCalpontXml( string entry) */ bool makeRClocal(string moduleType, string moduleName, int IserverTypeInstall) { - + return true; - + vector lines; string mount1; @@ -5147,7 +5284,7 @@ bool storageSetup(bool amazonInstall) //check if hadoop is installed string hadoopLog = tmpDir + "/hadoop.log"; - + string cmd = "which hadoop > " + hadoopLog + " 2>&1"; system(cmd.c_str()); @@ -5732,7 +5869,7 @@ void setSystemName() Oam oam; //setup System Name - + if ( systemName.empty() ) { try { @@ -5743,7 +5880,7 @@ void setSystemName() systemName = oam::UnassignedName; } } - + if ( systemName.empty() ) systemName = "columnstore-1"; @@ -5759,7 +5896,7 @@ void setSystemName() callFree(pcommand); } } - + try { sysConfig->setConfig(SystemSection, "SystemName", systemName); @@ -6275,7 +6412,7 @@ std::string launchInstance(ModuleIP moduleip) return instanceName; } -bool glusterSetup(string password) +bool glusterSetup(string password, bool doNotResolveHostNames) { Oam oam; int dataRedundancyCopies = 0; @@ -6429,10 +6566,19 @@ bool glusterSetup(string password) callFree(pcommand); } - if ( moduleIPAddr == oam::UnassignedIpAddr ) + if ( moduleIPAddr == oam::UnassignedIpAddr) { //get IP Address - string IPAddress = oam.getIPAddress( moduleHostName); + string IPAddress; + if (doNotResolveHostNames) + if (resolveHostNamesToReverseDNSNames) { + IPAddress = resolveHostNameToReverseDNSName(moduleHostName); + } + else { + IPAddress = moduleHostName; + } + else + IPAddress = oam.getIPAddress( moduleHostName); if ( !IPAddress.empty() ) moduleIPAddr = IPAddress; @@ -6446,7 +6592,7 @@ bool glusterSetup(string password) //prompt for IP address while (true) { - prompt = "Enter PM #" + oam.itoa(DataRedundancyConfigs[pm].pmID) + " IP Address of " + moduleHostName + " (" + moduleIPAddr + ") > "; + prompt = "Enter PM #" + oam.itoa(DataRedundancyConfigs[pm].pmID) + " IP Address or hostname of " + moduleHostName + " (" + moduleIPAddr + ") > "; pcommand = callReadline(prompt.c_str()); if (pcommand) @@ -6455,6 +6601,13 @@ bool glusterSetup(string password) callFree(pcommand); } + + if (!doNotResolveHostNames) + { + string ugh = oam.getIPAddress(moduleIPAddr); + if (ugh.length() > 0) + moduleIPAddr = ugh; + } if (moduleIPAddr == "127.0.0.1" || moduleIPAddr == "0.0.0.0" || moduleIPAddr == "128.0.0.1") { @@ -6467,7 +6620,7 @@ bool glusterSetup(string password) continue; } - if (oam.isValidIP(moduleIPAddr)) + if (oam.isValidIP(moduleIPAddr) || doNotResolveHostNames) { // run ping test to validate @@ -6891,29 +7044,29 @@ bool glusterSetup(string password) command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-uid " + oam.itoa(user) + " >> /tmp/glusterCommands.txt 2>&1";; status = system(command.c_str()); - if (WEXITSTATUS(status) != 0 ) - { - cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; - exit(1); - } +if (WEXITSTATUS(status) != 0) +{ + cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; + exit(1); +} - command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-gid " + oam.itoa(group) + " >> /tmp/glusterCommands.txt 2>&1";; - status = system(command.c_str()); +command = "sudo gluster volume set dbroot" + oam.itoa(dbrootID) + " storage.owner-gid " + oam.itoa(group) + " >> /tmp/glusterCommands.txt 2>&1";; +status = system(command.c_str()); - if (WEXITSTATUS(status) != 0 ) - { - cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; - exit(1); - } +if (WEXITSTATUS(status) != 0) +{ + cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; + exit(1); +} - command = "sudo gluster volume start dbroot" + oam.itoa(dbrootID) + " >> /tmp/glusterCommands.txt 2>&1"; - status = system(command.c_str()); +command = "sudo gluster volume start dbroot" + oam.itoa(dbrootID) + " >> /tmp/glusterCommands.txt 2>&1"; +status = system(command.c_str()); - if (WEXITSTATUS(status) != 0 ) - { - cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; - exit(1); - } +if (WEXITSTATUS(status) != 0) +{ + cout << "ERROR: Failed to start dbroot" << oam.itoa(dbrootID) << endl; + exit(1); +} } cout << "DONE" << endl; @@ -6926,71 +7079,90 @@ bool glusterSetup(string password) void singleServerConfigSetup(Config* sysConfig) { - - try - { - sysConfig->setConfig("ExeMgr1", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("ExeMgr1", "Module", "pm1"); - sysConfig->setConfig("ProcMgr", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("ProcMgr_Alarm", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("ProcStatusControl", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("pm1_ProcessMonitor", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("pm1_ServerMonitor", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("pm1_WriteEngineServer", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("DDLProc", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("DMLProc", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS1", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS2", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS3", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS4", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS5", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS6", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS7", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS8", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS9", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS10", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS11", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS12", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS13", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS14", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS15", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS16", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS17", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS18", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS19", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS20", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS21", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS22", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS23", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS24", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS25", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS26", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS27", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS28", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS29", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS30", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS31", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("PMS32", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("SystemModuleConfig", "ModuleCount2", "0"); - sysConfig->setConfig("SystemModuleConfig", "ModuleIPAddr1-1-3", "127.0.0.1"); - sysConfig->setConfig("SystemModuleConfig", "ModuleHostName1-1-3", "localhost"); - sysConfig->setConfig("DBRM_Controller", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("DBRM_Worker1", "IPAddr", "127.0.0.1"); - sysConfig->setConfig("DBRM_Worker1", "Module", "pm1"); - sysConfig->setConfig("DBBC", "NumBlocksPct", "50"); - sysConfig->setConfig("Installation", "InitialInstallFlag", "y"); - sysConfig->setConfig("Installation", "SingleServerInstall", "y"); - sysConfig->setConfig("HashJoin", "TotalUmMemory", "25%"); - } - catch (...) - { - cout << "ERROR: Problem setting for Single Server in the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } - return; + try + { + sysConfig->setConfig("ExeMgr1", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("ExeMgr1", "Module", "pm1"); + sysConfig->setConfig("ProcMgr", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("ProcMgr_Alarm", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("ProcStatusControl", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("pm1_ProcessMonitor", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("pm1_ServerMonitor", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("pm1_WriteEngineServer", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("DDLProc", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("DMLProc", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS1", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS2", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS3", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS4", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS5", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS6", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS7", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS8", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS9", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS10", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS11", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS12", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS13", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS14", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS15", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS16", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS17", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS18", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS19", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS20", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS21", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS22", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS23", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS24", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS25", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS26", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS27", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS28", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS29", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS30", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS31", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("PMS32", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("SystemModuleConfig", "ModuleCount2", "0"); + sysConfig->setConfig("SystemModuleConfig", "ModuleIPAddr1-1-3", "127.0.0.1"); + sysConfig->setConfig("SystemModuleConfig", "ModuleHostName1-1-3", "localhost"); + sysConfig->setConfig("DBRM_Controller", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("DBRM_Worker1", "IPAddr", "127.0.0.1"); + sysConfig->setConfig("DBRM_Worker1", "Module", "pm1"); + sysConfig->setConfig("DBBC", "NumBlocksPct", "50"); + sysConfig->setConfig("Installation", "InitialInstallFlag", "y"); + sysConfig->setConfig("Installation", "SingleServerInstall", "y"); + sysConfig->setConfig("HashJoin", "TotalUmMemory", "25%"); + } + catch (...) + { + cout << "ERROR: Problem setting for Single Server in the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + + return; } - +/** + Resolves the given hostname into its reverse DNS name. + + @param hostname the hostname to resolve. + @return the reverse dns name of given hostname or an empty string in case the hostname could not be resolved. +*/ +std::string resolveHostNameToReverseDNSName(std::string hostname) { + struct hostent *hp = gethostbyname(hostname.c_str()); + if (hp == NULL) { + std::cout << "Error: Couldn't resolve hostname " << hostname << " to ip address" << std::endl; + return ""; + } + struct hostent *rl = gethostbyaddr(hp->h_addr_list[0], sizeof hp->h_addr_list[0], AF_INET); + if (rl == NULL) { + std::cout << "Error: Couldn't resolve ip address of hostname " << hostname << " back to a hostname" << std::endl; + return ""; + } + hostname = rl->h_name; + return hostname; +} // vim:ts=4 sw=4: diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index f21459f22..686243f87 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -2205,10 +2205,15 @@ void processMSG(messageqcpp::IOSocket* cfIos) string value; uint16_t count, ivalue, nicCount; + uint8_t tmp8; oam::DeviceNetworkConfig devicenetworkconfig; oam::DeviceNetworkList devicenetworklist; oam::HostConfig hostconfig; + bool storeHostnames; + msg >> tmp8; + storeHostnames = (tmp8 != 0); + //get module count to add msg >> count; @@ -2223,7 +2228,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) devicenetworkconfig.UserTempDeviceName = value; msg >> value; devicenetworkconfig.DisableState = value; - + msg >> nicCount; for (int j = 0 ; j < nicCount ; j ++ ) @@ -2244,7 +2249,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) string password; msg >> password; - status = processManager.addModule(devicenetworklist, password); + status = processManager.addModule(devicenetworklist, password, storeHostnames); log.writeLog(__LINE__, "ADDMODULE: ACK received from Process-Monitor, return status = " + oam.itoa(status)); } @@ -4835,7 +4840,8 @@ int ProcessManager::reinitProcessType( std::string processName ) * purpose: Add Module to system configuration * ******************************************************************************************/ -int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::string password, bool manualFlag) +int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::string password, bool storeHostnames, + bool manualFlag) { ProcessLog log; Configuration config; @@ -5262,7 +5268,10 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str } hostconfig.HostName = hostName; - hostconfig.IPAddr = IPAddr; + if (storeHostnames) + hostconfig.IPAddr = hostName; + else + hostconfig.IPAddr = IPAddr; hostconfig.NicID = (*pt1).NicID; devicenetworkconfig.hostConfigList.push_back(hostconfig); } diff --git a/procmgr/processmanager.h b/procmgr/processmanager.h index 9a57b38b5..8dc39a540 100644 --- a/procmgr/processmanager.h +++ b/procmgr/processmanager.h @@ -416,7 +416,8 @@ public: /** *@brief Add Module */ - int addModule(oam::DeviceNetworkList devicenetworklist, std::string password, bool manualFlag = true); + int addModule(oam::DeviceNetworkList devicenetworklist, std::string password, bool storeHostnames, + bool manualFlag = true); /** *@brief Configure Module diff --git a/tools/cleartablelock/cleartablelock.cpp b/tools/cleartablelock/cleartablelock.cpp index 1f00a12e6..a18148343 100644 --- a/tools/cleartablelock/cleartablelock.cpp +++ b/tools/cleartablelock/cleartablelock.cpp @@ -650,6 +650,11 @@ int main(int argc, char** argv) rc = execBulkRollbackReq( msgQueClts, pmList, &brm, tInfo, tblName.toString(), rollbackOnly, errMsg ); + BRM::TxnID txn; + txn.id = tInfo.ownerTxnID; + txn.valid = true; + brm.rolledback(txn); + if (rc != 0) { logFinalStatus( tblName.toString(), lockID, errMsg ); diff --git a/utils/funcexp/func_truncate.cpp b/utils/funcexp/func_truncate.cpp index b822caacd..2da482ab8 100644 --- a/utils/funcexp/func_truncate.cpp +++ b/utils/funcexp/func_truncate.cpp @@ -136,7 +136,34 @@ uint64_t Func_truncate::getUintVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - return parm[0]->data()->getUintVal(row, isNull); + uint64_t val = parm[0]->data()->getUintVal(row, isNull); + + if (isNull) + return val; + + int64_t d = parm[1]->data()->getIntVal(row, isNull); + + if (isNull || d >= 0) + return val; + + uint64_t p = 1; + int64_t i = (-d); + + // Handle overflow since p can't have more than 19 0's + if (i >= 20) + { + val = 0; + } + else + { + while (i--) + p *= 10; + + val /= p; + val *= p; + } + + return val; } diff --git a/utils/regr/corr.cpp b/utils/regr/corr.cpp index 9d12680ea..c1c388da9 100644 --- a/utils/regr/corr.cpp +++ b/utils/regr/corr.cpp @@ -39,11 +39,11 @@ static Add_corr_ToUDAFMap addToMap; struct corr_data { uint64_t cnt; - double sumx; - double sumx2; // sum of (x squared) - double sumy; - double sumy2; // sum of (y squared) - double sumxy; // sum of x * y + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumy2; // sum of (y squared) + long double sumxy; // sum of x * y }; @@ -57,6 +57,13 @@ mcsv1_UDAF::ReturnCode corr::init(mcsv1Context* context, context->setErrorMessage("corr() with other than 2 arguments"); return mcsv1_UDAF::ERROR; } + if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("corr() with non-numeric arguments"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(corr_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); @@ -146,29 +153,29 @@ mcsv1_UDAF::ReturnCode corr::evaluate(mcsv1Context* context, static_any::any& va double N = data->cnt; if (N > 1) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumx2 = data->sumx2; - double sumy2 = data->sumy2; - double sumxy = data->sumxy; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumy2 = data->sumy2; + long double sumxy = data->sumxy; - double var_popx = (sumx2 - (sumx * sumx / N)) / N; + long double var_popx = (sumx2 - (sumx * sumx / N)) / N; if (var_popx == 0) { // When var_popx is 0, NULL is the result. return mcsv1_UDAF::SUCCESS; } - double var_popy = (sumy2 - (sumy * sumy / N)) / N; + long double var_popy = (sumy2 - (sumy * sumy / N)) / N; if (var_popy == 0) { // When var_popy is 0, NULL is the result return mcsv1_UDAF::SUCCESS; } - double std_popx = sqrt(var_popx); - double std_popy = sqrt(var_popy); - double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; - double corr = covar_pop / (std_popy * std_popx); - valOut = corr; + long double std_popx = sqrt(var_popx); + long double std_popy = sqrt(var_popy); + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; + long double corr = covar_pop / (std_popy * std_popx); + valOut = static_cast(corr); } return mcsv1_UDAF::SUCCESS; } diff --git a/utils/regr/covar_pop.cpp b/utils/regr/covar_pop.cpp index 51d9a036f..876be1f30 100644 --- a/utils/regr/covar_pop.cpp +++ b/utils/regr/covar_pop.cpp @@ -39,9 +39,9 @@ static Add_covar_pop_ToUDAFMap addToMap; struct covar_pop_data { uint64_t cnt; - double sumx; - double sumy; - double sumxy; // sum of x * y + long double sumx; + long double sumy; + long double sumxy; // sum of x * y }; @@ -55,6 +55,13 @@ mcsv1_UDAF::ReturnCode covar_pop::init(mcsv1Context* context, context->setErrorMessage("covar_pop() with other than 2 arguments"); return mcsv1_UDAF::ERROR; } + if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("covar_pop() with non-numeric arguments"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(covar_pop_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); @@ -138,12 +145,12 @@ mcsv1_UDAF::ReturnCode covar_pop::evaluate(mcsv1Context* context, static_any::an double N = data->cnt; if (N > 0) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumxy = data->sumxy; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumxy = data->sumxy; - double covar_pop = (sumxy - ((sumx * sumy) / N)) / N ; - valOut = covar_pop; + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N ; + valOut = static_cast(covar_pop); } return mcsv1_UDAF::SUCCESS; } diff --git a/utils/regr/covar_samp.cpp b/utils/regr/covar_samp.cpp index b0ebb168b..ccc302046 100644 --- a/utils/regr/covar_samp.cpp +++ b/utils/regr/covar_samp.cpp @@ -39,9 +39,9 @@ static Add_covar_samp_ToUDAFMap addToMap; struct covar_samp_data { uint64_t cnt; - double sumx; - double sumy; - double sumxy; // sum of x * y + long double sumx; + long double sumy; + long double sumxy; // sum of x * y }; @@ -55,6 +55,13 @@ mcsv1_UDAF::ReturnCode covar_samp::init(mcsv1Context* context, context->setErrorMessage("covar_samp() with other than 2 arguments"); return mcsv1_UDAF::ERROR; } + if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("covar_samp() with non-numeric arguments"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(covar_samp_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); @@ -138,12 +145,12 @@ mcsv1_UDAF::ReturnCode covar_samp::evaluate(mcsv1Context* context, static_any::a double N = data->cnt; if (N > 1) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumxy = data->sumxy; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumxy = data->sumxy; - double covar_samp = (sumxy - ((sumx * sumy) / N)) / (N - 1); - valOut = covar_samp; + long double covar_samp = (sumxy - ((sumx * sumy) / N)) / (N - 1); + valOut = static_cast(covar_samp); } else if (N == 1) diff --git a/utils/regr/regr_avgx.cpp b/utils/regr/regr_avgx.cpp index 2041647a5..bf010e648 100644 --- a/utils/regr/regr_avgx.cpp +++ b/utils/regr/regr_avgx.cpp @@ -40,7 +40,7 @@ static Add_regr_avgx_ToUDAFMap addToMap; // Use the simple data model struct regr_avgx_data { - double sum; + long double sum; uint64_t cnt; }; @@ -63,6 +63,13 @@ mcsv1_UDAF::ReturnCode regr_avgx::init(mcsv1Context* context, context->setErrorMessage("regr_avgx() with a non-numeric x argument"); return mcsv1_UDAF::ERROR; } + if (!(isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_avgx() with a non-numeric independant (second) argument"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(regr_avgx_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); @@ -123,13 +130,9 @@ mcsv1_UDAF::ReturnCode regr_avgx::evaluate(mcsv1Context* context, static_any::an { struct regr_avgx_data* data = (struct regr_avgx_data*)context->getUserData()->data; - if (data->cnt == 0) + if (data->cnt > 0) { - valOut = 0; - } - else - { - valOut = data->sum / (double)data->cnt; + valOut = static_cast(data->sum / (long double)data->cnt); } return mcsv1_UDAF::SUCCESS; diff --git a/utils/regr/regr_avgy.cpp b/utils/regr/regr_avgy.cpp index 69c654acf..7325d991f 100644 --- a/utils/regr/regr_avgy.cpp +++ b/utils/regr/regr_avgy.cpp @@ -40,7 +40,7 @@ static Add_regr_avgy_ToUDAFMap addToMap; // Use the simple data model struct regr_avgy_data { - double sum; + long double sum; uint64_t cnt; }; @@ -63,6 +63,13 @@ mcsv1_UDAF::ReturnCode regr_avgy::init(mcsv1Context* context, context->setErrorMessage("regr_avgy() with a non-numeric x argument"); return mcsv1_UDAF::ERROR; } + if (!(isNumeric(colTypes[0].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_avgy() with a non-numeric dependant (first) argument"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(regr_avgy_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); @@ -121,13 +128,9 @@ mcsv1_UDAF::ReturnCode regr_avgy::evaluate(mcsv1Context* context, static_any::an { struct regr_avgy_data* data = (struct regr_avgy_data*)context->getUserData()->data; - if (data->cnt == 0) + if (data->cnt > 0) { - valOut = 0; - } - else - { - valOut = data->sum / (double)data->cnt; + valOut = static_cast(data->sum / (long double)data->cnt); } return mcsv1_UDAF::SUCCESS; } diff --git a/utils/regr/regr_intercept.cpp b/utils/regr/regr_intercept.cpp index 6d4c35a47..df9310f03 100644 --- a/utils/regr/regr_intercept.cpp +++ b/utils/regr/regr_intercept.cpp @@ -39,10 +39,10 @@ static Add_regr_intercept_ToUDAFMap addToMap; struct regr_intercept_data { uint64_t cnt; - double sumx; - double sumx2; // sum of (x squared) - double sumy; - double sumxy; // sum of (x*y) + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumxy; // sum of x * y }; @@ -56,6 +56,13 @@ mcsv1_UDAF::ReturnCode regr_intercept::init(mcsv1Context* context, context->setErrorMessage("regr_intercept() with other than 2 arguments"); return mcsv1_UDAF::ERROR; } + if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_intercept() with non-numeric arguments"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(regr_intercept_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); @@ -139,17 +146,17 @@ mcsv1_UDAF::ReturnCode regr_intercept::evaluate(mcsv1Context* context, static_an { struct regr_intercept_data* data = (struct regr_intercept_data*)context->getUserData()->data; double N = data->cnt; - if (N > 0) + if (N > 1) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumx2 = data->sumx2; - double sumxy = data->sumxy; - double numerator = sumy * sumx2 - sumx * sumxy; - double var_pop = (N * sumx2) - (sumx * sumx); + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumxy = data->sumxy; + long double numerator = sumy * sumx2 - sumx * sumxy; + long double var_pop = (N * sumx2) - (sumx * sumx); if (var_pop != 0) { - valOut = numerator / var_pop; + valOut = static_cast(numerator / var_pop); } } return mcsv1_UDAF::SUCCESS; diff --git a/utils/regr/regr_r2.cpp b/utils/regr/regr_r2.cpp index 34e8888e8..1abd3ea2e 100644 --- a/utils/regr/regr_r2.cpp +++ b/utils/regr/regr_r2.cpp @@ -39,11 +39,11 @@ static Add_regr_r2_ToUDAFMap addToMap; struct regr_r2_data { uint64_t cnt; - double sumx; - double sumx2; // sum of (x squared) - double sumy; - double sumy2; // sum of (y squared) - double sumxy; // sum of x * y + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumy2; // sum of (y squared) + long double sumxy; // sum of x * y }; @@ -57,6 +57,13 @@ mcsv1_UDAF::ReturnCode regr_r2::init(mcsv1Context* context, context->setErrorMessage("regr_r2() with other than 2 arguments"); return mcsv1_UDAF::ERROR; } + if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_r2() with non-numeric arguments"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(regr_r2_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); @@ -144,15 +151,15 @@ mcsv1_UDAF::ReturnCode regr_r2::evaluate(mcsv1Context* context, static_any::any& { struct regr_r2_data* data = (struct regr_r2_data*)context->getUserData()->data; double N = data->cnt; - if (N > 0) + if (N > 1) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumx2 = data->sumx2; - double sumy2 = data->sumy2; - double sumxy = data->sumxy; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumy2 = data->sumy2; + long double sumxy = data->sumxy; - double var_popx = (sumx2 - (sumx * sumx / N)) / N; + long double var_popx = (sumx2 - (sumx * sumx / N)) / N; if (var_popx == 0) { // When var_popx is 0, NULL is the result. @@ -165,11 +172,11 @@ mcsv1_UDAF::ReturnCode regr_r2::evaluate(mcsv1Context* context, static_any::any& valOut = 1.0; return mcsv1_UDAF::SUCCESS; } - double std_popx = sqrt(var_popx); - double std_popy = sqrt(var_popy); - double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; - double corr = covar_pop / (std_popy * std_popx); - valOut = corr * corr; + long double std_popx = sqrt(var_popx); + long double std_popy = sqrt(var_popy); + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; + long double corr = covar_pop / (std_popy * std_popx); + valOut = static_cast(corr * corr); } return mcsv1_UDAF::SUCCESS; } diff --git a/utils/regr/regr_slope.cpp b/utils/regr/regr_slope.cpp index da178673a..de9eab5c7 100644 --- a/utils/regr/regr_slope.cpp +++ b/utils/regr/regr_slope.cpp @@ -39,10 +39,10 @@ static Add_regr_slope_ToUDAFMap addToMap; struct regr_slope_data { uint64_t cnt; - double sumx; - double sumx2; // sum of (x squared) - double sumy; - double sumxy; // sum of (x*y) + long double sumx; + long double sumx2; // sum of (x squared) + long double sumy; + long double sumxy; // sum of x * y }; @@ -56,7 +56,13 @@ mcsv1_UDAF::ReturnCode regr_slope::init(mcsv1Context* context, context->setErrorMessage("regr_slope() with other than 2 arguments"); return mcsv1_UDAF::ERROR; } - + if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_slope() with non-numeric arguments"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(regr_slope_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); context->setColWidth(8); @@ -139,19 +145,19 @@ mcsv1_UDAF::ReturnCode regr_slope::evaluate(mcsv1Context* context, static_any::a { struct regr_slope_data* data = (struct regr_slope_data*)context->getUserData()->data; double N = data->cnt; - if (N > 0) + if (N > 1) { // COVAR_POP(y, x) / VAR_POP(x) - double sumx = data->sumx; - double sumy = data->sumy; - double sumx2 = data->sumx2; - double sumxy = data->sumxy; - double covar_pop = N * sumxy - sumx * sumy; - double var_pop = N * sumx2 - sumx * sumx; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumx2 = data->sumx2; + long double sumxy = data->sumxy; + long double covar_pop = N * sumxy - sumx * sumy; + long double var_pop = N * sumx2 - sumx * sumx; if (var_pop != 0) { - double slope = covar_pop / var_pop; - valOut = slope; + long double slope = covar_pop / var_pop; + valOut = static_cast(slope); } } return mcsv1_UDAF::SUCCESS; diff --git a/utils/regr/regr_sxx.cpp b/utils/regr/regr_sxx.cpp index 3f06af61b..5769a227b 100644 --- a/utils/regr/regr_sxx.cpp +++ b/utils/regr/regr_sxx.cpp @@ -39,8 +39,8 @@ static Add_regr_sxx_ToUDAFMap addToMap; struct regr_sxx_data { uint64_t cnt; - double sumx; - double sumx2; // sum of (x squared) + long double sumx; + long double sumx2; // sum of (x squared) }; @@ -54,6 +54,13 @@ mcsv1_UDAF::ReturnCode regr_sxx::init(mcsv1Context* context, context->setErrorMessage("regr_sxx() with other than 2 arguments"); return mcsv1_UDAF::ERROR; } + if (!(isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_sxx() with a non-numeric independant (second) argument"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(regr_sxx_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); @@ -121,11 +128,11 @@ mcsv1_UDAF::ReturnCode regr_sxx::evaluate(mcsv1Context* context, static_any::any double N = data->cnt; if (N > 0) { - double sumx = data->sumx; - double sumx2 = data->sumx2; + long double sumx = data->sumx; + long double sumx2 = data->sumx2; - double var_popx = (sumx2 - (sumx * sumx / N)) / N; - valOut = data->cnt * var_popx; + long double var_popx = (sumx2 - (sumx * sumx / N)) / N; + valOut = static_cast(data->cnt * var_popx); } return mcsv1_UDAF::SUCCESS; } diff --git a/utils/regr/regr_sxy.cpp b/utils/regr/regr_sxy.cpp index e6d005597..76e1373c4 100644 --- a/utils/regr/regr_sxy.cpp +++ b/utils/regr/regr_sxy.cpp @@ -39,9 +39,9 @@ static Add_regr_sxy_ToUDAFMap addToMap; struct regr_sxy_data { uint64_t cnt; - double sumx; - double sumy; - double sumxy; // sum of x * y + long double sumx; + long double sumy; + long double sumxy; // sum of x * y }; @@ -55,6 +55,13 @@ mcsv1_UDAF::ReturnCode regr_sxy::init(mcsv1Context* context, context->setErrorMessage("regr_sxy() with other than 2 arguments"); return mcsv1_UDAF::ERROR; } + if (!(isNumeric(colTypes[0].dataType) && isNumeric(colTypes[1].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_sxy() with non-numeric arguments"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(regr_sxy_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); @@ -81,8 +88,8 @@ mcsv1_UDAF::ReturnCode regr_sxy::nextValue(mcsv1Context* context, ColumnDatum* v static_any::any& valIn_y = valsIn[0].columnData; static_any::any& valIn_x = valsIn[1].columnData; struct regr_sxy_data* data = (struct regr_sxy_data*)context->getUserData()->data; - double valx = 0.0; - double valy = 0.0; + long double valx = 0.0; + long double valy = 0.0; valx = convertAnyTo(valIn_x); valy = convertAnyTo(valIn_y); @@ -138,13 +145,13 @@ mcsv1_UDAF::ReturnCode regr_sxy::evaluate(mcsv1Context* context, static_any::any double N = data->cnt; if (N > 0) { - double sumx = data->sumx; - double sumy = data->sumy; - double sumxy = data->sumxy; + long double sumx = data->sumx; + long double sumy = data->sumy; + long double sumxy = data->sumxy; - double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; - double regr_sxy = data->cnt * covar_pop; - valOut = regr_sxy; + long double covar_pop = (sumxy - ((sumx * sumy) / N)) / N; + long double regr_sxy = data->cnt * covar_pop; + valOut = static_cast(regr_sxy); } return mcsv1_UDAF::SUCCESS; } diff --git a/utils/regr/regr_syy.cpp b/utils/regr/regr_syy.cpp index d0841f723..014a28389 100644 --- a/utils/regr/regr_syy.cpp +++ b/utils/regr/regr_syy.cpp @@ -39,8 +39,8 @@ static Add_regr_syy_ToUDAFMap addToMap; struct regr_syy_data { uint64_t cnt; - double sumy; - double sumy2; // sum of (y squared) + long double sumy; + long double sumy2; // sum of (y squared) }; @@ -54,6 +54,13 @@ mcsv1_UDAF::ReturnCode regr_syy::init(mcsv1Context* context, context->setErrorMessage("regr_syy() with other than 2 arguments"); return mcsv1_UDAF::ERROR; } + if (!(isNumeric(colTypes[0].dataType))) + { + // The error message will be prepended with + // "The storage engine for the table doesn't support " + context->setErrorMessage("regr_syy() with a non-numeric dependant (first) argument"); + return mcsv1_UDAF::ERROR; + } context->setUserDataSize(sizeof(regr_syy_data)); context->setResultType(CalpontSystemCatalog::DOUBLE); @@ -118,14 +125,14 @@ mcsv1_UDAF::ReturnCode regr_syy::subEvaluate(mcsv1Context* context, const UserDa mcsv1_UDAF::ReturnCode regr_syy::evaluate(mcsv1Context* context, static_any::any& valOut) { struct regr_syy_data* data = (struct regr_syy_data*)context->getUserData()->data; - double N = data->cnt; + long double N = data->cnt; if (N > 0) { - double sumy = data->sumy; - double sumy2 = data->sumy2; + long double sumy = data->sumy; + long double sumy2 = data->sumy2; - double var_popy = (sumy2 - (sumy * sumy / N)) / N; - valOut = data->cnt * var_popy; + long double var_popy = (sumy2 - (sumy * sumy / N)) / N; + valOut = static_cast(data->cnt * var_popy); } return mcsv1_UDAF::SUCCESS; } diff --git a/utils/regr/regrmysql.cpp b/utils/regr/regrmysql.cpp index 08f46bb11..4980108e3 100644 --- a/utils/regr/regrmysql.cpp +++ b/utils/regr/regrmysql.cpp @@ -2,12 +2,30 @@ #include #include #include +#include using namespace std; #include "idb_mysql.h" namespace { +inline bool isNumeric(int type, const char* attr) +{ + if (type == INT_RESULT || type == REAL_RESULT || type == DECIMAL_RESULT) + { + return true; + } +#if _MSC_VER + if (_strnicmp("NULL", attr, 4) == 0)) +#else + if (strncasecmp("NULL", attr, 4) == 0) +#endif + { + return true; + } + return false; +} + inline double cvtArgToDouble(int t, const char* v) { double d = 0.0; @@ -144,7 +162,12 @@ extern "C" strcpy(message,"regr_avgx() requires two arguments"); return 1; } - + if (!(isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message,"regr_avgx() with a non-numeric independant (second) argument"); + return 1; + } + if (!(data = (struct regr_avgx_data*) malloc(sizeof(struct regr_avgx_data)))) { strmov(message,"Couldn't allocate memory"); @@ -228,6 +251,11 @@ extern "C" strcpy(message,"regr_avgy() requires two arguments"); return 1; } + if (!(isNumeric(args->arg_type[0], args->attributes[0]))) + { + strcpy(message,"regr_avgy() with a non-numeric dependant (first) argument"); + return 1; + } if (!(data = (struct regr_avgy_data*) malloc(sizeof(struct regr_avgy_data)))) { @@ -394,6 +422,11 @@ extern "C" strcpy(message,"regr_slope() requires two arguments"); return 1; } + if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message,"regr_slope() with non-numeric arguments"); + return 1; + } if (!(data = (struct regr_slope_data*) malloc(sizeof(struct regr_slope_data)))) { @@ -505,6 +538,11 @@ extern "C" strcpy(message,"regr_intercept() requires two arguments"); return 1; } + if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message,"regr_intercept() with non-numeric arguments"); + return 1; + } if (!(data = (struct regr_intercept_data*) malloc(sizeof(struct regr_intercept_data)))) { @@ -619,6 +657,11 @@ extern "C" strcpy(message,"regr_r2() requires two arguments"); return 1; } + if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message,"regr_r2() with non-numeric arguments"); + return 1; + } if (!(data = (struct regr_r2_data*) malloc(sizeof(struct regr_r2_data)))) { @@ -748,6 +791,11 @@ extern "C" strcpy(message,"corr() requires two arguments"); return 1; } + if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message,"corr() with non-numeric arguments"); + return 1; + } if (!(data = (struct corr_data*) malloc(sizeof(struct corr_data)))) { @@ -874,6 +922,11 @@ extern "C" strcpy(message,"regr_sxx() requires two arguments"); return 1; } + if (!(isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message,"regr_avgx() with a non-numeric independant (second) argument"); + return 1; + } if (!(data = (struct regr_sxx_data*) malloc(sizeof(struct regr_sxx_data)))) { @@ -970,6 +1023,11 @@ extern "C" strcpy(message,"regr_syy() requires two arguments"); return 1; } + if (!(isNumeric(args->arg_type[0], args->attributes[0]))) + { + strcpy(message,"regr_syy() with a non-numeric dependant (first) argument"); + return 1; + } if (!(data = (struct regr_syy_data*) malloc(sizeof(struct regr_syy_data)))) { @@ -1068,6 +1126,11 @@ extern "C" strcpy(message,"regr_sxy() requires two arguments"); return 1; } + if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message,"regr_sxy() with non-numeric arguments"); + return 1; + } if (!(data = (struct regr_sxy_data*) malloc(sizeof(struct regr_sxy_data)))) { @@ -1171,6 +1234,11 @@ extern "C" strcpy(message,"covar_pop() requires two arguments"); return 1; } + if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message,"covar_pop() with non-numeric arguments"); + return 1; + } if (!(data = (struct covar_pop_data*) malloc(sizeof(struct covar_pop_data)))) { @@ -1273,6 +1341,11 @@ extern "C" strcpy(message,"covar_samp() requires two arguments"); return 1; } + if (!(isNumeric(args->arg_type[0], args->attributes[0]) && isNumeric(args->arg_type[1], args->attributes[1]))) + { + strcpy(message,"covar_samp() with non-numeric arguments"); + return 1; + } if (!(data = (struct covar_samp_data*) malloc(sizeof(struct covar_samp_data)))) { diff --git a/utils/rowgroup/rowgroup.cpp b/utils/rowgroup/rowgroup.cpp index 3411f6fa6..3ecb4998f 100644 --- a/utils/rowgroup/rowgroup.cpp +++ b/utils/rowgroup/rowgroup.cpp @@ -1211,12 +1211,13 @@ int64_t Row::getSignedNullValue(uint32_t colIndex) const RowGroup::RowGroup() : columnCount(0), data(NULL), rgData(NULL), strings(NULL), useStringTable(true), hasLongStringField(false), sTableThreshold(20) { - oldOffsets.reserve(1024); - oids.reserve(1024); - keys.reserve(1024); - types.reserve(1024); - scale.reserve(1024); - precision.reserve(1024); + // 1024 is too generous to waste. + oldOffsets.reserve(10); + oids.reserve(10); + keys.reserve(10); + types.reserve(10); + scale.reserve(10); + precision.reserve(10); } RowGroup::RowGroup(uint32_t colCount, diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index e1c3073c4..ca05360c2 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -1183,6 +1183,7 @@ inline bool Row::equals(const Row& r2, const std::vector& keyCols) con inline bool Row::equals(const Row& r2, uint32_t lastCol) const { + // This check fires with empty r2 only. if (lastCol >= columnCount) return true; diff --git a/utils/udfsdk/mcsv1_udaf.h b/utils/udfsdk/mcsv1_udaf.h index ec0d0cb79..d6ba04483 100644 --- a/utils/udfsdk/mcsv1_udaf.h +++ b/utils/udfsdk/mcsv1_udaf.h @@ -976,7 +976,7 @@ inline mcsv1_UDAF::ReturnCode mcsv1_UDAF::createUserData(UserData*& userData, in template inline T mcsv1_UDAF::convertAnyTo(static_any::any& valIn) { - T val; + T val = 0; if (valIn.compatible(longTypeId)) { val = valIn.cast(); diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index 8a021f8d8..db5cdf1c1 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -461,7 +461,8 @@ uint64_t IdbOrderBy::Hasher::operator()(const Row::Pointer& p) const { Row& row = ts->row1; row.setPointer(p); - uint64_t ret = row.hash(colCount); + // MCOL-1829 Row::h uses colcount as an array idx down a callstack. + uint64_t ret = row.hash(); //cout << "hash(): returning " << ret << " for row: " << row.toString() << endl; return ret; } @@ -471,7 +472,9 @@ bool IdbOrderBy::Eq::operator()(const Row::Pointer& d1, const Row::Pointer& d2) Row& r1 = ts->row1, &r2 = ts->row2; r1.setPointer(d1); r2.setPointer(d2); - bool ret = r1.equals(r2, colCount); + // MCOL-1829 Row::equals uses 2nd argument as container size boundary + // so it must be column count - 1. + bool ret = r1.equals(r2, colCount - 1); //cout << "equals(): returning " << (int) ret << " for r1: " << r1.toString() << " r2: " << r2.toString() // << endl; diff --git a/utils/windowfunction/wf_udaf.cpp b/utils/windowfunction/wf_udaf.cpp index 2b7e28abc..91ae5f9b2 100644 --- a/utils/windowfunction/wf_udaf.cpp +++ b/utils/windowfunction/wf_udaf.cpp @@ -84,7 +84,6 @@ void WF_udaf::resetData() getContext().getFunction()->reset(&getContext()); fDistinctMap.clear(); WindowFunctionType::resetData(); - fValOut.reset(); } void WF_udaf::parseParms(const std::vector& parms) @@ -714,6 +713,7 @@ void WF_udaf::operator()(int64_t b, int64_t e, int64_t c) (fPrev == -1) || (!fPeer->operator()(getPointer(fRowData->at(c)), getPointer(fRowData->at(fPrev))))) { + fValOut.reset(); // for unbounded - current row special handling if (fPrev >= b && fPrev < c) b = c; diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index efede3ef5..68015d120 100644 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -325,6 +325,28 @@ template<> void WindowFunctionType::setValue(uint64_t i, string& t) fRow.setStringField(t, i); } +// MCOL-1676 Need a separate specialization for string now. +template<> +void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, string* v) +{ + if (c != WF__BOUND_ALL) + b = e = c; + + uint64_t i = fFieldIndex[0]; + + if (v == NULL) + v = (string*) getNullValueByType(ct, i); + + for (int64_t j = b; j <= e; j++) + { + if (j % 1000 == 0 && fStep->cancelled()) + break; + + fRow.setData(getPointer((*fRowData)[j])); + setValue(i, *v); + } +} + template void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, T* v) { @@ -342,8 +364,54 @@ void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, T* v) break; fRow.setData(getPointer((*fRowData)[j])); - setValue(i, *v); - } + // MCOL-1676 Set the data based on out column type (ct) + switch (ct) + { + case CalpontSystemCatalog::TINYINT: + case CalpontSystemCatalog::SMALLINT: + case CalpontSystemCatalog::MEDINT: + case CalpontSystemCatalog::INT: + case CalpontSystemCatalog::BIGINT: + case CalpontSystemCatalog::DECIMAL: + { + int64_t iv = *v; + setValue(i, iv); + break; + } + + case CalpontSystemCatalog::UTINYINT: + case CalpontSystemCatalog::USMALLINT: + case CalpontSystemCatalog::UMEDINT: + case CalpontSystemCatalog::UINT: + case CalpontSystemCatalog::UBIGINT: + case CalpontSystemCatalog::UDECIMAL: + { + uint64_t uv = *v; + setValue(i, uv); + break; + } + + case CalpontSystemCatalog::DOUBLE: + case CalpontSystemCatalog::UDOUBLE: + { + double dv = *v; + setValue(i, dv); + break; + } + + case CalpontSystemCatalog::FLOAT: + case CalpontSystemCatalog::UFLOAT: + { + float fv = *v; + setValue(i, fv); + break; + } + default: + { + setValue(i, *v); + } + } + } } template @@ -476,8 +544,6 @@ template void WindowFunctionType::setValue(int, int64_t, int64_t, int64 template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, uint64_t*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, float*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, double*); -template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, string*); - void* WindowFunctionType::getNullValueByType(int ct, int pos) { diff --git a/versioning/BRM/dbrm.cpp b/versioning/BRM/dbrm.cpp index a763bbd3f..984c4f9de 100644 --- a/versioning/BRM/dbrm.cpp +++ b/versioning/BRM/dbrm.cpp @@ -1884,7 +1884,7 @@ int DBRM::deletePartition(const std::vector& oids, std::set::const_iterator partIt; for (partIt = partitionNums.begin(); partIt != partitionNums.end(); ++partIt) - oss << (*it) << " " + oss << (*partIt) << " "; oss << "; OIDS: "; std::vector::const_iterator it; diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 63cf70092..5da10433e 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -2742,6 +2742,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, boost::scoped_array preBlkNums(new int[columnsUpdated.size()]); boost::scoped_array oids(new OID[columnsUpdated.size()]); + BRMWrapper::setUseVb(true); for (unsigned int j = 0; j < columnsUpdated.size(); j++) { //timer.start("lookupsyscat"); diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index 73340ff13..72d2c6429 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -178,97 +178,43 @@ const std::string Convertor::int2Str(int val) * converted long long for specified "field" ******************************************************************************/ /* static */ + long long Convertor::convertDecimalString( const char* field, - int fieldLength, - int scale ) + int fieldLength, + int scale) { - long long llVal = 0; + long double dval = strtold(field, NULL); + long long ret = 0; - int nDigitsBeforeDecPt = 0; - int nDigitsAfterDecPt = 0; - long long roundUp = 0; //@bug 3405 round off decimal column values + + // move scale digits to the left of the decimal point + for (int i = 0; i < scale; i++) + dval *= 10; - // Determine the number of digits before and after the decimal point - char* posDecPt = (char*)memchr(field, '.', fieldLength); - - if (posDecPt) + + // range check against int64 + if (dval > LLONG_MAX) { - nDigitsBeforeDecPt = posDecPt - field; - nDigitsAfterDecPt = fieldLength - nDigitsBeforeDecPt - 1; - - //@bug 3405 round off decimal column values - // We look at the scale+1 digit to see if we need to round up. - if (nDigitsAfterDecPt > scale) - { - char roundOffDigit = *(posDecPt + 1 + scale); - - if ( (roundOffDigit > '4') && - (roundOffDigit <= '9') ) // round up - { - roundUp = 1; - - // We can't just use the sign of llVal to determine whether to - // add +1 or -1, because if we read in -0.005 with scale 2, we - // end up parsing "-0.00", which yields 0; meaning we lose the - // sign. So better (though maybe slower) to look for any lead- - // ing negative sign in the input string. - for (int k = 0; k < fieldLength; k++) - { - if (!isspace(field[k])) - { - if (field[k] == '-') - roundUp = -1; - - break; - } - } - } - } + errno = ERANGE; + return LLONG_MAX; } - else + if (dval < LLONG_MIN) { - nDigitsBeforeDecPt = fieldLength; - nDigitsAfterDecPt = 0; + errno = ERANGE; + return LLONG_MIN; } - - // Strip out the decimal point by stringing together - // the digits before and after the decimal point. - char* data = (char*)alloca(nDigitsBeforeDecPt + scale + 1); - memcpy(data, field, nDigitsBeforeDecPt); - - if (nDigitsAfterDecPt) - { - if (scale > nDigitsAfterDecPt) - memcpy(data + nDigitsBeforeDecPt, - field + nDigitsBeforeDecPt + 1, - nDigitsAfterDecPt); - else // (scale <= nDigitsAfterDecPt) - memcpy(data + nDigitsBeforeDecPt, - field + nDigitsBeforeDecPt + 1, - scale); - } - - // Add on any necessary zero padding at the end - if (scale > nDigitsAfterDecPt) - { - memset(data + nDigitsBeforeDecPt + nDigitsAfterDecPt, - '0', - scale - nDigitsAfterDecPt); - } - - data[nDigitsBeforeDecPt + scale] = '\0'; - - // Convert our constructed decimal string back to a long long - //@bug 1814 Force strtoll to use base 10 errno = 0; - llVal = strtoll(data, 0, 10); - //@bug 3405 round off decimal values - if ((roundUp) && (errno == 0)) - llVal += roundUp; + ret = dval; - return llVal; + // get the fractional part of what's left & round ret up or down. + dval -= ret; + if (dval >= 0.5 && ret < LLONG_MAX) + ++ret; + else if (dval <= -0.5 && ret > LLONG_MIN) + --ret; + return ret; } /*******************************************************************************