diff --git a/VERSION b/VERSION index f5d54e457..ebe283cd3 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ COLUMNSTORE_VERSION_MAJOR=1 -COLUMNSTORE_VERSION_MINOR=3 -COLUMNSTORE_VERSION_PATCH=0 +COLUMNSTORE_VERSION_MINOR=2 +COLUMNSTORE_VERSION_PATCH=3 COLUMNSTORE_VERSION_RELEASE=1 diff --git a/dbcon/joblist/limitedorderby.cpp b/dbcon/joblist/limitedorderby.cpp index 7da1accfe..d5db83552 100644 --- a/dbcon/joblist/limitedorderby.cpp +++ b/dbcon/joblist/limitedorderby.cpp @@ -113,7 +113,6 @@ void LimitedOrderBy::processRow(const rowgroup::Row& row) if (fOrderByQueue.size() < fStart + fCount) { copyRow(row, &fRow0); - //memcpy(fRow0.getData(), row.getData(), row.getSize()); OrderByRow newRow(fRow0, fRule); fOrderByQueue.push(newRow); @@ -121,8 +120,6 @@ void LimitedOrderBy::processRow(const rowgroup::Row& row) if (fDistinct) fDistinctMap->insert(fRow0.getPointer()); - //fDistinctMap->insert(make_pair((fRow0.getData()+2), fRow0.getData())); - fRowGroup.incRowCount(); fRow0.nextRow(); @@ -150,23 +147,16 @@ void LimitedOrderBy::processRow(const rowgroup::Row& row) { OrderByRow swapRow = fOrderByQueue.top(); row1.setData(swapRow.fData); - fOrderByQueue.pop(); - if (!fDistinct) + copyRow(row, &row1); + + if (fDistinct) { - copyRow(row, &row1); - //memcpy(swapRow.fData, row.getData(), row.getSize()); - } - else - { - fDistinctMap->erase(row.getPointer()); - copyRow(row, &row1); + fDistinctMap->erase(fOrderByQueue.top().fData); fDistinctMap->insert(row1.getPointer()); - //fDistinctMap->erase(fDistinctMap->find(row.getData() + 2)); - //memcpy(swapRow.fData, row.getData(), row.getSize()); - //fDistinctMap->insert(make_pair((swapRow.fData+2), swapRow.fData)); } + fOrderByQueue.pop(); fOrderByQueue.push(swapRow); } } @@ -228,6 +218,7 @@ void LimitedOrderBy::finalize() const OrderByRow& topRow = fOrderByQueue.top(); row1.setData(topRow.fData); copyRow(row1, &fRow0); + //cerr << "LimitedOrderBy::finalize fRow0 " << fRow0.toString() << endl; fRowGroup.incRowCount(); offset--; fRow0.prevRow(rSize); diff --git a/dbcon/joblist/tupleaggregatestep.cpp b/dbcon/joblist/tupleaggregatestep.cpp index 9ddf42cc9..4c1893115 100644 --- a/dbcon/joblist/tupleaggregatestep.cpp +++ b/dbcon/joblist/tupleaggregatestep.cpp @@ -5554,6 +5554,8 @@ void TupleAggregateStep::threadedAggregateRowGroups(uint32_t threadID) rowIn.copyField(distRow[j], k, multiDist->subAggregators()[j]->getGroupByCols()[k].get()->fInputColumnIndex); } + // TBD This approach could potentiall + // put all values in on bucket. bucketID = distRow[j].hash(hashLens[j] - 1) % fNumOfBuckets; rowBucketVecs[bucketID][j].push_back(rowIn.getPointer()); rowIn.nextRow(); @@ -5572,6 +5574,8 @@ void TupleAggregateStep::threadedAggregateRowGroups(uint32_t threadID) for (uint64_t i = 0; i < fRowGroupIns[threadID].getRowCount(); ++i) { // The key is the groupby columns, which are the leading columns. + // TBD This approach could potentiall + // put all values in on bucket. int bucketID = rowIn.hash(hashLens[0] - 1) % fNumOfBuckets; rowBucketVecs[bucketID][0].push_back(rowIn.getPointer()); rowIn.nextRow(); diff --git a/oam/etc/ConsoleCmds.xml b/oam/etc/ConsoleCmds.xml index 56a6e678f..17cf0581d 100644 --- a/oam/etc/ConsoleCmds.xml +++ b/oam/etc/ConsoleCmds.xml @@ -309,8 +309,10 @@ MariaDB Columnstore Packages and setup the module to make it ready to be restored Required: Module-type or Module-name being added Required: Number-of-Modules being added when Module-type is specified - Optional: Server-Hostnames/Amazon-Instance-Names seperated by commas - Optional: Server-root-password + Required: Store hostnames in the configuration instead of IP addresses (y/n) + If not Amazon, then Required: hostnames separated by commas + If Amazon, then Optional: Amazon-Instance-Names separated by commas + Optional: Server-user-password removeModule diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index 539b48bbd..805c7d5d0 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -1272,10 +1272,12 @@ void Oam::setSystemConfig(const std::string module, ModuleConfig moduleconfig) * ********************************************************************/ -void Oam::addModule(DeviceNetworkList devicenetworklist, const std::string password, const std::string mysqlpw) +void Oam::addModule(DeviceNetworkList devicenetworklist, const std::string password, const std::string mysqlpw, + bool storeHostnames) { // build and send msg - int returnStatus = sendMsgToProcMgr2(ADDMODULE, devicenetworklist, FORCEFUL, ACK_YES, password, mysqlpw); + int returnStatus = sendAddModuleToProcMgr(ADDMODULE, devicenetworklist, FORCEFUL, ACK_YES, storeHostnames, + password, mysqlpw); if (returnStatus != API_SUCCESS) exceptionControl("addModule", returnStatus); @@ -10007,6 +10009,124 @@ int Oam::sendMsgToProcMgr2(messageqcpp::ByteStream::byte requestType, DeviceNetw return returnStatus; } +/* A slightly different version of sendMsgToProcMgr2. Add-module needs to send one add'l + parameter, and this was the best of a couple bad options. */ +int Oam::sendAddModuleToProcMgr(messageqcpp::ByteStream::byte requestType, DeviceNetworkList devicenetworklist, + GRACEFUL_FLAG gracefulflag, ACK_FLAG ackflag, bool storeHostnames, const std::string password, + const std::string mysqlpw) +{ + if (!checkSystemRunning()) + return API_CONN_REFUSED; + + int returnStatus = API_TIMEOUT; //default + ByteStream msg; + ByteStream receivedMSG; + ByteStream::byte msgType; + ByteStream::byte actionType; + ByteStream::byte status; + + // get current requesting process, an error will occur if process is a UI tool (not kept in Status Table) + // this will be used to determine if this is a manually or auto request down within Process-Monitor + bool requestManual; + myProcessStatus_t t; + + try + { + t = getMyProcessStatus(); + requestManual = false; // set to auto + } + catch (...) + { + requestManual = true; // set to manual + } + + // setup message + msg << (ByteStream::byte) REQUEST; + msg << requestType; + msg << (std::string) " "; + msg << (ByteStream::byte) gracefulflag; + msg << (ByteStream::byte) ackflag; + msg << (ByteStream::byte) requestManual; + msg << (uint8_t) storeHostnames; + msg << (uint16_t) devicenetworklist.size(); + + DeviceNetworkList::iterator pt = devicenetworklist.begin(); + + for ( ; pt != devicenetworklist.end() ; pt++) + { + msg << (*pt).DeviceName; + + if ( (*pt).UserTempDeviceName.empty() ) + msg << " "; + else + msg << (*pt).UserTempDeviceName; + + if ( (*pt).DisableState.empty() ) + msg << " "; + else + msg << (*pt).DisableState; + + msg << (uint16_t) (*pt).hostConfigList.size(); + + HostConfigList::iterator pt1 = (*pt).hostConfigList.begin(); + + for ( ; pt1 != (*pt).hostConfigList.end() ; pt1++) + { + msg << (*pt1).IPAddr; + msg << (*pt1).HostName; + msg << (*pt1).NicID; + } + } + + msg << password; + msg << mysqlpw; + + try + { + //send the msg to Process Manager + MessageQueueClient procmgr("ProcMgr"); + procmgr.write(msg); + + // check for Ack msg if needed + if ( ackflag == ACK_YES ) + { + // wait 15 minutes for ACK from Process Manager + struct timespec ts = { 900, 0 }; + + receivedMSG = procmgr.read(&ts); + + if (receivedMSG.length() > 0) + { + receivedMSG >> msgType; + receivedMSG >> actionType; + receivedMSG >> status; + + if ( msgType == oam::ACK && actionType == requestType) + { + // ACK for this request + returnStatus = status; + } + } + else // timeout + returnStatus = API_TIMEOUT; + } + else + // No ACK, assume write success + returnStatus = API_SUCCESS; + + // shutdown connection + procmgr.shutdown(); + } + catch (...) + { + returnStatus = API_FAILURE; + } + + return returnStatus; +} + + + /*************************************************************************** * * Function: sendMsgToProcMgr3 diff --git a/oam/oamcpp/liboamcpp.h b/oam/oamcpp/liboamcpp.h index 39e2a51de..5d5a09c6c 100644 --- a/oam/oamcpp/liboamcpp.h +++ b/oam/oamcpp/liboamcpp.h @@ -1488,7 +1488,8 @@ public: * @param DeviceNetworkConfig the Modules added * @param password Host Root Password */ - EXPORT void addModule(DeviceNetworkList devicenetworklist, const std::string password, const std::string mysqlpw); + EXPORT void addModule(DeviceNetworkList devicenetworklist, const std::string password, const std::string mysqlpw, + bool storeHostnames); /** @brief remove Module * @@ -2503,6 +2504,12 @@ private: int sendMsgToProcMgr2(messageqcpp::ByteStream::byte requestType, DeviceNetworkList devicenetworklist, GRACEFUL_FLAG gracefulflag, ACK_FLAG ackflag, const std::string password = oam::UnassignedName, const std::string mysqlpw = oam::UnassignedName); + /** @brief a slightly different version of sendMsgToProcMgr2, which is for addmodule only. + */ + int sendAddModuleToProcMgr(messageqcpp::ByteStream::byte requestType, DeviceNetworkList devicenetworklist, + GRACEFUL_FLAG gracefulflag, ACK_FLAG ackflag, bool storeHostnames, const std::string password = oam::UnassignedName, + const std::string mysqlpw = oam::UnassignedName); + /** @brief build and send request message to Process Manager * Check for status messages */ diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index cf28b15bb..68355980a 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -5660,6 +5660,7 @@ int processCommand(string* arguments) DeviceNetworkList enabledevicenetworklist; HostConfig hostconfig; + bool storeHostnames = false; string moduleType; string moduleName; int moduleCount; @@ -5670,7 +5671,7 @@ int processCommand(string* arguments) umStorageNames umstoragenames; int hostArg; int dbrootPerPM = 0; - + //check if module type or module name was entered if ( arguments[1].size() == 2 ) { @@ -5685,8 +5686,47 @@ int processCommand(string* arguments) //Module Type was entered moduleType = arguments[1]; moduleCount = atoi(arguments[2].c_str()); + hostArg = 4; + + // MCOL-1607. Check whether we should store host names or IP addresses. + if (arguments[3] != "" && (arguments[3][0] == 'y' || arguments[3][0] == 'Y')) + storeHostnames = true; + + //check for a non-distrubuted install setup, dont need password + if ( DistributedInstall != "y" ) + { + if (arguments[5] != "") + password = arguments[5]; + else + { + cout << endl; + string prompt = "Enter the 'User' Password or 'ssh' if configured with ssh-keys"; + password = dataPrompt(prompt); + } + } + + if (arguments[6] != "") + dbrootPerPM = atoi(arguments[6].c_str()); + } + else + { + //Module Name was entered + if (arguments[2] == "" && cloud == oam::UnassignedName) + { + // need at least arguments + cout << endl << "**** addModule Failed : Missing a required Parameter, enter 'help' for additional information" << endl; + break; + } + + moduleName = arguments[1]; + moduleType = arguments[1].substr(0, MAX_MODULE_TYPE_SIZE); + moduleCount = 1; hostArg = 3; + // MCOL-1607. Check whether we should store host names or IP addresses. + if (arguments[2] != "" && (arguments[2][0] == 'y' || arguments[2][0] == 'Y')) + storeHostnames = true; + //check for a non-distrubuted install setup, dont need password if ( DistributedInstall != "y" ) { @@ -5703,37 +5743,6 @@ int processCommand(string* arguments) if (arguments[5] != "") dbrootPerPM = atoi(arguments[5].c_str()); } - else - { - //Module Name was entered - if (arguments[2] == "" && cloud == oam::UnassignedName) - { - // need at least arguments - cout << endl << "**** addModule Failed : Missing a required Parameter, enter 'help' for additional information" << endl; - break; - } - - moduleName = arguments[1]; - moduleType = arguments[1].substr(0, MAX_MODULE_TYPE_SIZE); - moduleCount = 1; - hostArg = 2; - - //check for a non-distrubuted install setup, dont need password - if ( DistributedInstall != "y" ) - { - if (arguments[3] != "") - password = arguments[3]; - else - { - cout << endl; - string prompt = "Enter the 'User' Password or 'ssh' if configured with ssh-keys"; - password = dataPrompt(prompt); - } - } - - if (arguments[4] != "") - dbrootPerPM = atoi(arguments[4].c_str()); - } //do we needed this check???? if ( moduleCount < 1 || moduleCount > 10 ) @@ -5925,7 +5934,24 @@ int processCommand(string* arguments) string hostName; string IPAddress; - if ( cloud == "amazon-ec2") + // MCOL-1607. Store hostnames in the config file if they entered one */ + if (storeHostnames) + { + // special case + if (cloud == "amazon-vpc" && *listPT1 == "autoassign") + { + hostName = oam::UnassignedName; + IPAddress = *listPT1; + } + else if (oam.isValidIP(*listPT1)) // they entered an IP addr + { + hostName = oam::UnassignedName; + IPAddress = *listPT1; + } + else // they entered a hostname + IPAddress = hostName = *listPT1; + } + else if ( cloud == "amazon-ec2") { hostName = *listPT1; @@ -5978,14 +6004,13 @@ int processCommand(string* arguments) // non-amazon hostName = *listPT1; IPAddress = oam.getIPAddress(hostName); - if ( IPAddress.empty() ) { // prompt for IP Address string prompt = "IP Address of " + hostName + " not found, enter IP Address or enter 'abort'"; IPAddress = dataPrompt(prompt); - - if ( IPAddress == "abort" || !oam.isValidIP(IPAddress)) + + if ( IPAddress == "abort" || !oam.isValidIP(IPAddress) ) return 1; } } @@ -6023,18 +6048,22 @@ int processCommand(string* arguments) { string prompt = "DataRedundancy is configured for dedicated network, enter a hostname"; DataRedundancyHostname = dataPrompt(prompt); - DataRedundancyIPAddress = oam.getIPAddress(DataRedundancyHostname); - - if ( DataRedundancyIPAddress.empty() ) + if (storeHostnames) + DataRedundancyIPAddress = DataRedundancyHostname; + else { - // prompt for IP Address - string prompt = "IP Address of " + DataRedundancyHostname + " not found, enter IP Address"; - DataRedundancyIPAddress = dataPrompt(prompt); + DataRedundancyIPAddress = oam.getIPAddress(DataRedundancyHostname); - if (!oam.isValidIP(DataRedundancyIPAddress)) - return 1; + if ( DataRedundancyIPAddress.empty() ) + { + // prompt for IP Address + string prompt = "IP Address of " + DataRedundancyHostname + " not found, enter IP Address"; + DataRedundancyIPAddress = dataPrompt(prompt); + + if (!oam.isValidIP(DataRedundancyIPAddress)) + return 1; + } } - sysConfig->setConfig("DataRedundancyConfig", dataDupHostName, DataRedundancyHostname); sysConfig->setConfig("DataRedundancyConfig", dataDupIPaddr, DataRedundancyIPAddress); } @@ -6105,7 +6134,7 @@ int processCommand(string* arguments) cout << "please wait..." << endl; - oam.addModule(devicenetworklist, password, mysqlpassword); + oam.addModule(devicenetworklist, password, mysqlpassword, storeHostnames); cout << "Add Module(s) successfully completed" << endl; diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 94850deae..81b69e548 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -126,10 +126,22 @@ int main(int argc, char* argv[]) mysqlpw = argv[10]; installer_debug = argv[11]; - if (argc >= 13) + string numBlocksPctParam = ""; + string totalUmMemoryParam = ""; + if (argc >= 13) { installDir = argv[12]; - else + if (argc >= 15) { + if (string(argv[13]) != "-") { + numBlocksPctParam = argv[13]; + } + if (string(argv[14]) != "-") { + totalUmMemoryParam = argv[14]; + } + } + } + else { installDir = "/usr/local/mariadb/columnstore"; + } ofstream file("/dev/null"); @@ -151,6 +163,12 @@ int main(int argc, char* argv[]) cout << mysqlpw << endl; cout << installer_debug << endl; cout << installDir << endl; + if (!numBlocksPctParam.empty()) { + cout << numBlocksPctParam << endl; + } + if (!totalUmMemoryParam.empty()) { + cout << totalUmMemoryParam << endl; + } // restore cout stream buffer cout.rdbuf (strm_buffer); @@ -255,16 +273,26 @@ int main(int argc, char* argv[]) return false; } - string numBlocksPct = "50"; - - if ( DBRootStorageType == "hdfs") - numBlocksPct = "25"; + string numBlocksPct; + if (numBlocksPctParam.empty()) { + numBlocksPct = "50"; + if (DBRootStorageType == "hdfs") + numBlocksPct = "25"; + } + else { + numBlocksPct = numBlocksPctParam; + } try { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + } } catch (...) { @@ -282,15 +310,21 @@ int main(int argc, char* argv[]) // long long total = myinfo.totalram / 1024 / 1000; // adjust max memory, 25% of total memory - string percent = "25%"; + string percent; + if (totalUmMemoryParam.empty()) { + percent = "25%"; - if ( DBRootStorageType == "hdfs") - { - percent = "12%"; + if (DBRootStorageType == "hdfs") + { + percent = "12%"; + } + + cout << " Setting 'TotalUmMemory' to " << percent << " of total memory. " << endl; + } + else { + percent = totalUmMemoryParam; + cout << " Setting 'TotalUmMemory' to " << percent << endl; } - - cout << " Setting 'TotalUmMemory' to " << percent << " of total memory. " << endl; - try { sysConfig->setConfig("HashJoin", "TotalUmMemory", percent); @@ -305,29 +339,47 @@ int main(int argc, char* argv[]) { try { - string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); - string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + string numBlocksPct; + string totalUmMemory; - if (numBlocksPct.empty() || numBlocksPct == "" ) - { - numBlocksPct = "50"; - - try - { - sysConfig->setConfig("DBBC", "NumBlocksPct", "50"); - - cout << endl << "NOTE: Setting 'NumBlocksPct' to 50%" << endl; - } - catch (...) - { - cout << "ERROR: Problem setting NumBlocksPct in the MariaDB ColumnStore System Configuration file" << endl; - exit(1); - } + if (!numBlocksPctParam.empty()) { + numBlocksPct = numBlocksPctParam; + } + else { + numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + } + if (!totalUmMemoryParam.empty()) { + totalUmMemory = totalUmMemoryParam; + } + else { + totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); } - cout << endl << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; - cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; + if (numBlocksPct.empty() || numBlocksPct == "") + { + numBlocksPct = "50"; + } + if (totalUmMemory.empty() || totalUmMemory == "") { + totalUmMemory = "25%"; + } + try + { + sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); + sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemory); + if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + } + cout << " Setting 'TotalUmMemory' to " << totalUmMemory << endl; + } + catch (...) + { + cout << "ERROR: Problem setting NumBlocksPct/TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } } catch (...) { @@ -350,7 +402,23 @@ int main(int argc, char* argv[]) // are we using settings from previous config file? if ( reuseConfig == "n" ) { - cout << endl << "NOTE: Using the default setting for 'NumBlocksPct' at 70%" << endl; + string numBlocksPct = "70"; + string totalUmMemory = "50%"; + + if (!numBlocksPctParam.empty()) { + numBlocksPct = numBlocksPctParam; + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Using the default setting for 'NumBlocksPct' at " << numBlocksPct << "%" << endl; + } + if (!totalUmMemoryParam.empty()) { + totalUmMemory = totalUmMemoryParam; + cout << endl << "Setting 'TotalUmMemory' to " << totalUmMemory << endl; + } + else { + cout << endl << "Setting 'TotalUmMemory' to " << totalUmMemory << " of total memory." << endl; + } try { @@ -358,17 +426,14 @@ int main(int argc, char* argv[]) } catch (...) {} - // adjust max memory, 50% of total memory - - cout << endl << "Setting 'TotalUmMemory' to 50% of total memory." << endl; - try { - sysConfig->setConfig("HashJoin", "TotalUmMemory", "50%"); + sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); + sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemory); } catch (...) { - cout << "ERROR: Problem setting TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl; + cout << "ERROR: Problem setting TotalUmMemory/NumBlocksPct in the MariaDB ColumnStore System Configuration file" << endl; exit(1); } } @@ -376,15 +441,47 @@ int main(int argc, char* argv[]) { try { - string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); - string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + string numBlocksPct; + string totalUmMemory; - if (numBlocksPct.empty() || numBlocksPct == "" ) + if (!numBlocksPctParam.empty()) { + numBlocksPct = numBlocksPctParam; + } + else { + numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + } + if (!totalUmMemoryParam.empty()) { + totalUmMemory = totalUmMemoryParam; + } + else { + totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + } + + if (numBlocksPct.empty() || numBlocksPct == "") + { numBlocksPct = "70"; + } + if (totalUmMemory.empty() || totalUmMemory == "") { + totalUmMemory = "50%"; + } + try + { + sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); + sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemory); - cout << endl << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; - cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; - + if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + } + cout << " Setting 'TotalUmMemory' to " << totalUmMemory << endl; + } + catch (...) + { + cout << "ERROR: Problem setting NumBlocksPct/TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } } catch (...) { diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index bb828b5be..c92697127 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -14,7 +14,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - MA 02110-1301, USA. */ + MA 02110-1301, USA. */ /****************************************************************************************** * $Id: postConfigure.cpp 64 2006-10-12 22:21:51Z dhill $ @@ -129,7 +129,7 @@ void singleServerConfigSetup(Config* sysConfig); void remoteInstallThread(void*); -bool glusterSetup(string password); +bool glusterSetup(string password, bool doNotResolveHostNames); std::string launchInstance(ModuleIP moduleip); @@ -192,6 +192,7 @@ bool nonDistributeFlag = false; bool single_server_quick_install = false; bool multi_server_quick_install = false; bool amazon_quick_install = false; +bool doNotResolveHostNames = false; string DataFileEnvFile; @@ -230,6 +231,8 @@ int main(int argc, char* argv[]) string cmd; string pmIpAddrs = ""; string umIpAddrs = ""; + string numBlocksPctParam = ""; + string totalUmMemoryParam = ""; // struct sysinfo myinfo; @@ -275,12 +278,12 @@ int main(int argc, char* argv[]) usergroup = getgid(); string SUDO = ""; - if (user != 0) + if (user != 0) { rootUser = false; SUDO = "sudo "; } - + char* p = getenv("USER"); if (p && *p) @@ -298,7 +301,7 @@ int main(int argc, char* argv[]) for ( int i = 1; i < argc; i++ ) { - if( string("-h") == argv[i] ) + if( string("-h") == argv[i] ) { cout << endl; cout << "This is the MariaDB ColumnStore System Configuration and Installation tool." << endl; @@ -313,7 +316,7 @@ int main(int argc, char* argv[]) cout << " Enter one of the options within [], if available, or" << endl; cout << " Enter a new value" << endl << endl; cout << endl; - cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count]" << endl; + cout << "Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-numBlocksPct][-totalUmMemory]" << endl; cout << " -h Help" << endl; cout << " -c Config File to use to extract configuration data, default is Columnstore.xml.rpmsave" << endl; cout << " -u Upgrade, Install using the Config File from -c, default to Columnstore.xml.rpmsave" << endl; @@ -322,13 +325,22 @@ int main(int argc, char* argv[]) cout << " -qs Quick Install - Single Server" << endl; cout << " -qm Quick Install - Multi Server" << endl; cout << " -port MariaDB ColumnStore Port Address" << endl; - cout << " -i Non-root Install directory, Only use for non-root installs" << endl; - cout << " -n Non-distributed install, meaning postConfigure will not install packages on remote nodes" << endl; - cout << " -d Distributed install, meaning postConfigure will install packages on remote nodes" << endl; + cout << " -i Non-root Install directory, Only use for non-root installs" << endl; + cout << " -n Non-distributed install, meaning postConfigure will not install packages on remote nodes" << endl; + cout << " -d Distributed install, meaning postConfigure will install packages on remote nodes" << endl; cout << " -sn System Name" << endl; cout << " -pm-ip-addrs Performance Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; cout << " -um-ip-addrs User Module IP Addresses xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx" << endl; + cout << " -x Do not resolve IP Addresses from host names" << endl; + cout << " -numBlocksPct amount of physical memory to utilize for disk block caching" << endl; + cout << " (percentages of the total memory need to be stated without suffix, explcit values with suffixes M or G)" << endl; + cout << " -totalUmMemory amount of physical memory to utilize for joins, intermediate results and set operations on the UM" << endl; + cout << " (percentages of the total memory need to be stated with suffix %, explcit values with suffixes M or G)" << endl; exit (0); + } + else if (string("-x") == argv[i]) + { + doNotResolveHostNames = true; } else if( string("-qs") == argv[i] ) { @@ -349,33 +361,33 @@ int main(int argc, char* argv[]) nodeps = "--nodeps"; else if ( string("-o") == argv[i] ) startOfflinePrompt = true; - else if( string("-c") == argv[i] ) + else if( string("-c") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: Config File not provided" << endl; exit (1); } oldFileName = argv[i]; - if ( oldFileName.find("Columnstore.xml") == string::npos ) + if ( oldFileName.find("Columnstore.xml") == string::npos ) { cout << " ERROR: Config File is not a Columnstore.xml file name" << endl; exit (1); } } - else if( string("-p") == argv[i] ) + else if( string("-p") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: Password not provided" << endl; exit (1); } password = argv[i]; - if ( password.find("-") != string::npos ) + if ( password.find("-") != string::npos ) { cout << " ERROR: Valid Password not provided" << endl; exit (1); @@ -394,10 +406,10 @@ int main(int argc, char* argv[]) nonDistribute = false; nonDistributeFlag = true; } - else if( string("-port") == argv[i] ) + else if( string("-port") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: MariaDB ColumnStore Port ID not supplied" << endl; exit (1); @@ -411,10 +423,10 @@ int main(int argc, char* argv[]) exit (1); } } - else if( string("-i") == argv[i] ) + else if( string("-i") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: Path not provided" << endl; exit (1); @@ -422,64 +434,94 @@ int main(int argc, char* argv[]) installDir = argv[i]; } - else if( string("-sn") == argv[i] ) + else if( string("-sn") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: System-name not provided" << endl; exit (1); } systemName = argv[i]; } - else if( string("-pm-ip-addrs") == argv[i] ) + else if( string("-pm-ip-addrs") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: PM-IP-ADRESSES not provided" << endl; exit (1); } pmIpAddrs = argv[i]; } - else if( string("-um-ip-addrs") == argv[i] ) + else if( string("-um-ip-addrs") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: UM-IP-ADRESSES not provided" << endl; exit (1); } umIpAddrs = argv[i]; } - else if( string("-pm-count") == argv[i] ) + else if( string("-pm-count") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: PM-COUNT not provided" << endl; exit (1); } pmNumber = atoi(argv[i]); } - else if( string("-um-count") == argv[i] ) + else if( string("-um-count") == argv[i] ) { i++; - if (i >= argc ) + if (i >= argc ) { cout << " ERROR: UM-COUNT not provided" << endl; exit (1); } umNumber = atoi(argv[i]); } + else if ( string("-numBlocksPct") == argv[i] ) + { + i++; + if (i >= argc) + { + cout << " ERROR: Memory settings for numBlocksPct not provided" << endl; + exit(1); + } + numBlocksPctParam = argv[i]; + // check that the parameter ends with a number M or G + if (!(isdigit(*numBlocksPctParam.rbegin()) || *numBlocksPctParam.rbegin() == 'M' || *numBlocksPctParam.rbegin() == 'G')) { + cout << " ERROR: Memory settings for numBlocksPct need to end on a digit, M or G" << endl; + exit(1); + } + } + else if (string("-totalUmMemory") == argv[i]) + { + i++; + if (i >= argc) + { + cout << " ERROR: Memory settings for totalUmMemory not provided" << endl; + exit(1); + } + totalUmMemoryParam = argv[i]; + // check that the parameter ends with a %, M, or G + if (!(*totalUmMemoryParam.rbegin() == '%' || *totalUmMemoryParam.rbegin() == 'M' || *totalUmMemoryParam.rbegin() == 'G')) { + cout << " ERROR: Memory settings for totalUmMemory need to end on %, M or G" << endl; + exit(1); + } + } else { cout << " ERROR: Invalid Argument = " << argv[i] << endl; - cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count]" << endl; + cout << " Usage: postConfigure [-h][-c][-u][-p][-qs][-qm][-qa][-port][-i][-n][-d][-sn][-pm-ip-addrs][-um-ip-addrs][-pm-count][-um-count][-numBlocksPct][-totalUmMemory]" << endl; exit (1); } } - + //check if quick install multi-server has been given ip address if (multi_server_quick_install) { @@ -553,7 +595,7 @@ int main(int argc, char* argv[]) // redirectStandardOutputToFile(postConfigureLog, false ); } } - + //check if MariaDB ColumnStore is up and running if (oam.checkSystemRunning()) { @@ -567,7 +609,7 @@ int main(int argc, char* argv[]) cout << "ERROR: Configuration File not setup" << endl; exit(1); } - + //determine package type string EEPackageType; @@ -605,7 +647,7 @@ int main(int argc, char* argv[]) cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); } - + //check for local ip address as pm1 ModuleConfig moduleconfig; @@ -789,7 +831,7 @@ int main(int argc, char* argv[]) } cout << endl; - + if (single_server_quick_install) { cout << "===== Quick Install Single-Server Configuration =====" << endl << endl; @@ -797,13 +839,13 @@ int main(int argc, char* argv[]) cout << "Single-Server install is used when there will only be 1 server configured" << endl; cout << "on the system. It can also be used for production systems, if the plan is" << endl; cout << "to stay single-server." << endl; - + singleServerInstall = "1"; } else if (multi_server_quick_install) { cout << "===== Quick Install Multi-Server Configuration =====" << endl << endl; - + cout << "Multi-Server install defaulting to using local storage" << endl; singleServerInstall = "2"; @@ -811,7 +853,7 @@ int main(int argc, char* argv[]) else if (amazon_quick_install) { cout << "===== Quick Install Amazon Configuration =====" << endl << endl; - + cout << "Amazon AMI EC2 install defaulting to using local storage" << endl; singleServerInstall = "2"; @@ -842,7 +884,7 @@ int main(int argc, char* argv[]) else singleServerInstall = "2"; - while(true) + while(true) { string temp = singleServerInstall; prompt = "Select the type of System Server install [1=single, 2=multi] (" + singleServerInstall + ") > "; @@ -889,7 +931,7 @@ int main(int argc, char* argv[]) //setup to Columnstore.xml file for single server singleServerConfigSetup(sysConfig); } - + //module ProcessConfig.xml to setup all apps on the pm if ( !updateProcessConfig() ) cout << "Update ProcessConfig.xml error" << endl; @@ -952,7 +994,6 @@ int main(int argc, char* argv[]) offLineAppCheck(); checkMysqlPort(mysqlPort, sysConfig); - if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; @@ -961,13 +1002,20 @@ int main(int argc, char* argv[]) cout << endl << "===== Performing Configuration Setup and MariaDB ColumnStore Startup =====" << endl; - cmd = installDir + "/bin/installer dummy.rpm dummy.rpm dummy.rpm dummy.rpm dummy.rpm initial dummy " + reuseConfig + " --nodeps ' ' 1 " + installDir; + if (numBlocksPctParam.empty()) { + numBlocksPctParam = "-"; + } + if (totalUmMemoryParam.empty()) { + totalUmMemoryParam = "-"; + } + + cmd = installDir + "/bin/installer dummy.rpm dummy.rpm dummy.rpm dummy.rpm dummy.rpm initial dummy " + reuseConfig + " --nodeps ' ' 1 " + installDir + " " + numBlocksPctParam + " " + totalUmMemoryParam; system(cmd.c_str()); exit(0); } // perform multi-node install - + try { sysConfig->setConfig(InstallSection, "SingleServerInstall", "n"); @@ -987,7 +1035,7 @@ int main(int argc, char* argv[]) // // Multi-server install // - + ModuleIP InputModuleIP; ModuleIpList InputModuleIPList; @@ -1001,7 +1049,7 @@ int main(int argc, char* argv[]) } catch(...) {} - + if (umIpAddrs == "" ) { // set Server Type Installation to combined @@ -1026,10 +1074,10 @@ int main(int argc, char* argv[]) InputModuleIP.moduleName = module; InputModuleIPList.push_back(InputModuleIP); } - + umNumber = id-1; } - + if (pmIpAddrs != "" ) { int id = 1; @@ -1045,16 +1093,16 @@ int main(int argc, char* argv[]) InputModuleIP.moduleName = module; InputModuleIPList.push_back(InputModuleIP); } - + pmNumber = id-1; } - if ( !writeConfig(sysConfig) ) + if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); } - + MaxNicID = 1; } else @@ -1067,13 +1115,13 @@ int main(int argc, char* argv[]) } catch(...) {} - + try { sysConfig->setConfig(InstallSection, "Cloud", "amazon-vpc"); } catch(...) {} - + if (umNumber == 0 ) { // set Server Type Installation to combined @@ -1084,12 +1132,12 @@ int main(int argc, char* argv[]) {} } - if ( !writeConfig(sysConfig) ) + if ( !writeConfig(sysConfig) ) { cout << "ERROR: Failed trying to update MariaDB ColumnStore System Configuration file" << endl; exit(1); } - + MaxNicID = 1; } } @@ -1350,7 +1398,7 @@ int main(int argc, char* argv[]) //amazon install setup check bool amazonInstall = false; string cloud = oam::UnassignedName; - + if (!multi_server_quick_install) { string amazonLog = tmpDir + "/amazon.log"; @@ -1367,7 +1415,7 @@ int main(int argc, char* argv[]) // not running on amazon with ec2-api-tools if (amazon_quick_install) { - cout << "ERROR: Amazon Quick Installer was specified, bu the Amazon CLI API packages isnt installed, exiting" << endl; + cout << "ERROR: Amazon Quick Installer was specified, but the Amazon CLI API packages is not installed, exiting" << endl; exit(1); } @@ -1380,7 +1428,7 @@ int main(int argc, char* argv[]) // not running on amazon with ec2-api-tools if (amazon_quick_install) { - cout << "ERROR: Amazon Quick Installer was specified, bu the AMazon CLI API packages isnt installed, exiting" << endl; + cout << "ERROR: Amazon Quick Installer was specified, but the Amazon CLI API packages is not installed, exiting" << endl; exit(1); } @@ -1390,7 +1438,7 @@ int main(int argc, char* argv[]) amazonInstall = true; } } - + try { cloud = sysConfig->getConfig(InstallSection, "Cloud"); @@ -1623,36 +1671,48 @@ int main(int argc, char* argv[]) case (oam::INSTALL_COMBINE_DM_UM_PM): // combined #1 - dm/um/pm on a single server { // are we using settings from previous config file? - if ( reuseConfig == "n" ) + if (reuseConfig == "n") { - if ( !uncommentCalpontXml("NumBlocksPct") ) - { - cout << "Update Columnstore.xml NumBlocksPct Section" << endl; - exit(1); - } string numBlocksPct; - try - { - numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + // if numBlocksPct was set as command line parameter use the command line parameter value + if (!numBlocksPctParam.empty()) { + numBlocksPct = numBlocksPctParam; } - catch (...) - {} + else { + if (!uncommentCalpontXml("NumBlocksPct")) + { + cout << "Update Columnstore.xml NumBlocksPct Section" << endl; + exit(1); + } - if ( numBlocksPct == "70" || numBlocksPct.empty() ) - { - numBlocksPct = "50"; + try + { + numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + } + catch (...) + { + } - if (hdfs) - numBlocksPct = "25"; + if (numBlocksPct == "70" || numBlocksPct.empty()) + { + numBlocksPct = "50"; + + if (hdfs) + numBlocksPct = "25"; + } } - try { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + } } catch (...) { @@ -1660,11 +1720,20 @@ int main(int argc, char* argv[]) exit(1); } - string percent = "25%"; + + string percent; - if (hdfs) - { - percent = "12%"; + if (!totalUmMemoryParam.empty()) { // if totalUmMemory was set as command line parameter use the command line parameter value + percent = totalUmMemoryParam; + } + else { //otherwise use reasonable defaults + + percent = "25%"; + + if (hdfs) + { + percent = "12%"; + } } cout << " Setting 'TotalUmMemory' to " << percent << endl; @@ -1689,22 +1758,38 @@ int main(int argc, char* argv[]) { try { - string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); - cout << endl; - if ( numBlocksPct.empty() ) - cout << "NOTE: Using the default setting for 'NumBlocksPct' at 70%" << endl; - else - cout << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; + if (!numBlocksPctParam.empty()) { // if numBlocksPct was set as command line parameter use the command line parameter value + sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPctParam); + if (*numBlocksPctParam.rbegin() == 'M' || *numBlocksPctParam.rbegin() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << "%" << endl; + } + } + else { //otherwise use the settings from the previous config file + string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); - string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + if (numBlocksPct.empty()) + cout << "NOTE: Using the default setting for 'NumBlocksPct' at 70%" << endl; + else + cout << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; + } - cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; + if (!totalUmMemoryParam.empty()) { // if totalUmMemory was set as command line parameter use the command line parameter value + sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemoryParam); + cout << " Setting 'TotalUmMemory' to " << totalUmMemoryParam << endl; + } + else { //otherwise use the settings from the previous config file + string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; + } } catch (...) { - cout << "ERROR: Problem reading NumBlocksPct/TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl; + cout << "ERROR: Problem reading/writing NumBlocksPct/TotalUmMemory in/to the MariaDB ColumnStore System Configuration file" << endl; exit(1); } } @@ -1717,35 +1802,39 @@ int main(int argc, char* argv[]) // are we using settings from previous config file? if ( reuseConfig == "n" ) { - - - - - - - string numBlocksPct; - try - { - numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + // if numBlocksPct was set as command line parameter use the command line parameter value + if (!numBlocksPctParam.empty()) { + numBlocksPct = numBlocksPctParam; } - catch (...) - {} + else { + try + { + numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + } + catch (...) + { + } - if ( numBlocksPct.empty() ) - { - numBlocksPct = "70"; + if (numBlocksPct.empty()) + { + numBlocksPct = "70"; - if (hdfs) - numBlocksPct = "35"; + if (hdfs) + numBlocksPct = "35"; + } } try { sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPct); - - cout << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + if (*numBlocksPct.rbegin() == 'M' || *numBlocksPct.rbegin() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPct << "%" << endl; + } } catch (...) { @@ -1753,11 +1842,18 @@ int main(int argc, char* argv[]) exit(1); } - string percent = "50%"; + string percent; - if (hdfs) - { - percent = "25%"; + if (!totalUmMemoryParam.empty()) { // if totalUmMemory was set as command line parameter use the command line parameter value + percent = totalUmMemoryParam; + } + else { //otherwise use reasonable defaults + percent = "50%"; + + if (hdfs) + { + percent = "25%"; + } } cout << " Setting 'TotalUmMemory' to " << percent << endl; @@ -1782,20 +1878,36 @@ int main(int argc, char* argv[]) { try { - string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); + if (!numBlocksPctParam.empty()) { // if numBlocksPct was set as command line parameter use the command line parameter value + sysConfig->setConfig("DBBC", "NumBlocksPct", numBlocksPctParam); + if (*numBlocksPctParam.rbegin() == 'M' || *numBlocksPctParam.rbegin() == 'G') { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << endl; + } + else { + cout << endl << "NOTE: Setting 'NumBlocksPct' to " << numBlocksPctParam << "%" << endl; + } + } + else { //otherwise use the settings from the previous config file + string numBlocksPct = sysConfig->getConfig("DBBC", "NumBlocksPct"); - if ( numBlocksPct.empty() ) - cout << "NOTE: Using the default setting for 'NumBlocksPct' at 70%" << endl; - else - cout << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; + if (numBlocksPct.empty()) + cout << "NOTE: Using the default setting for 'NumBlocksPct' at 70%" << endl; + else + cout << "NOTE: Using previous configuration setting for 'NumBlocksPct' = " << numBlocksPct << "%" << endl; + } - string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); - - cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; + if (!totalUmMemoryParam.empty()) { // if totalUmMemory was set as command line parameter use the command line parameter value + sysConfig->setConfig("HashJoin", "TotalUmMemory", totalUmMemoryParam); + cout << " Setting 'TotalUmMemory' to " << totalUmMemoryParam << endl; + } + else { //otherwise use reasonable defaults + string totalUmMemory = sysConfig->getConfig("HashJoin", "TotalUmMemory"); + cout << " Using previous configuration setting for 'TotalUmMemory' = " << totalUmMemory << endl; + } } catch (...) { - cout << "ERROR: Problem reading NumBlocksPct/TotalUmMemory in the MariaDB ColumnStore System Configuration file" << endl; + cout << "ERROR: Problem reading/writing NumBlocksPct/TotalUmMemory in/to the MariaDB ColumnStore System Configuration file" << endl; exit(1); } } @@ -1877,14 +1989,14 @@ int main(int argc, char* argv[]) catch (...) {} } - + if ( moduleType == "um") if ( umNumber != 0 ) moduleCount = umNumber; - + if ( moduleType == "pm") if ( pmNumber != 0 ) - moduleCount = pmNumber; + moduleCount = pmNumber; //verify and setup of modules count switch ( IserverTypeInstall ) @@ -2044,7 +2156,7 @@ int main(int argc, char* argv[]) moduleDisableState = oam::ENABLEDSTATE; //setup HostName/IPAddress for each NIC - + string moduleHostName = oam::UnassignedName; string moduleIPAddr = oam::UnassignedIpAddr; @@ -2063,7 +2175,7 @@ int main(int argc, char* argv[]) } } } - + unsigned int nicID=1; for( ; nicID < MaxNicID +1 ; nicID++ ) { @@ -2076,9 +2188,9 @@ int main(int argc, char* argv[]) for( ; listPT != sysModuleTypeConfig.moduletypeconfig[i].ModuleNetworkList.end() ; listPT++) { - if (newModuleName == (*listPT).DeviceName) + if (newModuleName == (*listPT).DeviceName) { - if ( nicID == 1 ) + if ( nicID == 1 ) { moduleDisableState = (*listPT).DisableState; @@ -2091,7 +2203,7 @@ int main(int argc, char* argv[]) for( ; pt1 != (*listPT).hostConfigList.end() ; pt1++) { - if ((*pt1).NicID == nicID) + if ((*pt1).NicID == nicID) { moduleHostName = (*pt1).HostName; moduleIPAddr = (*pt1).IPAddr; @@ -2103,8 +2215,8 @@ int main(int argc, char* argv[]) } } } - - if ( nicID == 1 ) + + if ( nicID == 1 ) { if ( moduleDisableState != oam::ENABLEDSTATE ) { @@ -2452,7 +2564,11 @@ int main(int argc, char* argv[]) if ( moduleIPAddr == oam::UnassignedIpAddr ) { //get IP Address - string IPAddress = oam.getIPAddress( newModuleHostName); + string IPAddress; + if (doNotResolveHostNames) + IPAddress = newModuleHostName; + else + IPAddress = oam.getIPAddress( newModuleHostName); if ( !IPAddress.empty() ) newModuleIPAddr = IPAddress; @@ -2468,7 +2584,7 @@ int main(int argc, char* argv[]) //prompt for IP address while (true) { - prompt = "Enter Nic Interface #" + oam.itoa(nicID) + " IP Address of " + newModuleHostName + " (" + newModuleIPAddr + ") > "; + prompt = "Enter Nic Interface #" + oam.itoa(nicID) + " IP Address or hostname of " + newModuleHostName + " (" + newModuleIPAddr + ") > "; pcommand = callReadline(prompt.c_str()); if (pcommand) @@ -2489,7 +2605,7 @@ int main(int argc, char* argv[]) continue; } - if (oam.isValidIP(newModuleIPAddr)) + if (oam.isValidIP(newModuleIPAddr) || doNotResolveHostNames) { //check and see if hostname already used bool matchFound = false; @@ -2948,7 +3064,7 @@ int main(int argc, char* argv[]) } } } - + vector dbroots; string tempdbrootList; @@ -3497,7 +3613,7 @@ int main(int argc, char* argv[]) } if ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM || - pmNumber > 1 ) + pmNumber > 1 ) { if ( password.empty() ) { @@ -3531,10 +3647,10 @@ int main(int argc, char* argv[]) if ( p1 == p2 ) { password = p2; break; - } + } else cout << "Password mismatch, please re-enter" << endl; - } + } //add single quote for special characters if ( password != "ssh" ) @@ -3543,7 +3659,7 @@ int main(int argc, char* argv[]) } } - } + } } int thread_id = 0; @@ -3824,7 +3940,7 @@ int main(int argc, char* argv[]) cout << " DONE" << endl; } } - + //configure data redundancy if (DataRedundancy) { @@ -3897,7 +4013,7 @@ int main(int argc, char* argv[]) { cout << endl << "===== Configuring MariaDB ColumnStore Data Redundancy Functionality =====" << endl << endl; - if (!glusterSetup(password)) + if (!glusterSetup(password, doNotResolveHostNames)) { cout << "ERROR: Problem setting up ColumnStore Data Redundancy" << endl; exit(1); @@ -4039,7 +4155,7 @@ int main(int argc, char* argv[]) } } } - + string dbbuilderLog = tmpDir + "/dbbuilder.log"; if (hdfs) @@ -4060,7 +4176,7 @@ int main(int argc, char* argv[]) //send message to procmon's to run upgrade script int status = sendUpgradeRequest(IserverTypeInstall, pmwithum); - + if ( status != 0 ) { cout << endl << "MariaDB Columnstore Install Failed" << endl << endl; exit(1); @@ -4423,7 +4539,7 @@ bool updateProcessConfig() vector oldModule; string newModule = ">pm"; oldModule.push_back(">um"); - + string fileName = installDir + "/etc/ProcessConfig.xml"; //Save a copy of the original version @@ -4564,9 +4680,9 @@ bool uncommentCalpontXml( string entry) */ bool makeRClocal(string moduleType, string moduleName, int IserverTypeInstall) { - + return true; - + vector lines; string mount1; @@ -5147,7 +5263,7 @@ bool storageSetup(bool amazonInstall) //check if hadoop is installed string hadoopLog = tmpDir + "/hadoop.log"; - + string cmd = "which hadoop > " + hadoopLog + " 2>&1"; system(cmd.c_str()); @@ -5732,7 +5848,7 @@ void setSystemName() Oam oam; //setup System Name - + if ( systemName.empty() ) { try { @@ -5743,7 +5859,7 @@ void setSystemName() systemName = oam::UnassignedName; } } - + if ( systemName.empty() ) systemName = "columnstore-1"; @@ -5759,7 +5875,7 @@ void setSystemName() callFree(pcommand); } } - + try { sysConfig->setConfig(SystemSection, "SystemName", systemName); @@ -6275,7 +6391,7 @@ std::string launchInstance(ModuleIP moduleip) return instanceName; } -bool glusterSetup(string password) +bool glusterSetup(string password, bool doNotResolveHostNames) { Oam oam; int dataRedundancyCopies = 0; @@ -6429,10 +6545,14 @@ bool glusterSetup(string password) callFree(pcommand); } - if ( moduleIPAddr == oam::UnassignedIpAddr ) + if ( moduleIPAddr == oam::UnassignedIpAddr) { //get IP Address - string IPAddress = oam.getIPAddress( moduleHostName); + string IPAddress; + if (doNotResolveHostNames) + IPAddress = moduleHostName; + else + IPAddress = oam.getIPAddress( moduleHostName); if ( !IPAddress.empty() ) moduleIPAddr = IPAddress; @@ -6467,7 +6587,7 @@ bool glusterSetup(string password) continue; } - if (oam.isValidIP(moduleIPAddr)) + if (oam.isValidIP(moduleIPAddr) || doNotResolveHostNames) { // run ping test to validate @@ -6926,7 +7046,7 @@ bool glusterSetup(string password) void singleServerConfigSetup(Config* sysConfig) { - + try { sysConfig->setConfig("ExeMgr1", "IPAddr", "127.0.0.1"); diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index f21459f22..686243f87 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -2205,10 +2205,15 @@ void processMSG(messageqcpp::IOSocket* cfIos) string value; uint16_t count, ivalue, nicCount; + uint8_t tmp8; oam::DeviceNetworkConfig devicenetworkconfig; oam::DeviceNetworkList devicenetworklist; oam::HostConfig hostconfig; + bool storeHostnames; + msg >> tmp8; + storeHostnames = (tmp8 != 0); + //get module count to add msg >> count; @@ -2223,7 +2228,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) devicenetworkconfig.UserTempDeviceName = value; msg >> value; devicenetworkconfig.DisableState = value; - + msg >> nicCount; for (int j = 0 ; j < nicCount ; j ++ ) @@ -2244,7 +2249,7 @@ void processMSG(messageqcpp::IOSocket* cfIos) string password; msg >> password; - status = processManager.addModule(devicenetworklist, password); + status = processManager.addModule(devicenetworklist, password, storeHostnames); log.writeLog(__LINE__, "ADDMODULE: ACK received from Process-Monitor, return status = " + oam.itoa(status)); } @@ -4835,7 +4840,8 @@ int ProcessManager::reinitProcessType( std::string processName ) * purpose: Add Module to system configuration * ******************************************************************************************/ -int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::string password, bool manualFlag) +int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::string password, bool storeHostnames, + bool manualFlag) { ProcessLog log; Configuration config; @@ -5262,7 +5268,10 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str } hostconfig.HostName = hostName; - hostconfig.IPAddr = IPAddr; + if (storeHostnames) + hostconfig.IPAddr = hostName; + else + hostconfig.IPAddr = IPAddr; hostconfig.NicID = (*pt1).NicID; devicenetworkconfig.hostConfigList.push_back(hostconfig); } diff --git a/procmgr/processmanager.h b/procmgr/processmanager.h index 9a57b38b5..8dc39a540 100644 --- a/procmgr/processmanager.h +++ b/procmgr/processmanager.h @@ -416,7 +416,8 @@ public: /** *@brief Add Module */ - int addModule(oam::DeviceNetworkList devicenetworklist, std::string password, bool manualFlag = true); + int addModule(oam::DeviceNetworkList devicenetworklist, std::string password, bool storeHostnames, + bool manualFlag = true); /** *@brief Configure Module diff --git a/tools/cleartablelock/cleartablelock.cpp b/tools/cleartablelock/cleartablelock.cpp index 1f00a12e6..a18148343 100644 --- a/tools/cleartablelock/cleartablelock.cpp +++ b/tools/cleartablelock/cleartablelock.cpp @@ -650,6 +650,11 @@ int main(int argc, char** argv) rc = execBulkRollbackReq( msgQueClts, pmList, &brm, tInfo, tblName.toString(), rollbackOnly, errMsg ); + BRM::TxnID txn; + txn.id = tInfo.ownerTxnID; + txn.valid = true; + brm.rolledback(txn); + if (rc != 0) { logFinalStatus( tblName.toString(), lockID, errMsg ); diff --git a/utils/funcexp/func_truncate.cpp b/utils/funcexp/func_truncate.cpp index b822caacd..2da482ab8 100644 --- a/utils/funcexp/func_truncate.cpp +++ b/utils/funcexp/func_truncate.cpp @@ -136,7 +136,34 @@ uint64_t Func_truncate::getUintVal(Row& row, bool& isNull, CalpontSystemCatalog::ColType& op_ct) { - return parm[0]->data()->getUintVal(row, isNull); + uint64_t val = parm[0]->data()->getUintVal(row, isNull); + + if (isNull) + return val; + + int64_t d = parm[1]->data()->getIntVal(row, isNull); + + if (isNull || d >= 0) + return val; + + uint64_t p = 1; + int64_t i = (-d); + + // Handle overflow since p can't have more than 19 0's + if (i >= 20) + { + val = 0; + } + else + { + while (i--) + p *= 10; + + val /= p; + val *= p; + } + + return val; } diff --git a/utils/rowgroup/rowgroup.h b/utils/rowgroup/rowgroup.h index e1c3073c4..ca05360c2 100644 --- a/utils/rowgroup/rowgroup.h +++ b/utils/rowgroup/rowgroup.h @@ -1183,6 +1183,7 @@ inline bool Row::equals(const Row& r2, const std::vector& keyCols) con inline bool Row::equals(const Row& r2, uint32_t lastCol) const { + // This check fires with empty r2 only. if (lastCol >= columnCount) return true; diff --git a/utils/windowfunction/idborderby.cpp b/utils/windowfunction/idborderby.cpp index 8a021f8d8..bac14bec8 100644 --- a/utils/windowfunction/idborderby.cpp +++ b/utils/windowfunction/idborderby.cpp @@ -461,7 +461,8 @@ uint64_t IdbOrderBy::Hasher::operator()(const Row::Pointer& p) const { Row& row = ts->row1; row.setPointer(p); - uint64_t ret = row.hash(colCount); + // MCOL-1829 Row::h uses colcount as an array idx down a callstack. + uint64_t ret = row.hash();//(colCount - 1); //cout << "hash(): returning " << ret << " for row: " << row.toString() << endl; return ret; } @@ -471,7 +472,9 @@ bool IdbOrderBy::Eq::operator()(const Row::Pointer& d1, const Row::Pointer& d2) Row& r1 = ts->row1, &r2 = ts->row2; r1.setPointer(d1); r2.setPointer(d2); - bool ret = r1.equals(r2, colCount); + // MCOL-1829 Row::equals uses 2nd argument as container size boundary + // so it must be column count - 1. + bool ret = r1.equals(r2, colCount - 1); //cout << "equals(): returning " << (int) ret << " for r1: " << r1.toString() << " r2: " << r2.toString() // << endl; diff --git a/utils/windowfunction/windowfunctiontype.cpp b/utils/windowfunction/windowfunctiontype.cpp index bdfe8c73c..68015d120 100644 --- a/utils/windowfunction/windowfunctiontype.cpp +++ b/utils/windowfunction/windowfunctiontype.cpp @@ -325,7 +325,7 @@ template<> void WindowFunctionType::setValue(uint64_t i, string& t) fRow.setStringField(t, i); } -//template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, string*); +// MCOL-1676 Need a separate specialization for string now. template<> void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, string* v) { @@ -343,7 +343,6 @@ void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t break; fRow.setData(getPointer((*fRowData)[j])); - // MCOL-1985 Set the data based on out column type (ct) setValue(i, *v); } } @@ -365,7 +364,7 @@ void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, T* v) break; fRow.setData(getPointer((*fRowData)[j])); - // MCOL-1985 Set the data based on out column type (ct) + // MCOL-1676 Set the data based on out column type (ct) switch (ct) { case CalpontSystemCatalog::TINYINT: @@ -412,7 +411,7 @@ void WindowFunctionType::setValue(int ct, int64_t b, int64_t e, int64_t c, T* v) setValue(i, *v); } } - } + } } template @@ -545,8 +544,6 @@ template void WindowFunctionType::setValue(int, int64_t, int64_t, int64 template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, uint64_t*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, float*); template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, double*); -//template void WindowFunctionType::setValue(int, int64_t, int64_t, int64_t, string*); - void* WindowFunctionType::getNullValueByType(int ct, int pos) { diff --git a/versioning/BRM/dbrm.cpp b/versioning/BRM/dbrm.cpp index a763bbd3f..984c4f9de 100644 --- a/versioning/BRM/dbrm.cpp +++ b/versioning/BRM/dbrm.cpp @@ -1884,7 +1884,7 @@ int DBRM::deletePartition(const std::vector& oids, std::set::const_iterator partIt; for (partIt = partitionNums.begin(); partIt != partitionNums.end(); ++partIt) - oss << (*it) << " " + oss << (*partIt) << " "; oss << "; OIDS: "; std::vector::const_iterator it; diff --git a/writeengine/server/we_dmlcommandproc.cpp b/writeengine/server/we_dmlcommandproc.cpp index 057ef2504..cbc84e26e 100644 --- a/writeengine/server/we_dmlcommandproc.cpp +++ b/writeengine/server/we_dmlcommandproc.cpp @@ -2742,6 +2742,7 @@ uint8_t WE_DMLCommandProc::processUpdate(messageqcpp::ByteStream& bs, boost::scoped_array preBlkNums(new int[columnsUpdated.size()]); boost::scoped_array oids(new OID[columnsUpdated.size()]); + BRMWrapper::setUseVb(true); for (unsigned int j = 0; j < columnsUpdated.size(); j++) { //timer.start("lookupsyscat"); diff --git a/writeengine/shared/we_convertor.cpp b/writeengine/shared/we_convertor.cpp index 8050426f3..53af4eb59 100644 --- a/writeengine/shared/we_convertor.cpp +++ b/writeengine/shared/we_convertor.cpp @@ -178,97 +178,38 @@ const std::string Convertor::int2Str(int val) * converted long long for specified "field" ******************************************************************************/ /* static */ + long long Convertor::convertDecimalString( const char* field, - int fieldLength, - int scale ) + int fieldLength, + int scale) { - long long llVal = 0; + long double dval = strtold(field, NULL); + long long ret = 0; - int nDigitsBeforeDecPt = 0; - int nDigitsAfterDecPt = 0; - long long roundUp = 0; //@bug 3405 round off decimal column values + + // move scale digits to the left of the decimal point + for (int i = 0; i < scale; i++) + dval *= 10; - // Determine the number of digits before and after the decimal point - char* posDecPt = (char*)memchr(field, '.', fieldLength); + + // range check against int64 + if (dval > LLONG_MAX || dval < LLONG_MIN) + errno = ERANGE; - if (posDecPt) - { - nDigitsBeforeDecPt = posDecPt - field; - nDigitsAfterDecPt = fieldLength - nDigitsBeforeDecPt - 1; - - //@bug 3405 round off decimal column values - // We look at the scale+1 digit to see if we need to round up. - if (nDigitsAfterDecPt > scale) - { - char roundOffDigit = *(posDecPt + 1 + scale); - - if ( (roundOffDigit > '4') && - (roundOffDigit <= '9') ) // round up - { - roundUp = 1; - - // We can't just use the sign of llVal to determine whether to - // add +1 or -1, because if we read in -0.005 with scale 2, we - // end up parsing "-0.00", which yields 0; meaning we lose the - // sign. So better (though maybe slower) to look for any lead- - // ing negative sign in the input string. - for (int k = 0; k < fieldLength; k++) - { - if (!isspace(field[k])) - { - if (field[k] == '-') - roundUp = -1; - - break; - } - } - } - } - } else - { - nDigitsBeforeDecPt = fieldLength; - nDigitsAfterDecPt = 0; - } - // Strip out the decimal point by stringing together - // the digits before and after the decimal point. - char* data = (char*)alloca(nDigitsBeforeDecPt + scale + 1); - memcpy(data, field, nDigitsBeforeDecPt); - - if (nDigitsAfterDecPt) - { - if (scale > nDigitsAfterDecPt) - memcpy(data + nDigitsBeforeDecPt, - field + nDigitsBeforeDecPt + 1, - nDigitsAfterDecPt); - else // (scale <= nDigitsAfterDecPt) - memcpy(data + nDigitsBeforeDecPt, - field + nDigitsBeforeDecPt + 1, - scale); - } - - // Add on any necessary zero padding at the end - if (scale > nDigitsAfterDecPt) - { - memset(data + nDigitsBeforeDecPt + nDigitsAfterDecPt, - '0', - scale - nDigitsAfterDecPt); - } - - data[nDigitsBeforeDecPt + scale] = '\0'; - - // Convert our constructed decimal string back to a long long - //@bug 1814 Force strtoll to use base 10 - errno = 0; - llVal = strtoll(data, 0, 10); - - //@bug 3405 round off decimal values - if ((roundUp) && (errno == 0)) - llVal += roundUp; - - return llVal; + errno = 0; + + ret = dval; + + // get the fractional part of what's left & round ret up or down. + dval -= ret; + if (dval >= 0.5 && ret < LLONG_MAX) + ++ret; + else if (dval <= -0.5 && ret > LLONG_MIN) + --ret; + return ret; } /*******************************************************************************