diff --git a/build/columnstore.spec.in b/build/columnstore.spec.in index f43f54718..489595d88 100644 --- a/build/columnstore.spec.in +++ b/build/columnstore.spec.in @@ -269,9 +269,6 @@ rm -rf $RPM_BUILD_ROOT $RPM_BUILD_DIR/%{name}-%{version}.%{release} /usr/local/mariadb/columnstore/bin/file2oid.pl /usr/local/mariadb/columnstore/bin/print_journal /usr/local/mariadb/columnstore/bin/healthcheck -/usr/local/mariadb/columnstore/bin/glusterctl -/usr/local/mariadb/columnstore/bin/glusteradd -/usr/local/mariadb/columnstore/bin/glusterconf /usr/local/mariadb/columnstore/bin/redistributeDB /usr/local/mariadb/columnstore/bin/databaseSizeReport /usr/local/mariadb/columnstore/bin/gdb diff --git a/oam/install_scripts/performance_installer.sh b/oam/install_scripts/performance_installer.sh index 00a2c7b4d..dd45f1833 100644 --- a/oam/install_scripts/performance_installer.sh +++ b/oam/install_scripts/performance_installer.sh @@ -32,9 +32,9 @@ if { $UNM != "" } { } set BASH "/bin/bash " -if { $DEBUG == "1" } { - set BASH "/bin/bash -x " -} +#if { $DEBUG == "1" } { +# set BASH "/bin/bash -x " +#} set HOME "$env(HOME)" @@ -99,13 +99,14 @@ set timeout 60 # check return expect { "Exit status 0" { send_user "DONE" } - "No such file" { send_user "ERROR: post-install Not Found\n" ; exit 1 } + # "No such file" { send_user "ERROR: post-install Not Found\n" ; exit 1 } "MariaDB Columnstore syslog logging not working" { send_user "ERROR: MariaDB Columnstore System logging not setup\n" ; exit 1 } "Permission denied, please try again" { send_user "ERROR: Invalid password\n" ; exit 1 } "Read-only file system" { send_user "ERROR: local disk - Read-only file system\n" ; exit 1} "Connection refused" { send_user "ERROR: Connection refused\n" ; exit 1 } "Connection closed" { send_user "ERROR: Connection closed\n" ; exit 1 } "No route to host" { send_user "ERROR: No route to host\n" ; exit 1 } + timeout { } } send_user "\n" @@ -131,6 +132,7 @@ if { $INSTALLTYPE == "initial" || $INSTALLTYPE == "uninstall" } { "Connection closed" { send_user "ERROR: Connection closed\n" ; exit 1 } "No route to host" { send_user "ERROR: No route to host\n" ; exit 1 } "Exit status 0" { send_user "DONE" } + timeout { } } send_user "\n" } @@ -155,9 +157,9 @@ expect { "Connection refused" { send_user "ERROR: Connection refused\n" ; exit 1 } "Connection closed" { send_user "ERROR: Connection closed\n" ; exit 1 } "No route to host" { send_user "ERROR: No route to host\n" ; exit 1 } - "Exit status 0" { send_user "DONE" } + timeout { } } - +set timeout 30 send "scp -v $HOME/mariadb-columnstore*$VERSION*$PKGTYPE $USERNAME@$SERVER:.\n" if { $PASSWORD != "ssh"} { set timeout 30 @@ -171,6 +173,7 @@ expect { "Connection closed" { send_user "ERROR: Connection closed\n" ; exit 1 } "Exit status 0" { send_user "DONE" } "Exit status 1" { send_user "ERROR: scp failed" ; exit 1 } + timeout { } } send_user "\n" @@ -199,6 +202,7 @@ if { $INSTALLTYPE == "initial"} { "needs" { send_user "ERROR: disk space issue\n" ; exit 1 } "conflicts" { send_user "ERROR: File Conflict issue\n" ; exit 1 } "Exit status 0" { send_user "DONE" } + timeout { } } } @@ -225,6 +229,7 @@ expect { "Connection closed" { send_user "ERROR: Connection closed\n" ; exit 1 } "No route to host" { send_user "ERROR: No route to host\n" ; exit 1 } "Starting MariaDB" { send_user "DONE" } + timeout { } } send_user "\n" @@ -250,6 +255,7 @@ if { $AMAZONINSTALL == "1" } { "Connection refused" { send_user "ERROR: Connection refused\n" ; exit 1 } "Connection closed" { send_user "ERROR: Connection closed\n" ; exit 1 } "No route to host" { send_user "ERROR: No route to host\n" ; exit 1 } + timeout { } } } diff --git a/oam/install_scripts/user_installer.sh b/oam/install_scripts/user_installer.sh index fa69d46ad..214bdfae8 100644 --- a/oam/install_scripts/user_installer.sh +++ b/oam/install_scripts/user_installer.sh @@ -96,13 +96,14 @@ set timeout 60 # check return expect { "Exit status 0" { send_user "DONE" } - "No such file" { send_user "ERROR: post-install Not Found\n" ; exit 1 } + #"No such file" { send_user "ERROR: post-install Not Found\n" ; exit 1 } "MariaDB Columnstore syslog logging not working" { send_user "ERROR: MariaDB Columnstore System logging not setup\n" ; exit 1 } "Permission denied, please try again" { send_user "ERROR: Invalid password\n" ; exit 1 } "Read-only file system" { send_user "ERROR: local disk - Read-only file system\n" ; exit 1} "Connection refused" { send_user "ERROR: Connection refused\n" ; exit 1 } "Connection closed" { send_user "ERROR: Connection closed\n" ; exit 1 } "No route to host" { send_user "ERROR: No route to host\n" ; exit 1 } + timeout { } } send_user "\n" @@ -125,7 +126,8 @@ if { $INSTALLTYPE == "initial" || $INSTALLTYPE == "uninstall" } { "error: Failed dependencies" { send_user "ERROR: Failed dependencies\n" ; exit 1 } "Permission denied, please try again" { send_user "ERROR: Invalid password\n" ; exit 1 } "Connection closed" { send_user "ERROR: Connection closed\n" ; exit 1 } - "Exit status 0" { send_user "DONE" } + "Exit status 0" { send_user "DONE" } + timeout { } } send_user "\n" } @@ -148,10 +150,10 @@ if { $PASSWORD != "ssh" } { } expect { "Exit status 0" { send_user "DONE" } - -re {[$#] } { } "Connection refused" { send_user "ERROR: Connection refused\n" ; exit 1 } "Connection closed" { send_user "ERROR: Connection closed\n" ; exit 1 } "No route to host" { send_user "ERROR: No route to host\n" ; exit 1 } + timeout { } } set timeout 30 @@ -168,6 +170,7 @@ expect { "Connection closed" { send_user "ERROR: Connection closed\n" ; exit 1 } "Exit status 0" { send_user "DONE" } "Exit status 1" { send_user "ERROR: scp failed" ;exit 1 } + timeout { } } send_user "\n" @@ -194,6 +197,7 @@ if { $INSTALLTYPE == "initial"} { "needs" { send_user "ERROR: disk space issue\n" ; exit 1 } "conflicts" { send_user "ERROR: File Conflict issue\n" ; exit 1 } "Exit status 0" { send_user "DONE" } + timeout { } } send_user "\n" } @@ -220,6 +224,7 @@ expect { "Connection closed" { send_user "ERROR: Connection closed\n" ; exit 1 } "No route to host" { send_user "ERROR: No route to host\n" ; exit 1 } "Starting MariaDB" { send_user "DONE" } + timeout { } } send_user "\n" @@ -245,6 +250,7 @@ if { $AMAZONINSTALL == "1" } { "Connection refused" { send_user "ERROR: Connection refused\n" ; exit 1 } "Connection closed" { send_user "ERROR: Connection closed\n" ; exit 1 } "No route to host" { send_user "ERROR: No route to host\n" ; exit 1 } + timeout { } } } diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp index c230cbdbb..47a235527 100644 --- a/oam/oamcpp/liboamcpp.cpp +++ b/oam/oamcpp/liboamcpp.cpp @@ -6899,11 +6899,12 @@ namespace oam if ( GlusterConfig == "y") { try { - string errmsg; - int ret = glusterctl(oam::GLUSTER_DELETE, itoa(dbrootID), errmsg, errmsg); + string errmsg1; + string errmsg2; + int ret = glusterctl(oam::GLUSTER_DELETE, itoa(dbrootID), errmsg1, errmsg2); if ( ret != 0 ) { - cerr << "FAILURE: Error deleting gluster dbroot# " + itoa(dbrootID) + ", error: " + errmsg << endl; + cerr << "FAILURE: Error deleting gluster dbroot# " + itoa(dbrootID) + ", error: " + errmsg1 << endl; exceptionControl("removeDbroot", API_FAILURE); } } @@ -8098,13 +8099,12 @@ namespace oam return; } - /****************************************************************************************** * @brief glusterctl * * purpose: gluster control and glusteradd * - * commands: status - Used to check status of kuster and disk good/bad + * commands: status - Used to check status of gluster and disk good/bad * returns * NOTINSTALLED * OK @@ -8156,40 +8156,42 @@ namespace oam #ifndef _MSC_VER int user; user = getuid(); - string glustercmd = InstallDir + "/bin/glusterctl"; + string glustercmd = "gluster "; - if (access(glustercmd.c_str(), X_OK) != 0 ) - exceptionControl("glusterctl", API_DISABLED); + if (user!=0) + { + glustercmd = "sudo " + glustercmd; + } errmsg = ""; switch ( command ) { case (oam::GLUSTER_STATUS): { - glustercmd = glustercmd + " status > /tmp/gluster_status.log 2>&1"; + string command = glustercmd + "volume status"; - int ret; - ret = system(glustercmd.c_str()); - if ( WEXITSTATUS(ret) == 0 ) - return 0; - - ret = checkGlusterLog("/tmp/gluster_status.log", errmsg); - return ret; + char buffer[128]; + string result = ""; + FILE* pipe = popen(command.c_str(), "r"); + if (!pipe) exceptionControl("GLUSTER_STATUS", API_FAILURE,"popen() failed"); + try { + while (!feof(pipe)) { + if (fgets(buffer, 128, pipe) != NULL) + result += buffer; + } + } catch (...) { + exceptionControl("GLUSTER_STATUS", API_FAILURE); + throw; + } + pclose(pipe); + argument2 = result; + return 0; } case (oam::GLUSTER_SETDDC): { - string copies = argument1; - - writeLog("glusterctl: GLUSTER_SETDDC: " + copies, LOG_TYPE_DEBUG ); - glustercmd = glustercmd + " setddc " + copies + " > /tmp/gluster_setddc.log 2>&1"; - int ret; - ret = system(glustercmd.c_str()); - if ( WEXITSTATUS(ret) == 0 ) - return 0; - - ret = checkGlusterLog("/tmp/gluster_setddc.log", errmsg); - return ret; + // The way this was implemented it doesn't seem possible to actually change the value.. + return 0; } case (oam::GLUSTER_ASSIGN): @@ -8201,19 +8203,8 @@ namespace oam int returnStatus = sendMsgToProcMgr(GLUSTERASSIGN, pm, FORCEFUL, ACK_YES, dbrootID); if (returnStatus != API_SUCCESS) - exceptionControl("GLUSTER_ASSIGN", returnStatus); -/* - writeLog("glusterctl call: GLUSTER_ASSIGN: dbroot = " + dbrootID + " pm = " + pmID, LOG_TYPE_DEBUG ); - glustercmd = glustercmd + " assign " + dbrootID + " " + pmID + " > /tmp/gluster_assign.log 2>&1"; - int ret; - ret = system(glustercmd.c_str()); - writeLog("glusterctl return: GLUSTER_ASSIGN: dbroot = " + dbrootID + " pm = " + pmID, LOG_TYPE_DEBUG ); - if ( WEXITSTATUS(ret) == 0 ) - return 0; + exceptionControl("GLUSTER_ASSIGN", returnStatus); - ret = checkGlusterLog("/tmp/gluster_assign.log", errmsg); - return ret; -*/ break; } @@ -8228,84 +8219,188 @@ namespace oam if (returnStatus != API_SUCCESS) exceptionControl("GLUSTER_UNASSIGN", returnStatus); -/* writeLog("glusterctl: GLUSTER_UNASSIGN: dbroot = " + dbrootID + " pm = " + pmID, LOG_TYPE_DEBUG ); - glustercmd = glustercmd + " unassign " + dbrootID + " " + pmID + " > /tmp/gluster_unassign.log 2>&1"; - int ret; - ret = system(glustercmd.c_str()); - if ( WEXITSTATUS(ret) == 0 ) - return 0; - - ret = checkGlusterLog("/tmp/gluster_unassign.log", errmsg); - return ret; -*/ break; } case (oam::GLUSTER_WHOHAS): { + + // Script returned a string of space separated PMIDs everywhere this is called it expects that return form. + // When configuring or adding Modules we need to update DataRedundancyConfig section of configuration file. + Config* sysConfig = Config::makeConfig(CalpontConfigFile.c_str()); string dbrootID = argument1; - string msg; - - for ( int retry = 0 ; retry < 5 ; retry++ ) + string moduleDBRootIDPMs = "DBRoot" + dbrootID + "PMs"; + string msg = sysConfig->getConfig("DataRedundancyConfig",moduleDBRootIDPMs); + if (msg.empty()) { - writeLog("glusterctl: GLUSTER_WHOHAS for dbroot : " + dbrootID, LOG_TYPE_DEBUG ); - glustercmd = glustercmd + " whohas " + dbrootID + " > /tmp/gluster_howhas.log 2>&1"; - system(glustercmd.c_str()); - - int ret; - ret = checkGlusterLog("/tmp/gluster_howhas.log", msg); - - if ( ret == 0 ) - { // OK return, get pm list - if ( msg.empty() ) - { - writeLog("glusterctl: GLUSTER_WHOHAS: empty pm list", LOG_TYPE_ERROR ); - exceptionControl("glusterctl", API_FAILURE); - } - else - { - writeLog("glusterctl: GLUSTER_WHOHAS: pm list = " + msg, LOG_TYPE_DEBUG ); - argument2 = msg; - return 0; - } - } - - // retry failure - writeLog("glusterctl: GLUSTER_WHOHAS: failure, retrying (restarting gluster) " + msg, LOG_TYPE_ERROR ); - - string cmd = "service glusterd restart > /dev/null 2>&1"; - if (user != 0) - cmd = "sudo " + cmd; - - system(cmd.c_str()); - - cmd = "systemctrl restart glusterd > /dev/null 2>&1"; - if (user != 0) - cmd = "sudo " + cmd; - - system(cmd.c_str()); - sleep(1); + exceptionControl("glusterctl", API_FAILURE); } - + argument2 = msg; + return 0; break; } case (oam::GLUSTER_ADD): { - string pmID = argument1; - string dbrootID = argument2; + int pmID = atoi(argument1.c_str()); + int dbrootID = atoi(argument2.c_str()); + string command = ""; + writeLog("glusterctl: GLUSTER_ADD: dbroot = " + argument2 + " pm = " + argument1, LOG_TYPE_DEBUG ); - string glustercmd = InstallDir + "/bin/glusteradd"; - writeLog("glusterctl: GLUSTER_ADD: dbroot = " + dbrootID + " pm = " + pmID, LOG_TYPE_DEBUG ); - glustercmd = glustercmd + " " + pmID + " " + dbrootID + " > /tmp/gluster_add.log 2>&1"; - int ret; - //writeLog("glusterctl: cmd = " + glustercmd, LOG_TYPE_DEBUG ); - ret = system(glustercmd.c_str()); - if ( WEXITSTATUS(ret) == 0 ) - return 0; + Config* sysConfig = Config::makeConfig(); + ModuleTypeConfig moduletypeconfig; + getSystemConfig("pm", moduletypeconfig); + int dataRedundancyCopies; + int dbrootCount; + getSystemConfig("DataRedundancyCopies",dataRedundancyCopies); + getSystemConfig("DBRootCount",dbrootCount); - ret = checkGlusterLog("/tmp/gluster_add.log", errmsg); - return ret; + int numberPMs = moduletypeconfig.ModuleCount; + int numberNewPMs = numberPMs - pmID + 1; + int numberNewDBRoots = (dbrootCount - dbrootID) + 1; + int numberDBRootsPerPM = numberNewDBRoots/numberNewPMs; + + std::vector dbrootPms[dbrootCount]; + DataRedundancyConfig DataRedundancyConfigs[numberPMs]; + int startDBRootID = dbrootID; + + for (int pm=(pmID-1); pm < numberPMs; pm++,startDBRootID++) + { + DataRedundancyConfigs[pm].pmID = (pm+1); + string moduleHostName = "ModuleHostName" + itoa(pm+1) + "-1-3"; + string moduleIpAddr = "ModuleIPAddr" + itoa(pm+1) + "-1-3"; + + DataRedundancyConfigs[pm].pmHostname = sysConfig->getConfig("DataRedundancyConfig",moduleHostName); + DataRedundancyConfigs[pm].pmIpAddr = sysConfig->getConfig("DataRedundancyConfig",moduleIpAddr); + if (DataRedundancyConfigs[pm].pmHostname.empty()) + { + DataRedundancyConfigs[pm].pmHostname = sysConfig->getConfig("SystemModuleConfig",moduleHostName); + } + if (DataRedundancyConfigs[pm].pmIpAddr.empty()) + { + DataRedundancyConfigs[pm].pmIpAddr = sysConfig->getConfig("SystemModuleConfig",moduleIpAddr); + } + + int nextPM = (pm+1); + int dbrootCopy = startDBRootID; + if (nextPM >= numberPMs) + { + nextPM=(pmID-1); + } + for ( int i=0; i= numberPMs) + { + nextPM=(pmID-1); + } + if (nextPM == pm) + { + nextPM++; + } + if (nextPM >= numberPMs) + { + nextPM=(pmID-1); + } + } + dbrootCopy += numberNewPMs; + } + } + + for (int db=(dbrootID-1); db < dbrootCount; db++) + { + int newDBrootID = db + 1; + string dbrootIDPMs = ""; + string moduleDBRootIDPMs = "DBRoot" + itoa(newDBrootID) + "PMs"; + for (int pm=(pmID-1); pm < numberPMs; pm++) + { + if(find(DataRedundancyConfigs[pm].dbrootCopies.begin(),DataRedundancyConfigs[pm].dbrootCopies.end(),newDBrootID) != DataRedundancyConfigs[pm].dbrootCopies.end()) + { + dbrootPms[db].push_back(DataRedundancyConfigs[pm].pmID); + dbrootIDPMs += itoa(DataRedundancyConfigs[pm].pmID); + dbrootIDPMs += " "; + } + } + // Store to config and distribute so that GLUSTER_WHOHAS will work + sysConfig->setConfig("DataRedundancyConfig", moduleDBRootIDPMs, dbrootIDPMs); + } + + for (int pm=(pmID-1); pm < numberPMs; pm++) + { + command = glustercmd + "peer probe " + DataRedundancyConfigs[pm].pmIpAddr; + int status = system(command.c_str()); + if (WEXITSTATUS(status) != 0 ) + { + writeLog("ERROR: command failed: ",LOG_TYPE_DEBUG); + exceptionControl("GLUSTER_ADD", API_FAILURE); + } + } + sleep(5); + command = glustercmd + "peer status "; + int status = system(command.c_str()); + if (WEXITSTATUS(status) != 0 ) + { + cout << "ERROR: command failed: " << command << endl; + exit(1); + } + //Need to wait since peer probe success does not always mean it is ready for volume create command + sleep(10); + for (int db=(dbrootID-1); db < dbrootCount; db++) + { + int newDbrootID = db + 1; + + command = glustercmd + "volume create dbroot" + itoa(newDbrootID) + " transport tcp replica " + itoa(dataRedundancyCopies) + " "; + + vector::iterator dbrootPmIter = dbrootPms[db].begin(); + for (; dbrootPmIter < dbrootPms[db].end(); dbrootPmIter++ ) + { + int pm = (*dbrootPmIter) - 1; + command += DataRedundancyConfigs[pm].pmIpAddr + ":" + InstallDir +"/gluster/brick" + itoa(newDbrootID) + " "; + } + command += "force"; + int status = system(command.c_str()); + if (WEXITSTATUS(status) != 0 ) + { + writeLog("ERROR: command failed: " + command,LOG_TYPE_DEBUG); + exceptionControl("GLUSTER_ADD", API_FAILURE); + } + command = glustercmd + "volume start dbroot" + itoa(newDbrootID); + status = system(command.c_str()); + if (WEXITSTATUS(status) != 0 ) + { + writeLog("ERROR: command failed: ",LOG_TYPE_DEBUG); + exceptionControl("GLUSTER_ADD", API_FAILURE); + } + } + + try + { + sysConfig->write(); + } + catch(...) + { + exceptionControl("sysConfig->write", API_FAILURE); + } + + distributeConfigFile("system"); + + for (int pm=(pmID-1); pm < numberPMs; pm++) + { + for (int i=0; igetConfig("SystemModuleConfig",ModuleDBRootID); + string command = "" + DataRedundancyConfigs[pm].pmIpAddr + + ":/dbroot" + dbr + " " + InstallDir + "/data" + dbr + + " glusterfs defaults,direct-io-mode=enable 00"; + string toPM = "pm" + itoa(pm+1); + distributeFstabUpdates(command,toPM); + } + } break; } @@ -8313,18 +8408,27 @@ namespace oam case (oam::GLUSTER_DELETE): { string dbrootID = argument1; - + string command = ""; + int status; writeLog("glusterctl: GLUSTER_DELETE: dbroot = " + dbrootID, LOG_TYPE_DEBUG ); - glustercmd = glustercmd + " delete " + dbrootID + " > /tmp/gluster_delete.log 2>&1"; - int ret; - //writeLog("glusterctl: cmd = " + glustercmd, LOG_TYPE_DEBUG ); - ret = system(glustercmd.c_str()); - if ( WEXITSTATUS(ret) == 0 ) - return 0; - ret = checkGlusterLog("/tmp/gluster_delete.log", errmsg); - return ret; + command = glustercmd + "volume stop dbroot" + dbrootID; + status = system(command.c_str()); + if (WEXITSTATUS(status) != 0 ) + { + writeLog("ERROR: command failed: ",LOG_TYPE_DEBUG); + exceptionControl("GLUSTER_DELETE", API_FAILURE); + } + + command = glustercmd + "volume delete dbroot" + dbrootID; + + status = system(command.c_str()); + if (WEXITSTATUS(status) != 0 ) + { + writeLog("ERROR: command failed: ",LOG_TYPE_DEBUG); + exceptionControl("GLUSTER_DELETE", API_FAILURE); + } break; } @@ -8649,7 +8753,6 @@ namespace oam return entry; } - /*************************************************************************** * PRIVATE FUNCTIONS ***************************************************************************/ diff --git a/oam/oamcpp/liboamcpp.h b/oam/oamcpp/liboamcpp.h index 5d50b3bb0..7e708e45e 100644 --- a/oam/oamcpp/liboamcpp.h +++ b/oam/oamcpp/liboamcpp.h @@ -1295,6 +1295,23 @@ namespace oam }; typedef std::vector ActiveSqlStatements; + struct DataRedundancyStorageSetup + { + int brickID; + std::string storageLocation; + std::string storageFilesytemType; + }; + typedef std::vector DataRedundancyStorage; + + struct DataRedundancyConfig_s + { + int pmID; + std::string pmHostname; + std::string pmIpAddr; + std::vector dbrootCopies; + DataRedundancyStorage storageLocations; + }; + typedef struct DataRedundancyConfig_s DataRedundancyConfig; // username / password for smbclient use const std::string USERNAME = "oamuser"; diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp index 7e9eee816..745970c77 100644 --- a/oamapps/mcsadmin/mcsadmin.cpp +++ b/oamapps/mcsadmin/mcsadmin.cpp @@ -1877,20 +1877,20 @@ int processCommand(string* arguments) } string GlusterConfig; - string GlusterCopies; - string GlusterStorageType; + string DataRedundancyCopies; + string DataRedundancyStorageType; try { oam.getSystemConfig("GlusterConfig", GlusterConfig); - oam.getSystemConfig("GlusterCopies", GlusterCopies); - oam.getSystemConfig("GlusterStorageType", GlusterStorageType); + oam.getSystemConfig("DataRedundancyCopies", DataRedundancyCopies); + oam.getSystemConfig("DataRedundancyStorageType", DataRedundancyStorageType); } catch(...) {} if ( GlusterConfig == "y" ) { cout << endl << "Data Redundant Configuration" << endl << endl; - cout << "Copies Per DBroot = " << GlusterCopies << endl; - cout << "Storage Type = " << GlusterStorageType << endl; + cout << "Copies Per DBroot = " << DataRedundancyCopies << endl; + cout << "Storage Type = " << DataRedundancyStorageType << endl; oamModuleInfo_t st; string moduleType; @@ -3573,6 +3573,32 @@ int processCommand(string* arguments) { cout << endl << "**** getSystemStatus Failed = " << e.what() << endl; } + + string GlusterConfig; + string DataRedundancyCopies; + string DataRedundancyStorageType; + try { + oam.getSystemConfig("GlusterConfig", GlusterConfig); + oam.getSystemConfig("DataRedundancyCopies", DataRedundancyCopies); + oam.getSystemConfig("DataRedundancyStorageType", DataRedundancyStorageType); + } + catch(...) {} + + if ( GlusterConfig == "y" ) + { + string arg1 = ""; + string arg2 = ""; + string errmsg = ""; + int ret = oam.glusterctl(oam::GLUSTER_STATUS, arg1, arg2, errmsg); + if ( ret == 0 ) + { + cout << arg2 << endl; + } + else + { + cerr << "FAILURE: Status check error: " + errmsg << endl; + } + } } break; @@ -5044,6 +5070,8 @@ int processCommand(string* arguments) case 48: // addModule - parameters: Module type/Module Name, Number of Modules, Server Hostnames, // Server root password optional { + Config* sysConfig = Config::makeConfig(); + if ( SingleServerInstall == "y" ) { // exit out since not on single-server install cout << endl << "**** addModule Failed : not support on a Single-Server type installs " << endl; @@ -5076,18 +5104,19 @@ int processCommand(string* arguments) } string GlusterConfig = "n"; - int GlusterCopies; + int DataRedundancyCopies; string cloud = oam::UnassignedName; - string GlusterStorageType; + int DataRedundancyNetworkType; + int DataRedundancyStorageType; string AmazonVPCNextPrivateIP; string DistributedInstall = "y"; try { oam.getSystemConfig("Cloud", cloud); oam.getSystemConfig("AmazonVPCNextPrivateIP", AmazonVPCNextPrivateIP); oam.getSystemConfig("GlusterConfig", GlusterConfig); - oam.getSystemConfig("GlusterCopies", GlusterCopies); - oam.getSystemConfig("GlusterStorageType", GlusterStorageType); - oam.getSystemConfig("DistributedInstall", DistributedInstall); + oam.getSystemConfig("DataRedundancyCopies", DataRedundancyCopies); + oam.getSystemConfig("DataRedundancyNetworkType", DataRedundancyNetworkType); + oam.getSystemConfig("DataRedundancyStorageType", DataRedundancyStorageType); } catch(...) {} @@ -5184,8 +5213,8 @@ int processCommand(string* arguments) break; } - if ( fmod((float) moduleCount , (float) GlusterCopies) != 0 ) { - cout << endl << "**** addModule Failed : Failed to Add Module, invalid number-of-modules: must be multiple of Data Redundancy Copies, which is " << GlusterCopies << endl; + if ( fmod((float) moduleCount , (float) DataRedundancyCopies) != 0 ) { + cout << endl << "**** addModule Failed : Failed to Add Module, invalid number-of-modules: must be multiple of Data Redundancy Copies, which is " << DataRedundancyCopies << endl; break; } } @@ -5298,6 +5327,8 @@ int processCommand(string* arguments) umStorageNames::const_iterator listPT2 = umstoragenames.begin(); for ( int i = 0 ; i < moduleCount ; i++ ) { + string dataDupIPaddr = "ModuleIPAddr"+oam.itoa(moduleID)+"-1-3"; + string dataDupHostName = "ModuleHostName"+oam.itoa(moduleID)+"-1-3"; //validate or determine module name moduleNameList::const_iterator listPT = modulenamelist.begin(); for( ; listPT != modulenamelist.end() ; listPT++) @@ -5403,6 +5434,27 @@ int processCommand(string* arguments) devicenetworklist.push_back(devicenetworkconfig); devicenetworkconfig.hostConfigList.clear(); moduleName.clear(); + + if ( GlusterConfig == "y" && DataRedundancyNetworkType == 2 && moduleType == "pm") + { + string DataRedundancyIPAddress = sysConfig->getConfig("DataRedundancyConfig",dataDupIPaddr); + string DataRedundancyHostname = sysConfig->getConfig("DataRedundancyConfig",dataDupHostName); + if (DataRedundancyIPAddress.empty() || DataRedundancyHostname.empty()) + { + string prompt = "DataRedundancy is configured for dedicated network, enter a hostname"; + DataRedundancyHostname = dataPrompt(prompt); + DataRedundancyIPAddress = oam.getIPAddress(DataRedundancyHostname); + if ( DataRedundancyIPAddress.empty() ) { + // prompt for IP Address + string prompt = "IP Address of " + DataRedundancyHostname + " not found, enter IP Address"; + DataRedundancyIPAddress = dataPrompt(prompt); + if (!oam.isValidIP(DataRedundancyIPAddress)) + return 1; + } + sysConfig->setConfig("DataRedundancyConfig", dataDupHostName, DataRedundancyHostname); + sysConfig->setConfig("DataRedundancyConfig", dataDupIPaddr, DataRedundancyIPAddress); + } + } } DBRootConfigList dbrootlist; @@ -5417,11 +5469,6 @@ int processCommand(string* arguments) cout << "will be created with the Modules during this command." << endl; cout << "The Data Redundancy Packages should already be installed on the" << endl; cout << "Servers being installed." << endl; - cout << "Also the Servers should have either a password-less ssh configured" << endl; - cout << "for the local server or setup for a login via password." << endl; - cout << "If its setup for login via password, then you will be required to" << endl; - cout << "enter the password when prompted. You will be prompted 2 * the" << endl; - cout << "number of modules being added." << endl; // confirm request if (confirmPrompt(" ")) @@ -5438,14 +5485,14 @@ int processCommand(string* arguments) dbrootNumber = dbrootPerPM * moduleCount; - if ( GlusterStorageType == "storage" ) + if ( DataRedundancyStorageType == 2 ) { cout << endl << "Data Redundancy Storage Type is configured for 'storage'" << endl; - cout << "You will need " << oam.itoa(dbrootNumber*GlusterCopies); - cout << " total storage locations and " << oam.itoa(dbrootPerPM*GlusterCopies) << " storage locations per PM. You will now " << endl; + cout << "You will need " << oam.itoa(dbrootNumber*DataRedundancyCopies); + cout << " total storage locations and " << oam.itoa(dbrootPerPM*DataRedundancyCopies) << " storage locations per PM. You will now " << endl; cout << "be asked to enter the device names for the storage locations. You will enter " << endl; - cout << "them for each PM, on one line, separated by spaces (" << oam.itoa(dbrootPerPM*GlusterCopies) << " names on each line)." << endl; + cout << "them for each PM, on one line, separated by spaces (" << oam.itoa(dbrootPerPM*DataRedundancyCopies) << " names on each line)." << endl; DeviceNetworkList::iterator pt = devicenetworklist.begin(); string firstPM = (*pt).DeviceName.substr(MAX_MODULE_TYPE_SIZE,MAX_MODULE_ID_SIZE); @@ -5483,28 +5530,33 @@ int processCommand(string* arguments) if ( GlusterConfig == "y" && moduleType == "pm" ) { - if ( GlusterStorageType == "storage" ) { + { //send messages to update fstab to new modules, if needed DeviceNetworkList::iterator pt2 = devicenetworklist.begin(); storageDeviceList::iterator pt3 = storagedevicelist.begin(); for( ; pt2 != devicenetworklist.end() ; pt2++, pt3++) { + HostConfigList::iterator hostConfigIter = (*pt2).hostConfigList.begin(); string moduleName = (*pt2).DeviceName; - string devices = *pt3; - int brinkID = 1; - boost::char_separator sep(" "); - boost::tokenizer< boost::char_separator > tokens(devices, sep); - for ( boost::tokenizer< boost::char_separator >::iterator it = tokens.begin(); - it != tokens.end(); - ++it) + int brickID = 1; + if ( DataRedundancyStorageType == 2 ) { - string deviceName = *it; - string entry = deviceName + " " + startup::StartUp::installDir() + "/gluster/brick" + oam.itoa(brinkID) + " " + deviceType + " defaults 1 2"; - - //send update pm - oam.distributeFstabUpdates(entry, moduleName); - brinkID++; + string devices = *pt3; + boost::char_separator sep(" "); + boost::tokenizer< boost::char_separator > tokens(devices, sep); + for ( boost::tokenizer< boost::char_separator >::iterator it = tokens.begin(); + it != tokens.end(); + ++it) + { + string deviceName = *it; + string entry = deviceName + " " + startup::StartUp::installDir() + "/gluster/brick" + oam.itoa(brickID) + " " + deviceType + " defaults 1 2"; + //send update pm + oam.distributeFstabUpdates(entry, moduleName); + } } + string command = startup::StartUp::installDir() + "/bin/remote_command.sh " + (*hostConfigIter).IPAddr + " " + password + " 'mkdir -p " + startup::StartUp::installDir() + "/gluster/brick" + oam.itoa(brickID) + "'"; + int status = system(command.c_str()); + brickID++; } } diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index ba9dd8458..ee1abd2ae 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -118,6 +118,8 @@ bool attachVolume(string instanceName, string volumeName, string deviceName, str void remoteInstallThread(void *); +bool glusterSetup(string password); + typedef struct ModuleIP_struct { std::string IPaddress; @@ -3171,8 +3173,6 @@ int main(int argc, char *argv[]) } //configure data redundancy - string glusterconfig = installDir + "/bin/glusterconf"; - if (gluster ) { cout << endl; @@ -3182,7 +3182,7 @@ int main(int argc, char *argv[]) while(true) { - pcommand = callReadline("Would you like to configure MariaDB ColumnStore GlusterFS Data Redundancy? [y,n] (" + start + ") > "); + pcommand = callReadline("Would you like to configure MariaDB ColumnStore Data Redundancy? [y,n] (" + start + ") > "); if (pcommand) { if (strlen(pcommand) > 0) start = pcommand; @@ -3199,12 +3199,12 @@ int main(int argc, char *argv[]) if ( start == "y" ) { cout << endl << "===== Configuring MariaDB ColumnStore Data Redundancy Functionality =====" << endl << endl; - int ret = system(glusterconfig.c_str()); - if ( WEXITSTATUS(ret) != 0 ) + if (!glusterSetup(password)) { - cerr << endl << "There was an error in the Data Redundancy setup, exiting..." << endl; + cout << "ERROR: Problem setting up ColumnStore Data Redundancy" << endl; exit(1); } + } } @@ -3424,10 +3424,35 @@ int main(int argc, char *argv[]) cout << endl << "MariaDB ColumnStore Database Platform Starting, please wait ."; cout.flush(); - + if ( waitForActive() ) { cout << " DONE" << endl; + // IF gluster is enabled we need to modify fstab on remote systems. + if (gluster ) + { + int numberDBRootsPerPM = DBRootCount/pmNumber; + for (int pm=0; pm < pmNumber; pm++) + { + string pmIPaddr = "ModuleIPAddr"+oam.itoa(pm+1)+"-1-3"; + string moduleIPAddr = sysConfig->getConfig("DataRedundancyConfig",pmIPaddr); + if ( moduleIPAddr.empty() ) + { + moduleIPAddr = sysConfig->getConfig("SystemModuleConfig",pmIPaddr); + } + for (int i=0; igetConfig("SystemModuleConfig",ModuleDBRootID); + string command = "" + moduleIPAddr + + ":/dbroot" + dbr + " " + installDir + "/data" + dbr + + " glusterfs defaults,direct-io-mode=enable 00"; + string toPM = "pm" + oam.itoa(pm+1); + oam.distributeFstabUpdates(command,toPM); + } + } + } + if (hdfs) cmd = "bash -c '. " + installDir + "/bin/" + DataFileEnvFile + ";" + installDir + "/bin/dbbuilder 7 > /tmp/dbbuilder.log'"; else @@ -4407,28 +4432,23 @@ bool storageSetup(bool amazonInstall) } //check if gluster is installed - if (rootUser) - system("which gluster > /tmp/gluster.log 2>&1"); - else - system("sudo which gluster > /tmp/gluster.log 2>&1"); - - ifstream in("/tmp/gluster.log"); - - in.seekg(0, std::ios::end); - int size = in.tellg(); - if ( size == 0 || oam.checkLogStatus("/tmp/gluster.log", "no gluster")) - // no gluster - size=0; - else + int rtnCode = system("gluster --version > /tmp/gluster.log 2>&1"); + if (rtnCode == 0) + { glusterInstalled = "y"; + } + else + { + glusterInstalled = "n"; + } //check if hadoop is installed system("which hadoop > /tmp/hadoop.log 2>&1"); - ifstream in1("/tmp/hadoop.log"); + ifstream in("/tmp/hadoop.log"); - in1.seekg(0, std::ios::end); - size = in1.tellg(); + in.seekg(0, std::ios::end); + int size = in.tellg(); if ( size == 0 || oam.checkLogStatus("/tmp/hadoop.log", "no hadoop")) // no hadoop size=0; @@ -4458,8 +4478,8 @@ bool storageSetup(bool amazonInstall) if ( glusterInstalled == "y" && hadoopInstalled == "n" ) { - cout << "There are 3 options when configuring the storage: internal, external, or gluster" << endl << endl; - prompt = "Select the type of Data Storage [1=internal, 2=external, 3=GlusterFS] (" + storageType + ") > "; + cout << "There are 3 options when configuring the storage: internal, external, or DataRedundancy" << endl << endl; + prompt = "Select the type of Data Storage [1=internal, 2=external, 3=DataRedundancy] (" + storageType + ") > "; } if ( glusterInstalled == "n" && hadoopInstalled == "y" ) @@ -4470,8 +4490,8 @@ bool storageSetup(bool amazonInstall) if ( glusterInstalled == "y" && hadoopInstalled == "y" ) { - cout << "There are 5 options when configuring the storage: internal, external, GlusterFS, or hdfs" << endl << endl; - prompt = "Select the type of Data Storage [1=internal, 2=external, 3=GlusterFS, 4=hdfs] (" + storageType + ") > "; + cout << "There are 5 options when configuring the storage: internal, external, DataRedundancy, or hdfs" << endl << endl; + prompt = "Select the type of Data Storage [1=internal, 2=external, 3=DataRedundancy, 4=hdfs] (" + storageType + ") > "; } cout << " 'internal' - This is specified when a local disk is used for the DBRoot storage." << endl; @@ -4481,8 +4501,8 @@ bool storageSetup(bool amazonInstall) if ( glusterInstalled == "y" ) { - cout << " 'GlusterFS' - This is specified when GlusterFS Data Redundancy is installed" << endl; - cout << " and you want the DBRoot directories to be controlled by glusterfs." << endl; + cout << " 'DataRedundancy' - This is specified when gluster is installed and you want" << endl; + cout << " the DBRoot directories to be controlled by ColumnStore Data Redundancy." << endl; cout << " High Availability Server Failover is Supported in this mode." << endl << endl; } @@ -5374,6 +5394,493 @@ std::string launchInstance(ModuleIP moduleip) return instanceName; } +bool glusterSetup(string password) { + + Oam oam; + int dataRedundancyCopies = 0; + int dataRedundancyNetwork = 0; + int dataRedundancyStorage = 0; + int numberDBRootsPerPM = DBRootCount/pmNumber; + int numberBricksPM = 0; + std::vector dbrootPms[DBRootCount]; + DataRedundancyConfig DataRedundancyConfigs[pmNumber]; + string command = ""; + string remoteCommand = installDir + "/bin/remote_command.sh "; + // how many copies? + cout << endl; + cout << "Setup the Number of Copies: This is the total number of copies of the data" << endl; + cout << "in the system and a non-redundant system has 1 copy, so choose 2 or more," << endl; + cout << "but not more than the number of PMs which is " + oam.itoa(pmNumber) + "." << endl; + + while(dataRedundancyCopies < 2 || dataRedundancyCopies > pmNumber) + { + dataRedundancyCopies = 2; + prompt = "Enter Number of Copies [2-" + oam.itoa(pmNumber) + "] ("+ oam.itoa(dataRedundancyCopies) +") > "; + pcommand = callReadline(prompt.c_str()); + if (pcommand) { + if (strlen(pcommand) > 0) dataRedundancyCopies = atoi(pcommand); + callFree(pcommand); + } + + if ( dataRedundancyCopies < 2 || dataRedundancyCopies > pmNumber ) { + cout << endl << "ERROR: Invalid Copy Count '" + oam.itoa(dataRedundancyCopies) + "', please re-enter" << endl << endl; + if ( noPrompting ) + exit(1); + continue; + } + + //update count + try { + sysConfig->setConfig(InstallSection, "DataRedundancyCopies", oam.itoa(dataRedundancyCopies)); + } + catch(...) + { + cout << "ERROR: Problem setting DataRedundancyCopies in the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + } + + numberBricksPM = numberDBRootsPerPM * dataRedundancyCopies; + + cout << endl; + cout << "You can choose to run redundancy over the existing network that ColumnStore " << endl; + cout << "is currently using or you can configure a dedicated redundancy network. " << endl; + cout << "If you choose a dedicated redundancy network, you will need to provide " << endl; + cout << "hostname and IP address information." << endl; + while( dataRedundancyNetwork != 1 && dataRedundancyNetwork != 2 ) + { + dataRedundancyNetwork = 1; + prompt = "Select the data redundancy network [1=existing, 2=dedicated] (" + oam.itoa(dataRedundancyNetwork) + ") > "; + pcommand = callReadline(prompt.c_str()); + if (pcommand) + { + if (strlen(pcommand) > 0) dataRedundancyNetwork = atoi(pcommand);; + callFree(pcommand); + } + + if ( dataRedundancyNetwork != 1 && dataRedundancyNetwork != 2 ) { + cout << "Invalid Entry, please re-enter" << endl << endl; + dataRedundancyNetwork = 0; + if ( noPrompting ) + exit(1); + continue; + } + //update network type + try { + sysConfig->setConfig(InstallSection, "DataRedundancyNetworkType", oam.itoa(dataRedundancyNetwork)); + } + catch(...) + { + cout << "ERROR: Problem setting DataRedundancyNetworkType in the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + } + + if (dataRedundancyNetwork == 2) + { + //loop through pms and get hostname and IP address for each + for (int pm=0; pm < pmNumber; pm++) + { + DataRedundancyConfigs[pm].pmID = pm + 1; + string dataDupIPaddr = "ModuleIPAddr"+oam.itoa(DataRedundancyConfigs[pm].pmID)+"-1-3"; + string dataDupHostName = "ModuleHostName"+oam.itoa(DataRedundancyConfigs[pm].pmID)+"-1-3"; + string moduleIPAddr = sysConfig->getConfig("DataRedundancyConfig",dataDupIPaddr); + string moduleHostName = sysConfig->getConfig("DataRedundancyConfig",dataDupHostName); + if ( moduleIPAddr.empty() ) + { + moduleIPAddr = oam::UnassignedIpAddr; + } + if ( moduleHostName.empty() ) + { + moduleHostName = oam::UnassignedName; + } + + prompt = "Enter PM #" + oam.itoa(DataRedundancyConfigs[pm].pmID) + " Host Name (" + moduleHostName + ") > "; + pcommand = callReadline(prompt.c_str()); + if (pcommand) + { + if (strlen(pcommand) > 0) + { + moduleHostName = pcommand; + moduleIPAddr = oam::UnassignedIpAddr; + } + callFree(pcommand); + } + + if ( moduleIPAddr == oam::UnassignedIpAddr ) + { + //get IP Address + string IPAddress = oam.getIPAddress( moduleHostName); + if ( !IPAddress.empty() ) + moduleIPAddr = IPAddress; + else + moduleIPAddr = oam::UnassignedIpAddr; + } + + if ( moduleIPAddr == "127.0.0.1") + moduleIPAddr = "unassigned"; + + //prompt for IP address + while (true) + { + prompt = "Enter PM #" + oam.itoa(DataRedundancyConfigs[pm].pmID) + " IP Address of " + moduleHostName + " (" + moduleIPAddr + ") > "; + pcommand = callReadline(prompt.c_str()); + if (pcommand) + { + if (strlen(pcommand) > 0) moduleIPAddr = pcommand; + callFree(pcommand); + } + + if (moduleIPAddr == "127.0.0.1" || moduleIPAddr == "0.0.0.0" || moduleIPAddr == "128.0.0.1") { + cout << endl << moduleIPAddr + " is an Invalid IP Address for a multi-server system, please re-enter" << endl << endl; + moduleIPAddr = "unassigned"; + if ( noPrompting ) + exit(1); + continue; + } + + if (oam.isValidIP(moduleIPAddr)) { + + // run ping test to validate + string cmdLine = "ping "; + string cmdOption = " -c 1 -w 5 >> /dev/null"; + string cmd = cmdLine + moduleIPAddr + cmdOption; + int rtnCode = system(cmd.c_str()); + if ( WEXITSTATUS(rtnCode) != 0 ) { + //NIC failed to respond to ping + string temp = "2"; + while (true) + { + cout << endl; + prompt = "IP Address of '" + moduleIPAddr + "' failed ping test, please validate. Do you want to continue or re-enter [1=continue, 2=re-enter] (2) > "; + if ( noPrompting ) + exit(1); + pcommand = callReadline(prompt.c_str()); + if (pcommand) + { + if (strlen(pcommand) > 0) temp = pcommand; + callFree(pcommand); + } + + if ( temp == "1" || temp == "2") + break; + else + { + temp = "2"; + cout << endl << "Invalid entry, please re-enter" << endl; + if ( noPrompting ) + exit(1); + } + } + cout << endl; + if ( temp == "1") + break; + } + else // good ping + break; + } + else + { + cout << endl << "Invalid IP Address format, xxx.xxx.xxx.xxx, please re-enter" << endl << endl; + if ( noPrompting ) + exit(1); + } + } + //set Host Name and IP + try { + sysConfig->setConfig("DataRedundancyConfig", dataDupHostName, moduleHostName); + sysConfig->setConfig("DataRedundancyConfig", dataDupIPaddr, moduleIPAddr); + DataRedundancyConfigs[pm].pmID = pm + 1; + DataRedundancyConfigs[pm].pmIpAddr = moduleIPAddr; + DataRedundancyConfigs[pm].pmHostname = moduleHostName; + } + catch(...) + { + cout << "ERROR: Problem setting Host Name in the MariaDB ColumnStore Data Redundancy Configuration" << endl; + exit(1); + } + } + } + else + { + for (int pm=0; pm < pmNumber; pm++) + { + string pmIPaddr = "ModuleIPAddr"+oam.itoa(pm+1)+"-1-3"; + string pmHostName = "ModuleHostName"+oam.itoa(pm+1)+"-1-3"; + DataRedundancyConfigs[pm].pmID = pm + 1; + DataRedundancyConfigs[pm].pmIpAddr = sysConfig->getConfig("SystemModuleConfig",pmIPaddr); + DataRedundancyConfigs[pm].pmHostname = sysConfig->getConfig("SystemModuleConfig",pmHostName); + } + } + cout << endl; + cout << "OK. You have " + oam.itoa(pmNumber) + " PMs, " + oam.itoa(DBRootCount) + " DBRoots, and you have chosen to keep " + oam.itoa(dataRedundancyCopies) << endl; + cout << "copies of the data. You can choose to place the copies in " << endl; + cout << "/gluster or you can specify individual, mountable device names. " << endl; + /// [directory,storage] (directory) > + while( dataRedundancyStorage != 1 && dataRedundancyStorage != 2 ) + { + dataRedundancyStorage = 1; + prompt = "Select the data redundancy network [1=directory, 2=storage] (" + oam.itoa(dataRedundancyStorage) + ") > "; + pcommand = callReadline(prompt.c_str()); + if (pcommand) + { + if (strlen(pcommand) > 0) dataRedundancyStorage = atoi(pcommand);; + callFree(pcommand); + } + + if ( dataRedundancyStorage != 1 && dataRedundancyStorage != 2 ) { + cout << "Invalid Entry, please re-enter" << endl << endl; + dataRedundancyStorage = 0; + if ( noPrompting ) + exit(1); + continue; + } + //update network type + try { + sysConfig->setConfig(InstallSection, "DataRedundancyStorageType", oam.itoa(dataRedundancyStorage)); + } + catch(...) + { + cout << "ERROR: Problem setting DataRedundancyStorageType in the MariaDB ColumnStore System Configuration file" << endl; + exit(1); + } + } + + if (dataRedundancyStorage == 2) + { + int numberStorageLocations = DBRootCount*dataRedundancyCopies; + cout << endl; + cout << "You will need " + oam.itoa(numberStorageLocations) + "total storage locations," << endl; + cout << "and " + oam.itoa(numberBricksPM) + " storage locations per PM. You will now " << endl; + cout << "be asked to enter the device names for the storage locations." << endl; + + //loop through pms and get storage locations for each + for (int pm=0; pm < pmNumber; pm++) + { + for (int brick=0; brick < numberBricksPM; brick++) + { + prompt = "Enter a storage locations for PM#" + oam.itoa(DataRedundancyConfigs[pm].pmID) + " brick#" + oam.itoa(brick) + " : "; + pcommand = callReadline(prompt.c_str()); + if (pcommand) + { + if (strlen(pcommand) > 0) + { + DataRedundancyStorageSetup dataredundancyStorageItem; + dataredundancyStorageItem.brickID = brick; + dataredundancyStorageItem.storageLocation = pcommand; + //get filesystem type + prompt = "Enter a filesystem type for storage location " + dataredundancyStorageItem.storageLocation + " (ext2,ext3,xfs,etc): "; + pcommand = callReadline(prompt.c_str()); + if (pcommand) + { + if (strlen(pcommand) > 0) + { + dataredundancyStorageItem.storageFilesytemType = pcommand; + } + callFree(pcommand); + } + DataRedundancyConfigs[pm].storageLocations.push_back(dataredundancyStorageItem); + } + callFree(pcommand); + } + } + } + } + + // User config complete setup the gluster bricks + // This will distribute DBRootCopies evenly across PMs + for (int pm=0; pm < pmNumber; pm++) + { + int dbrootCopy = DataRedundancyConfigs[pm].pmID; + int nextPM = DataRedundancyConfigs[pm].pmID; + if (nextPM >= pmNumber) + { + nextPM=0; + } + for ( int i=0; i= pmNumber) + { + nextPM=0; + } + if (nextPM == pm) + { + nextPM++; + } + if (nextPM >= pmNumber) + { + nextPM=0; + } + } + dbrootCopy += pmNumber; + } + } + + // Store to config and distribute so that GLUSTER_WHOHAS will work + for (int db=0; db < DBRootCount; db++) + { + int dbrootID = db + 1; + string dbrootIDPMs = ""; + string moduleDBRootIDPMs = "DBRoot" + oam.itoa(dbrootID) + "PMs"; + for (int pm=0; pm < pmNumber; pm++) + { + if(find(DataRedundancyConfigs[pm].dbrootCopies.begin(),DataRedundancyConfigs[pm].dbrootCopies.end(),dbrootID) != DataRedundancyConfigs[pm].dbrootCopies.end()) + { + //dbrootPms for each dbroot there is a list of PMs for building the gluster volume commands below + dbrootPms[db].push_back(DataRedundancyConfigs[pm].pmID); + dbrootIDPMs += oam.itoa(DataRedundancyConfigs[pm].pmID); + dbrootIDPMs += " "; + } + } + // Store to config and distribute so that GLUSTER_WHOHAS will work + sysConfig->setConfig("DataRedundancyConfig", moduleDBRootIDPMs, dbrootIDPMs); + } + + int status; + for (int pm=0; pm < pmNumber; pm++) + { + for ( int brick=1; brick<=numberBricksPM; brick++) + { + // create the gluster brick directories now + command = remoteCommand + DataRedundancyConfigs[pm].pmIpAddr + " " + password + " 'mkdir -p " + installDir + "/gluster/brick" + oam.itoa(brick) + "'"; + status = system(command.c_str()); + if (WEXITSTATUS(status) != 0 ) + { + cout << "ERROR: command failed: " << command << endl; + exit(1); + } + if (dataRedundancyStorage == 2) + { + //walk data storage locations and modify fstab to reflect the storage locations entered by user + vector::iterator storageSetupIter=DataRedundancyConfigs[pm].storageLocations.begin(); + for (; storageSetupIter < DataRedundancyConfigs[pm].storageLocations.end(); storageSetupIter++ ) + { + if (rootUser) + { + command = remoteCommand + DataRedundancyConfigs[pm].pmIpAddr + " " + password + + " 'echo " + (*storageSetupIter).storageLocation + " " + + installDir + "/gluster/brick" + oam.itoa(brick) + " " + + (*storageSetupIter).storageFilesytemType + " defaults 1 2 >> /etc/fstab'"; + } + else + { + command = remoteCommand + DataRedundancyConfigs[pm].pmIpAddr + " " + password + + " 'sudo bash -c `sudo echo " + (*storageSetupIter).storageLocation + " " + + installDir + "/gluster/brick" + oam.itoa(brick) + " " + + (*storageSetupIter).storageFilesytemType + " defaults 1 2 >> /etc/fstab`'"; + } + status = system(command.c_str()); + if (WEXITSTATUS(status) != 0 ) + { + cout << "ERROR: command failed: " << command << endl; + exit(1); + } + if (rootUser) + { + command = remoteCommand + DataRedundancyConfigs[pm].pmIpAddr + " " + password + + " 'mount " + installDir + "/gluster/brick" + oam.itoa(brick) + "'"; + } + else + { + command = remoteCommand + DataRedundancyConfigs[pm].pmIpAddr + " " + password + + " 'sudo bash -c `sudo mount " + installDir + "/gluster/brick" + oam.itoa(brick) + "`'"; + } + status = system(command.c_str()); + if (WEXITSTATUS(status) != 0 ) + { + cout << "ERROR: command failed: " << command << endl; + exit(1); + } + if (!rootUser) + { + int user; + user = getuid(); + command = remoteCommand + DataRedundancyConfigs[pm].pmIpAddr + " " + password + + "'sudo bash -c `sudo chown -R " + oam.itoa(user) + ":" + oam.itoa(user) + " " + installDir + "/gluster/brick" + oam.itoa(brick) + "`'"; + status = system(command.c_str()); + if (WEXITSTATUS(status) != 0 ) + { + cout << "ERROR(" << status <<"): command failed: " << command << endl; + } + } + + } + } + } + if (rootUser) + { + command = "gluster peer probe " + DataRedundancyConfigs[pm].pmIpAddr; + } + else + { + command = "sudo gluster peer probe " + DataRedundancyConfigs[pm].pmIpAddr; + } + status = system(command.c_str()); + if (WEXITSTATUS(status) != 0 ) + { + cout << "ERROR: command failed: " << command << endl; + exit(1); + } + } + sleep(5); + command = "gluster peer status "; + status = system(command.c_str()); + if (WEXITSTATUS(status) != 0 ) + { + cout << "ERROR: command failed: " << command << endl; + exit(1); + } + //Need to wait since peer probe success does not always mean it is ready for volume create command + //TODO: figureout a cleaner way to do this. + sleep(10); + // Build the gluster volumes and start them for each dbroot + for (int db=0; db < DBRootCount; db++) + { + int dbrootID = db + 1; + if (rootUser) + { + command = "gluster volume create dbroot" + oam.itoa(dbrootID) + " transport tcp replica " + oam.itoa(dataRedundancyCopies) + " "; + } + else + { + command = "sudo gluster volume create dbroot" + oam.itoa(dbrootID) + " transport tcp replica " + oam.itoa(dataRedundancyCopies) + " "; + } + vector::iterator dbrootPmIter = dbrootPms[db].begin(); + for (; dbrootPmIter < dbrootPms[db].end(); dbrootPmIter++ ) + { + int pm = (*dbrootPmIter) - 1; + command += DataRedundancyConfigs[pm].pmIpAddr + ":" + installDir +"/gluster/brick" + oam.itoa(dbrootID) + " "; + } + command += "force"; + status = system(command.c_str()); + if (WEXITSTATUS(status) != 0 ) + { + cout << "ERROR: command failed: " << command << endl; + exit(1); + } + if (rootUser) + { + command = "gluster volume start dbroot" + oam.itoa(dbrootID); + } + else + { + command = "sudo gluster volume start dbroot" + oam.itoa(dbrootID); + } + status = system(command.c_str()); + if (WEXITSTATUS(status) != 0 ) + { + cout << "ERROR: command failed: " << command << endl; + exit(1); + } + } + + return true; +} // vim:ts=4 sw=4: diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp index dffcfbc25..dddc7adb8 100644 --- a/procmgr/processmanager.cpp +++ b/procmgr/processmanager.cpp @@ -5088,7 +5088,7 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str string logFile = "/tmp/" + remoteModuleName + "_user_installer.log"; log.writeLog(__LINE__, "addModule - user_installer run for " + remoteModuleName, LOG_TYPE_DEBUG); - string cmd = installDir + "/bin/user_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " + password + " " + version + " initial " + AmazonInstall + " " + packageType + " --nodeps none 1 > " + logFile; + string cmd = installDir + "/bin/user_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " + password + " " + version + " initial " + AmazonInstall + " " + packageType + " --nodeps 1 > " + logFile; log.writeLog(__LINE__, "addModule cmd: " + cmd, LOG_TYPE_DEBUG); @@ -5184,8 +5184,6 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str string cmd = installDir + "/bin/performance_installer.sh " + remoteModuleName + " " + remoteModuleIP + " " + password + " " + version + " initial " + AmazonInstall + " " + packageType + + " --nodeps 1 > " + logFile; log.writeLog(__LINE__, "addModule cmd: " + cmd, LOG_TYPE_DEBUG); - system(cmd.c_str()); - bool passed = false; for ( int retry = 0 ; retry < 20 ; retry++ ) { diff --git a/procmon/processmonitor.cpp b/procmon/processmonitor.cpp index 4ee884f89..f735923ef 100644 --- a/procmon/processmonitor.cpp +++ b/procmon/processmonitor.cpp @@ -1734,7 +1734,12 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO int status = system(cmd.c_str()); if (WEXITSTATUS(status) != 0 ) { - cmd = "echo " + entry + " >> /etc/fstab"; + if ( rootUser) { + cmd = "echo " + entry + " >> /etc/fstab"; + } + else { + cmd = "sudo echo " + entry + " >> /etc/fstab"; + } system(cmd.c_str()); log.writeLog(__LINE__, "Add line entry to /etc/fstab : " + entry); @@ -1745,7 +1750,12 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO status = system(cmd.c_str()); if (WEXITSTATUS(status) != 0 ) { - cmd = "echo " + entry + " >> " + startup::StartUp::installDir() + "/local/etc/pm1/fstab"; + if ( rootUser) { + cmd = "echo " + entry + " >> " + startup::StartUp::installDir() + "/local/etc/pm1/fstab"; + } + else { + cmd = "sudo echo " + entry + " >> " + startup::StartUp::installDir() + "/local/etc/pm1/fstab"; + } system(cmd.c_str()); log.writeLog(__LINE__, "Add line entry to ../local/etc/pm1/fstab : " + entry); @@ -1755,7 +1765,12 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO string::size_type pos = entry.find(" ",0); string::size_type pos1 = entry.find(" ",pos+1); string directory = entry.substr(pos+1,pos1-pos); - cmd = "mkdir " + directory; + if ( rootUser) { + cmd = "mkdir " + directory; + } + else { + cmd = "sudo mkdir " + directory; + } system(cmd.c_str()); log.writeLog(__LINE__, "create directory: " + directory); @@ -6087,28 +6102,42 @@ void ProcessMonitor::flushInodeCache() int ProcessMonitor::glusterAssign(std::string dbrootID) { Oam oam; - - log.writeLog(__LINE__, "glusterAssign called : " + dbrootID, LOG_TYPE_DEBUG); + Config* sysConfig = Config::makeConfig(); std::string errmsg = ""; - string glustercmd = startup::StartUp::installDir() + "/bin/glusterctl"; + log.writeLog(__LINE__, "glusterAssign called : " + dbrootID, LOG_TYPE_DEBUG); + string pmid = oam.itoa(config.moduleID()); + string dataDupIPaddr = "ModuleIPAddr"+pmid+"-1-3"; + string moduleIPAddr = sysConfig->getConfig("DataRedundancyConfig",dataDupIPaddr); - glustercmd = glustercmd + " assign " + dbrootID + " " + pmid + " > /tmp/gluster_assign.log 2>&1"; - int ret; - ret = system(glustercmd.c_str()); - if ( WEXITSTATUS(ret) == 0 ) - return oam::API_SUCCESS; + if (moduleIPAddr.empty() || moduleIPAddr == oam::UnassignedIpAddr) + { + moduleIPAddr = sysConfig->getConfig("SystemModuleConfig",dataDupIPaddr); + } + string command = "mount -tglusterfs -odirect-io-mode=enable " + moduleIPAddr + ":/dbroot" + + dbrootID + " " + startup::StartUp::installDir() + "/data" + dbrootID + " > /tmp/glusterAssign.txt 2>&1"; - ret = oam.checkGlusterLog("/tmp/gluster_assign.log", errmsg); - if ( ret == 0 ) - // OK return - return oam::API_SUCCESS; - else - log.writeLog(__LINE__, "glusterAssign failed, check /tmp/gluster_assign.log", LOG_TYPE_ERROR); + int ret = system(command.c_str()); + + if ( WEXITSTATUS(ret) != 0 ) + { + ifstream in("/tmp/glusterAssign.txt"); + in.seekg(0, std::ios::end); + int size = in.tellg(); + if ( size != 0 ) + { + if (!oam.checkLogStatus("/tmp/glusterAssign.txt", "already")) { + log.writeLog(__LINE__, "glusterAssign failed.", LOG_TYPE_ERROR); + system("mv -f /tmp/glusterAssign.txt /tmp/glusterAssign_failed.txt"); + return oam::API_FAILURE; + } + } + } + + return oam::API_SUCCESS; - return oam::API_FAILURE; } /****************************************************************************************** @@ -6122,27 +6151,31 @@ int ProcessMonitor::glusterUnassign(std::string dbrootID) { Oam oam; - log.writeLog(__LINE__, "glusterUnassign called: " + dbrootID, LOG_TYPE_DEBUG); - std::string errmsg = ""; - string glustercmd = startup::StartUp::installDir() + "/bin/glusterctl"; - string pmid = oam.itoa(config.moduleID()); + log.writeLog(__LINE__, "glusterUnassign called: " + dbrootID, LOG_TYPE_DEBUG); - glustercmd = glustercmd + " unassign " + dbrootID + " " + pmid + " > /tmp/gluster_unassign.log 2>&1"; - int ret; - ret = system(glustercmd.c_str()); - if ( WEXITSTATUS(ret) == 0 ) - return oam::API_SUCCESS; + string command = "umount -f " + startup::StartUp::installDir() + "/data" + dbrootID + " > /tmp/glusterUnassign.txt 2>&1"; - ret = oam.checkGlusterLog("/tmp/gluster_unassign.log", errmsg); - if ( ret == 0 ) - // OK return - return oam::API_SUCCESS; - else - log.writeLog(__LINE__, "glusterAssign failed, check /tmp/gluster_assign.log", LOG_TYPE_ERROR); + int ret = system(command.c_str()); + if ( WEXITSTATUS(ret) != 0 ) + { + ifstream in("/tmp/glusterUnassign.txt"); + in.seekg(0, std::ios::end); + int size = in.tellg(); + if ( size != 0 ) + { + if (!oam.checkLogStatus("/tmp/glusterAssign.txt", "not mounted")) { + log.writeLog(__LINE__, "glusterUnassign failed.", LOG_TYPE_ERROR); + system("mv -f /tmp/glusterUnassign.txt /tmp/glusterUnassign_failed.txt"); + return oam::API_FAILURE; + } + } + log.writeLog(__LINE__, "glusterUnassign failed.", LOG_TYPE_ERROR); + return oam::API_FAILURE; + } - return oam::API_FAILURE; + return oam::API_SUCCESS; }